]> git.proxmox.com Git - ceph.git/blame - ceph/src/test/librados/tier_cxx.cc
projects / ceph.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
bump version to 15.2.11-pve1
[ceph.git] / ceph / src / test / librados / tier_cxx.cc
CommitLineData
7c673cae
FG		 1// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2// vim: ts=8 sw=2 smarttab
3#include "gtest/gtest.h"
4
5#include "mds/mdstypes.h"
6#include "include/buffer.h"
7#include "include/rbd_types.h"
7c673cae
FG		 8#include "include/rados/librados.hpp"
9#include "include/stringify.h"
10#include "include/types.h"
11#include "global/global_context.h"
12#include "common/Cond.h"
11fdf7f2
TL		 13#include "test/librados/test_cxx.h"
14#include "test/librados/testcase_cxx.h"
7c673cae 15#include "json_spirit/json_spirit.h"
11fdf7f2 16#include "cls/cas/cls_cas_ops.h"
7c673cae
FG		 17
18#include "osd/HitSet.h"
19
20#include <errno.h>
21#include <map>
22#include <sstream>
23#include <string>
24
25using namespace librados;
26using std::map;
27using std::ostringstream;
28using std::string;
29
30typedef RadosTestPP LibRadosTierPP;
31typedef RadosTestECPP LibRadosTierECPP;
32
33void flush_evict_all(librados::Rados& cluster, librados::IoCtx& cache_ioctx)
34{
35 bufferlist inbl;
36 cache_ioctx.set_namespace(all_nspaces);
37 for (NObjectIterator it = cache_ioctx.nobjects_begin();
38 it != cache_ioctx.nobjects_end(); ++it) {
39 cache_ioctx.locator_set_key(it->get_locator());
40 cache_ioctx.set_namespace(it->get_nspace());
41 {
42 ObjectReadOperation op;
43 op.cache_flush();
44 librados::AioCompletion *completion = cluster.aio_create_completion();
45 cache_ioctx.aio_operate(
46 it->get_oid(), completion, &op,
47 librados::OPERATION_IGNORE_OVERLAY, NULL);
9f95a23c 48 completion->wait_for_complete();
7c673cae
FG		 49 completion->get_return_value();
50 completion->release();
51 }
52 {
53 ObjectReadOperation op;
54 op.cache_evict();
55 librados::AioCompletion *completion = cluster.aio_create_completion();
56 cache_ioctx.aio_operate(
57 it->get_oid(), completion, &op,
58 librados::OPERATION_IGNORE_OVERLAY, NULL);
9f95a23c 59 completion->wait_for_complete();
7c673cae
FG		 60 completion->get_return_value();
61 completion->release();
62 }
63 }
64}
65
9f95a23c
TL		 66static string _get_required_osd_release(Rados& cluster)
67{
68 bufferlist inbl;
69 string cmd = string("{\"prefix\": \"osd dump\",\"format\":\"json\"}");
70 bufferlist outbl;
71 int r = cluster.mon_command(cmd, inbl, &outbl, NULL);
72 ceph_assert(r >= 0);
73 string outstr(outbl.c_str(), outbl.length());
74 json_spirit::Value v;
75 if (!json_spirit::read(outstr, v)) {
76 cerr <<" unable to parse json " << outstr << std::endl;
77 return "";
78 }
79
80 json_spirit::Object& o = v.get_obj();
81 for (json_spirit::Object::size_type i=0; i<o.size(); i++) {
82 json_spirit::Pair& p = o[i];
83 if (p.name_ == "require_osd_release") {
84 cout << "require_osd_release = " << p.value_.get_str() << std::endl;
85 return p.value_.get_str();
86 }
87 }
88 cerr << "didn't find require_osd_release in " << outstr << std::endl;
89 return "";
90}
91
7c673cae
FG		 92class LibRadosTwoPoolsPP : public RadosTestPP
93{
94public:
95 LibRadosTwoPoolsPP() {};
96 ~LibRadosTwoPoolsPP() override {};
97protected:
98 static void SetUpTestCase() {
99 pool_name = get_temp_pool_name();
100 ASSERT_EQ("", create_one_pool_pp(pool_name, s_cluster));
101 }
102 static void TearDownTestCase() {
103 ASSERT_EQ(0, destroy_one_pool_pp(pool_name, s_cluster));
104 }
105 static std::string cache_pool_name;
106
107 void SetUp() override {
108 cache_pool_name = get_temp_pool_name();
109 ASSERT_EQ(0, s_cluster.pool_create(cache_pool_name.c_str()));
110 RadosTestPP::SetUp();
c07f9fc5 111
7c673cae 112 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
c07f9fc5 113 cache_ioctx.application_enable("rados", true);
7c673cae
FG		 114 cache_ioctx.set_namespace(nspace);
115 }
116 void TearDown() override {
117 // flush + evict cache
118 flush_evict_all(cluster, cache_ioctx);
119
120 bufferlist inbl;
121 // tear down tiers
122 ASSERT_EQ(0, cluster.mon_command(
123 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
124 "\"}",
125 inbl, NULL, NULL));
126 ASSERT_EQ(0, cluster.mon_command(
127 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
128 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
129 inbl, NULL, NULL));
130
131 // wait for maps to settle before next test
132 cluster.wait_for_latest_osdmap();
133
134 RadosTestPP::TearDown();
135
136 cleanup_default_namespace(cache_ioctx);
137 cleanup_namespace(cache_ioctx, nspace);
138
139 cache_ioctx.close();
140 ASSERT_EQ(0, s_cluster.pool_delete(cache_pool_name.c_str()));
141 }
142 librados::IoCtx cache_ioctx;
143};
144
145class Completions
146{
147public:
148 Completions() = default;
149 librados::AioCompletion* getCompletion() {
150 librados::AioCompletion* comp = librados::Rados::aio_create_completion();
151 m_completions.push_back(comp);
152 return comp;
153 }
154
155 ~Completions() {
156 for (auto& comp : m_completions) {
157 comp->release();
158 }
159 }
160
161private:
162 vector<librados::AioCompletion *> m_completions;
163};
164
165Completions completions;
166
167std::string LibRadosTwoPoolsPP::cache_pool_name;
168
169TEST_F(LibRadosTierPP, Dirty) {
170 {
171 ObjectWriteOperation op;
172 op.undirty();
173 ASSERT_EQ(0, ioctx.operate("foo", &op)); // still get 0 if it dne
174 }
175 {
176 ObjectWriteOperation op;
177 op.create(true);
178 ASSERT_EQ(0, ioctx.operate("foo", &op));
179 }
180 {
181 bool dirty = false;
182 int r = -1;
183 ObjectReadOperation op;
184 op.is_dirty(&dirty, &r);
185 ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
186 ASSERT_TRUE(dirty);
187 ASSERT_EQ(0, r);
188 }
189 {
190 ObjectWriteOperation op;
191 op.undirty();
192 ASSERT_EQ(0, ioctx.operate("foo", &op));
193 }
194 {
195 ObjectWriteOperation op;
196 op.undirty();
197 ASSERT_EQ(0, ioctx.operate("foo", &op)); // still 0 if already clean
198 }
199 {
200 bool dirty = false;
201 int r = -1;
202 ObjectReadOperation op;
203 op.is_dirty(&dirty, &r);
204 ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
205 ASSERT_FALSE(dirty);
206 ASSERT_EQ(0, r);
207 }
208 {
209 ObjectWriteOperation op;
210 op.truncate(0); // still a write even tho it is a no-op
211 ASSERT_EQ(0, ioctx.operate("foo", &op));
212 }
213 {
214 bool dirty = false;
215 int r = -1;
216 ObjectReadOperation op;
217 op.is_dirty(&dirty, &r);
218 ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
219 ASSERT_TRUE(dirty);
220 ASSERT_EQ(0, r);
221 }
222}
223
224TEST_F(LibRadosTwoPoolsPP, Overlay) {
225 // create objects
226 {
227 bufferlist bl;
228 bl.append("base");
229 ObjectWriteOperation op;
230 op.write_full(bl);
231 ASSERT_EQ(0, ioctx.operate("foo", &op));
232 }
233 {
234 bufferlist bl;
235 bl.append("cache");
236 ObjectWriteOperation op;
237 op.write_full(bl);
238 ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
239 }
240
241 // configure cache
242 bufferlist inbl;
243 ASSERT_EQ(0, cluster.mon_command(
244 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
245 "\", \"tierpool\": \"" + cache_pool_name +
246 "\", \"force_nonempty\": \"--force-nonempty\" }",
247 inbl, NULL, NULL));
248 ASSERT_EQ(0, cluster.mon_command(
249 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
250 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
251 inbl, NULL, NULL));
252
253 // wait for maps to settle
254 cluster.wait_for_latest_osdmap();
255
256 // by default, the overlay sends us to cache pool
257 {
258 bufferlist bl;
259 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
260 ASSERT_EQ('c', bl[0]);
261 }
262 {
263 bufferlist bl;
264 ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
265 ASSERT_EQ('c', bl[0]);
266 }
267
268 // unless we say otherwise
269 {
270 bufferlist bl;
271 ObjectReadOperation op;
272 op.read(0, 1, &bl, NULL);
273 librados::AioCompletion *completion = cluster.aio_create_completion();
274 ASSERT_EQ(0, ioctx.aio_operate(
275 "foo", completion, &op,
276 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 277 completion->wait_for_complete();
7c673cae
FG		 278 ASSERT_EQ(0, completion->get_return_value());
279 completion->release();
280 ASSERT_EQ('b', bl[0]);
281 }
282}
283
284TEST_F(LibRadosTwoPoolsPP, Promote) {
285 // create object
286 {
287 bufferlist bl;
288 bl.append("hi there");
289 ObjectWriteOperation op;
290 op.write_full(bl);
291 ASSERT_EQ(0, ioctx.operate("foo", &op));
292 }
293
294 // configure cache
295 bufferlist inbl;
296 ASSERT_EQ(0, cluster.mon_command(
297 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
298 "\", \"tierpool\": \"" + cache_pool_name +
299 "\", \"force_nonempty\": \"--force-nonempty\" }",
300 inbl, NULL, NULL));
301 ASSERT_EQ(0, cluster.mon_command(
302 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
303 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
304 inbl, NULL, NULL));
305 ASSERT_EQ(0, cluster.mon_command(
306 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
307 "\", \"mode\": \"writeback\"}",
308 inbl, NULL, NULL));
309
310 // wait for maps to settle
311 cluster.wait_for_latest_osdmap();
312
313 // read, trigger a promote
314 {
315 bufferlist bl;
316 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
317 }
318
319 // read, trigger a whiteout
320 {
321 bufferlist bl;
322 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
323 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
324 }
325
326 // verify the object is present in the cache tier
327 {
328 NObjectIterator it = cache_ioctx.nobjects_begin();
329 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
330 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
331 ++it;
332 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
333 ++it;
334 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
335 }
336}
337
338TEST_F(LibRadosTwoPoolsPP, PromoteSnap) {
339 // create object
340 {
341 bufferlist bl;
342 bl.append("hi there");
343 ObjectWriteOperation op;
344 op.write_full(bl);
345 ASSERT_EQ(0, ioctx.operate("foo", &op));
346 }
347 {
348 bufferlist bl;
349 bl.append("hi there");
350 ObjectWriteOperation op;
351 op.write_full(bl);
352 ASSERT_EQ(0, ioctx.operate("bar", &op));
353 }
354 {
355 bufferlist bl;
356 bl.append("hi there");
357 ObjectWriteOperation op;
358 op.write_full(bl);
359 ASSERT_EQ(0, ioctx.operate("baz", &op));
360 }
361 {
362 bufferlist bl;
363 bl.append("hi there");
364 ObjectWriteOperation op;
365 op.write_full(bl);
366 ASSERT_EQ(0, ioctx.operate("bam", &op));
367 }
368
369 // create a snapshot, clone
370 vector<uint64_t> my_snaps(1);
371 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
372 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
373 my_snaps));
374 {
375 bufferlist bl;
376 bl.append("ciao!");
377 ObjectWriteOperation op;
378 op.write_full(bl);
379 ASSERT_EQ(0, ioctx.operate("foo", &op));
380 }
381 {
382 bufferlist bl;
383 bl.append("ciao!");
384 ObjectWriteOperation op;
385 op.write_full(bl);
386 ASSERT_EQ(0, ioctx.operate("bar", &op));
387 }
388 {
389 ObjectWriteOperation op;
390 op.remove();
391 ASSERT_EQ(0, ioctx.operate("baz", &op));
392 }
393 {
394 bufferlist bl;
395 bl.append("ciao!");
396 ObjectWriteOperation op;
397 op.write_full(bl);
398 ASSERT_EQ(0, ioctx.operate("bam", &op));
399 }
400
401 // configure cache
402 bufferlist inbl;
403 ASSERT_EQ(0, cluster.mon_command(
404 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
405 "\", \"tierpool\": \"" + cache_pool_name +
406 "\", \"force_nonempty\": \"--force-nonempty\" }",
407 inbl, NULL, NULL));
408 ASSERT_EQ(0, cluster.mon_command(
409 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
410 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
411 inbl, NULL, NULL));
412 ASSERT_EQ(0, cluster.mon_command(
413 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
414 "\", \"mode\": \"writeback\"}",
415 inbl, NULL, NULL));
416
417 // wait for maps to settle
418 cluster.wait_for_latest_osdmap();
419
420 // read, trigger a promote on the head
421 {
422 bufferlist bl;
423 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
424 ASSERT_EQ('c', bl[0]);
425 }
426 {
427 bufferlist bl;
428 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
429 ASSERT_EQ('c', bl[0]);
430 }
431
432 ioctx.snap_set_read(my_snaps[0]);
433
434 // read foo snap
435 {
436 bufferlist bl;
437 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
438 ASSERT_EQ('h', bl[0]);
439 }
440
441 // read bar snap
442 {
443 bufferlist bl;
444 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
445 ASSERT_EQ('h', bl[0]);
446 }
447
448 // read baz snap
449 {
450 bufferlist bl;
451 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
452 ASSERT_EQ('h', bl[0]);
453 }
454
455 ioctx.snap_set_read(librados::SNAP_HEAD);
456
457 // read foo
458 {
459 bufferlist bl;
460 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
461 ASSERT_EQ('c', bl[0]);
462 }
463
464 // read bar
465 {
466 bufferlist bl;
467 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
468 ASSERT_EQ('c', bl[0]);
469 }
470
471 // read baz
472 {
473 bufferlist bl;
474 ASSERT_EQ(-ENOENT, ioctx.read("baz", bl, 1, 0));
475 }
476
477 // cleanup
478 ioctx.selfmanaged_snap_remove(my_snaps[0]);
479}
480
481TEST_F(LibRadosTwoPoolsPP, PromoteSnapScrub) {
482 int num = 100;
483
484 // create objects
485 for (int i=0; i<num; ++i) {
486 bufferlist bl;
487 bl.append("hi there");
488 ObjectWriteOperation op;
489 op.write_full(bl);
490 ASSERT_EQ(0, ioctx.operate(string("foo") + stringify(i), &op));
491 }
492
493 vector<uint64_t> my_snaps;
494 for (int snap=0; snap<4; ++snap) {
495 // create a snapshot, clone
496 vector<uint64_t> ns(1);
497 ns.insert(ns.end(), my_snaps.begin(), my_snaps.end());
498 my_snaps.swap(ns);
499 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
500 cout << "my_snaps " << my_snaps << std::endl;
501 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
502 my_snaps));
503 for (int i=0; i<num; ++i) {
504 bufferlist bl;
505 bl.append(string("ciao! snap") + stringify(snap));
506 ObjectWriteOperation op;
507 op.write_full(bl);
508 ASSERT_EQ(0, ioctx.operate(string("foo") + stringify(i), &op));
509 }
510 }
511
512 // configure cache
513 bufferlist inbl;
514 ASSERT_EQ(0, cluster.mon_command(
515 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
516 "\", \"tierpool\": \"" + cache_pool_name +
517 "\", \"force_nonempty\": \"--force-nonempty\" }",
518 inbl, NULL, NULL));
519 ASSERT_EQ(0, cluster.mon_command(
520 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
521 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
522 inbl, NULL, NULL));
523 ASSERT_EQ(0, cluster.mon_command(
524 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
525 "\", \"mode\": \"writeback\"}",
526 inbl, NULL, NULL));
527
528 // wait for maps to settle
529 cluster.wait_for_latest_osdmap();
530
531 // read, trigger a promote on _some_ heads to make sure we handle cases
532 // where snaps are present and where they are not.
533 cout << "promoting some heads" << std::endl;
534 for (int i=0; i<num; ++i) {
535 if (i % 5 == 0 || i > num - 3) {
536 bufferlist bl;
537 ASSERT_EQ(1, ioctx.read(string("foo") + stringify(i), bl, 1, 0));
538 ASSERT_EQ('c', bl[0]);
539 }
540 }
541
542 for (unsigned snap = 0; snap < my_snaps.size(); ++snap) {
543 cout << "promoting from clones for snap " << my_snaps[snap] << std::endl;
544 ioctx.snap_set_read(my_snaps[snap]);
545
546 // read some snaps, semi-randomly
547 for (int i=0; i<50; ++i) {
548 bufferlist bl;
549 string o = string("foo") + stringify((snap * i * 137) % 80);
550 //cout << o << std::endl;
551 ASSERT_EQ(1, ioctx.read(o, bl, 1, 0));
552 }
553 }
554
555 // ok, stop and scrub this pool (to make sure scrub can handle
556 // missing clones in the cache tier).
557 {
558 IoCtx cache_ioctx;
559 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
560 for (int i=0; i<10; ++i) {
561 do {
562 ostringstream ss;
563 ss << "{\"prefix\": \"pg scrub\", \"pgid\": \""
564 << cache_ioctx.get_id() << "." << i
565 << "\"}";
566 int r = cluster.mon_command(ss.str(), inbl, NULL, NULL);
567 if (r == -ENOENT || // in case mgr osdmap is stale
568 r == -EAGAIN) {
569 sleep(5);
570 continue;
571 }
572 } while (false);
573 }
574
575 // give it a few seconds to go. this is sloppy but is usually enough time
576 cout << "waiting for scrubs..." << std::endl;
577 sleep(30);
578 cout << "done waiting" << std::endl;
579 }
580
581 ioctx.snap_set_read(librados::SNAP_HEAD);
582
583 //cleanup
584 for (unsigned snap = 0; snap < my_snaps.size(); ++snap) {
585 ioctx.selfmanaged_snap_remove(my_snaps[snap]);
586 }
587}
588
589TEST_F(LibRadosTwoPoolsPP, PromoteSnapTrimRace) {
590 // create object
591 {
592 bufferlist bl;
593 bl.append("hi there");
594 ObjectWriteOperation op;
595 op.write_full(bl);
596 ASSERT_EQ(0, ioctx.operate("foo", &op));
597 }
598
599 // create a snapshot, clone
600 vector<uint64_t> my_snaps(1);
601 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
602 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
603 my_snaps));
604 {
605 bufferlist bl;
606 bl.append("ciao!");
607 ObjectWriteOperation op;
608 op.write_full(bl);
609 ASSERT_EQ(0, ioctx.operate("foo", &op));
610 }
611
612 // configure cache
613 bufferlist inbl;
614 ASSERT_EQ(0, cluster.mon_command(
615 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
616 "\", \"tierpool\": \"" + cache_pool_name +
617 "\", \"force_nonempty\": \"--force-nonempty\" }",
618 inbl, NULL, NULL));
619 ASSERT_EQ(0, cluster.mon_command(
620 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
621 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
622 inbl, NULL, NULL));
623 ASSERT_EQ(0, cluster.mon_command(
624 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
625 "\", \"mode\": \"writeback\"}",
626 inbl, NULL, NULL));
627
628 // wait for maps to settle
629 cluster.wait_for_latest_osdmap();
630
631 // delete the snap
632 ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps[0]));
633
634 ioctx.snap_set_read(my_snaps[0]);
635
9f95a23c
TL		 636 // read foo snap. the OSD may or may not realize that this snap has
637 // been logically deleted; either response is valid.
7c673cae
FG		 638 {
639 bufferlist bl;
9f95a23c
TL		 640 int r = ioctx.read("foo", bl, 1, 0);
641 ASSERT_TRUE(r == 1 || r == -ENOENT);
7c673cae
FG		 642 }
643
644 // cleanup
645 ioctx.selfmanaged_snap_remove(my_snaps[0]);
646}
647
648TEST_F(LibRadosTwoPoolsPP, Whiteout) {
649 // create object
650 {
651 bufferlist bl;
652 bl.append("hi there");
653 ObjectWriteOperation op;
654 op.write_full(bl);
655 ASSERT_EQ(0, ioctx.operate("foo", &op));
656 }
657
658 // configure cache
659 bufferlist inbl;
660 ASSERT_EQ(0, cluster.mon_command(
661 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
662 "\", \"tierpool\": \"" + cache_pool_name +
663 "\", \"force_nonempty\": \"--force-nonempty\" }",
664 inbl, NULL, NULL));
665 ASSERT_EQ(0, cluster.mon_command(
666 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
667 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
668 inbl, NULL, NULL));
669 ASSERT_EQ(0, cluster.mon_command(
670 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
671 "\", \"mode\": \"writeback\"}",
672 inbl, NULL, NULL));
673
674 // wait for maps to settle
675 cluster.wait_for_latest_osdmap();
676
677 // create some whiteouts, verify they behave
678 {
679 ObjectWriteOperation op;
680 op.assert_exists();
681 op.remove();
682 ASSERT_EQ(0, ioctx.operate("foo", &op));
683 }
684
685 {
686 ObjectWriteOperation op;
687 op.assert_exists();
688 op.remove();
689 ASSERT_EQ(-ENOENT, ioctx.operate("bar", &op));
690 }
691 {
692 ObjectWriteOperation op;
693 op.assert_exists();
694 op.remove();
695 ASSERT_EQ(-ENOENT, ioctx.operate("bar", &op));
696 }
697
698 // verify the whiteouts are there in the cache tier
699 {
700 NObjectIterator it = cache_ioctx.nobjects_begin();
701 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
702 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
703 ++it;
704 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
705 ++it;
706 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
707 }
708
709 // delete a whiteout and verify it goes away
710 ASSERT_EQ(-ENOENT, ioctx.remove("foo"));
711 {
712 ObjectWriteOperation op;
713 op.remove();
714 librados::AioCompletion *completion = cluster.aio_create_completion();
715 ASSERT_EQ(0, ioctx.aio_operate("bar", completion, &op,
716 librados::OPERATION_IGNORE_CACHE));
9f95a23c 717 completion->wait_for_complete();
7c673cae
FG		 718 ASSERT_EQ(0, completion->get_return_value());
719 completion->release();
720
721 NObjectIterator it = cache_ioctx.nobjects_begin();
722 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
723 ASSERT_TRUE(it->get_oid() == string("foo"));
724 ++it;
725 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
726 }
727
728 // recreate an object and verify we can read it
729 {
730 bufferlist bl;
731 bl.append("hi there");
732 ObjectWriteOperation op;
733 op.write_full(bl);
734 ASSERT_EQ(0, ioctx.operate("foo", &op));
735 }
736 {
737 bufferlist bl;
738 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
739 ASSERT_EQ('h', bl[0]);
740 }
741}
742
743TEST_F(LibRadosTwoPoolsPP, WhiteoutDeleteCreate) {
744 // configure cache
745 bufferlist inbl;
746 ASSERT_EQ(0, cluster.mon_command(
747 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
748 "\", \"tierpool\": \"" + cache_pool_name +
749 "\", \"force_nonempty\": \"--force-nonempty\" }",
750 inbl, NULL, NULL));
751 ASSERT_EQ(0, cluster.mon_command(
752 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
753 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
754 inbl, NULL, NULL));
755 ASSERT_EQ(0, cluster.mon_command(
756 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
757 "\", \"mode\": \"writeback\"}",
758 inbl, NULL, NULL));
759
760 // wait for maps to settle
761 cluster.wait_for_latest_osdmap();
762
763 // create an object
764 {
765 bufferlist bl;
766 bl.append("foo");
767 ASSERT_EQ(0, ioctx.write_full("foo", bl));
768 }
769
770 // do delete + create operation
771 {
772 ObjectWriteOperation op;
773 op.remove();
774 bufferlist bl;
775 bl.append("bar");
776 op.write_full(bl);
777 ASSERT_EQ(0, ioctx.operate("foo", &op));
778 }
779
780 // verify it still "exists" (w/ new content)
781 {
782 bufferlist bl;
783 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
784 ASSERT_EQ('b', bl[0]);
785 }
786}
787
788TEST_F(LibRadosTwoPoolsPP, Evict) {
789 // create object
790 {
791 bufferlist bl;
792 bl.append("hi there");
793 ObjectWriteOperation op;
794 op.write_full(bl);
795 ASSERT_EQ(0, ioctx.operate("foo", &op));
796 }
797
798 // configure cache
799 bufferlist inbl;
800 ASSERT_EQ(0, cluster.mon_command(
801 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
802 "\", \"tierpool\": \"" + cache_pool_name +
803 "\", \"force_nonempty\": \"--force-nonempty\" }",
804 inbl, NULL, NULL));
805 ASSERT_EQ(0, cluster.mon_command(
806 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
807 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
808 inbl, NULL, NULL));
809 ASSERT_EQ(0, cluster.mon_command(
810 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
811 "\", \"mode\": \"writeback\"}",
812 inbl, NULL, NULL));
813
814 // wait for maps to settle
815 cluster.wait_for_latest_osdmap();
816
817 // read, trigger a promote
818 {
819 bufferlist bl;
820 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
821 }
822
823 // read, trigger a whiteout, and a dirty object
824 {
825 bufferlist bl;
826 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
827 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
828 ASSERT_EQ(0, ioctx.write("bar", bl, bl.length(), 0));
829 }
830
831 // verify the object is present in the cache tier
832 {
833 NObjectIterator it = cache_ioctx.nobjects_begin();
834 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
835 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
836 ++it;
837 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
838 ++it;
839 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
840 }
841
842 // pin
843 {
844 ObjectWriteOperation op;
845 op.cache_pin();
846 librados::AioCompletion *completion = cluster.aio_create_completion();
847 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 848 completion->wait_for_complete();
7c673cae
FG		 849 ASSERT_EQ(0, completion->get_return_value());
850 completion->release();
851 }
852
853 // evict the pinned object with -EPERM
854 {
855 ObjectReadOperation op;
856 op.cache_evict();
857 librados::AioCompletion *completion = cluster.aio_create_completion();
858 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
859 librados::OPERATION_IGNORE_CACHE,
860 NULL));
9f95a23c 861 completion->wait_for_complete();
7c673cae
FG		 862 ASSERT_EQ(-EPERM, completion->get_return_value());
863 completion->release();
864 }
865
866 // unpin
867 {
868 ObjectWriteOperation op;
869 op.cache_unpin();
870 librados::AioCompletion *completion = cluster.aio_create_completion();
871 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 872 completion->wait_for_complete();
7c673cae
FG		 873 ASSERT_EQ(0, completion->get_return_value());
874 completion->release();
875 }
876
877 // flush
878 {
879 ObjectReadOperation op;
880 op.cache_flush();
881 librados::AioCompletion *completion = cluster.aio_create_completion();
882 ASSERT_EQ(0, cache_ioctx.aio_operate(
883 "foo", completion, &op,
884 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 885 completion->wait_for_complete();
7c673cae
FG		 886 ASSERT_EQ(0, completion->get_return_value());
887 completion->release();
888 }
889
890 // verify clean
891 {
892 bool dirty = false;
893 int r = -1;
894 ObjectReadOperation op;
895 op.is_dirty(&dirty, &r);
896 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
897 ASSERT_FALSE(dirty);
898 ASSERT_EQ(0, r);
899 }
900
901 // evict
902 {
903 ObjectReadOperation op;
904 op.cache_evict();
905 librados::AioCompletion *completion = cluster.aio_create_completion();
906 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
907 librados::OPERATION_IGNORE_CACHE,
908 NULL));
9f95a23c 909 completion->wait_for_complete();
7c673cae
FG		 910 ASSERT_EQ(0, completion->get_return_value());
911 completion->release();
912 }
913 {
914 ObjectReadOperation op;
915 op.cache_evict();
916 librados::AioCompletion *completion = cluster.aio_create_completion();
917 ASSERT_EQ(0, cache_ioctx.aio_operate(
918 "foo", completion, &op,
919 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 920 completion->wait_for_complete();
7c673cae
FG		 921 ASSERT_EQ(0, completion->get_return_value());
922 completion->release();
923 }
924 {
925 ObjectReadOperation op;
926 op.cache_evict();
927 librados::AioCompletion *completion = cluster.aio_create_completion();
928 ASSERT_EQ(0, cache_ioctx.aio_operate(
929 "bar", completion, &op,
930 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 931 completion->wait_for_complete();
7c673cae
FG		 932 ASSERT_EQ(-EBUSY, completion->get_return_value());
933 completion->release();
934 }
935}
936
937TEST_F(LibRadosTwoPoolsPP, EvictSnap) {
938 // create object
939 {
940 bufferlist bl;
941 bl.append("hi there");
942 ObjectWriteOperation op;
943 op.write_full(bl);
944 ASSERT_EQ(0, ioctx.operate("foo", &op));
945 }
946 {
947 bufferlist bl;
948 bl.append("hi there");
949 ObjectWriteOperation op;
950 op.write_full(bl);
951 ASSERT_EQ(0, ioctx.operate("bar", &op));
952 }
953 {
954 bufferlist bl;
955 bl.append("hi there");
956 ObjectWriteOperation op;
957 op.write_full(bl);
958 ASSERT_EQ(0, ioctx.operate("baz", &op));
959 }
960 {
961 bufferlist bl;
962 bl.append("hi there");
963 ObjectWriteOperation op;
964 op.write_full(bl);
965 ASSERT_EQ(0, ioctx.operate("bam", &op));
966 }
967
968 // create a snapshot, clone
969 vector<uint64_t> my_snaps(1);
970 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
971 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
972 my_snaps));
973 {
974 bufferlist bl;
975 bl.append("ciao!");
976 ObjectWriteOperation op;
977 op.write_full(bl);
978 ASSERT_EQ(0, ioctx.operate("foo", &op));
979 }
980 {
981 bufferlist bl;
982 bl.append("ciao!");
983 ObjectWriteOperation op;
984 op.write_full(bl);
985 ASSERT_EQ(0, ioctx.operate("bar", &op));
986 }
987 {
988 ObjectWriteOperation op;
989 op.remove();
990 ASSERT_EQ(0, ioctx.operate("baz", &op));
991 }
992 {
993 bufferlist bl;
994 bl.append("ciao!");
995 ObjectWriteOperation op;
996 op.write_full(bl);
997 ASSERT_EQ(0, ioctx.operate("bam", &op));
998 }
999
1000 // configure cache
1001 bufferlist inbl;
1002 ASSERT_EQ(0, cluster.mon_command(
1003 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1004 "\", \"tierpool\": \"" + cache_pool_name +
1005 "\", \"force_nonempty\": \"--force-nonempty\" }",
1006 inbl, NULL, NULL));
1007 ASSERT_EQ(0, cluster.mon_command(
1008 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1009 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1010 inbl, NULL, NULL));
1011 ASSERT_EQ(0, cluster.mon_command(
1012 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1013 "\", \"mode\": \"writeback\"}",
1014 inbl, NULL, NULL));
1015
1016 // wait for maps to settle
1017 cluster.wait_for_latest_osdmap();
1018
1019 // read, trigger a promote on the head
1020 {
1021 bufferlist bl;
1022 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1023 ASSERT_EQ('c', bl[0]);
1024 }
1025 {
1026 bufferlist bl;
1027 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
1028 ASSERT_EQ('c', bl[0]);
1029 }
1030
1031 // evict bam
1032 {
1033 ObjectReadOperation op;
1034 op.cache_evict();
1035 librados::AioCompletion *completion = cluster.aio_create_completion();
1036 ASSERT_EQ(0, cache_ioctx.aio_operate(
1037 "bam", completion, &op,
1038 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1039 completion->wait_for_complete();
7c673cae
FG		 1040 ASSERT_EQ(0, completion->get_return_value());
1041 completion->release();
1042 }
1043 {
1044 bufferlist bl;
1045 ObjectReadOperation op;
1046 op.read(1, 0, &bl, NULL);
1047 librados::AioCompletion *completion = cluster.aio_create_completion();
1048 ASSERT_EQ(0, cache_ioctx.aio_operate(
1049 "bam", completion, &op,
1050 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1051 completion->wait_for_complete();
7c673cae
FG		 1052 ASSERT_EQ(-ENOENT, completion->get_return_value());
1053 completion->release();
1054 }
1055
1056 // read foo snap
1057 ioctx.snap_set_read(my_snaps[0]);
1058 {
1059 bufferlist bl;
1060 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1061 ASSERT_EQ('h', bl[0]);
1062 }
1063
1064 // evict foo snap
1065 {
1066 ObjectReadOperation op;
1067 op.cache_evict();
1068 librados::AioCompletion *completion = cluster.aio_create_completion();
1069 ASSERT_EQ(0, ioctx.aio_operate(
1070 "foo", completion, &op,
1071 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1072 completion->wait_for_complete();
7c673cae
FG		 1073 ASSERT_EQ(0, completion->get_return_value());
1074 completion->release();
1075 }
1076 // snap is gone...
1077 {
1078 bufferlist bl;
1079 ObjectReadOperation op;
1080 op.read(1, 0, &bl, NULL);
1081 librados::AioCompletion *completion = cluster.aio_create_completion();
1082 ASSERT_EQ(0, ioctx.aio_operate(
1083 "foo", completion, &op,
1084 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1085 completion->wait_for_complete();
7c673cae
FG		 1086 ASSERT_EQ(-ENOENT, completion->get_return_value());
1087 completion->release();
1088 }
1089 // head is still there...
1090 ioctx.snap_set_read(librados::SNAP_HEAD);
1091 {
1092 bufferlist bl;
1093 ObjectReadOperation op;
1094 op.read(1, 0, &bl, NULL);
1095 librados::AioCompletion *completion = cluster.aio_create_completion();
1096 ASSERT_EQ(0, ioctx.aio_operate(
1097 "foo", completion, &op,
1098 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1099 completion->wait_for_complete();
7c673cae
FG		 1100 ASSERT_EQ(0, completion->get_return_value());
1101 completion->release();
1102 }
1103
1104 // promote head + snap of bar
1105 ioctx.snap_set_read(librados::SNAP_HEAD);
1106 {
1107 bufferlist bl;
1108 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
1109 ASSERT_EQ('c', bl[0]);
1110 }
1111 ioctx.snap_set_read(my_snaps[0]);
1112 {
1113 bufferlist bl;
1114 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
1115 ASSERT_EQ('h', bl[0]);
1116 }
1117
1118 // evict bar head (fail)
1119 ioctx.snap_set_read(librados::SNAP_HEAD);
1120 {
1121 ObjectReadOperation op;
1122 op.cache_evict();
1123 librados::AioCompletion *completion = cluster.aio_create_completion();
1124 ASSERT_EQ(0, ioctx.aio_operate(
1125 "bar", completion, &op,
1126 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1127 completion->wait_for_complete();
7c673cae
FG		 1128 ASSERT_EQ(-EBUSY, completion->get_return_value());
1129 completion->release();
1130 }
1131
1132 // evict bar snap
1133 ioctx.snap_set_read(my_snaps[0]);
1134 {
1135 ObjectReadOperation op;
1136 op.cache_evict();
1137 librados::AioCompletion *completion = cluster.aio_create_completion();
1138 ASSERT_EQ(0, ioctx.aio_operate(
1139 "bar", completion, &op,
1140 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1141 completion->wait_for_complete();
7c673cae
FG		 1142 ASSERT_EQ(0, completion->get_return_value());
1143 completion->release();
1144 }
1145 // ...and then head
1146 ioctx.snap_set_read(librados::SNAP_HEAD);
1147 {
1148 bufferlist bl;
1149 ObjectReadOperation op;
1150 op.read(1, 0, &bl, NULL);
1151 librados::AioCompletion *completion = cluster.aio_create_completion();
1152 ASSERT_EQ(0, ioctx.aio_operate(
1153 "bar", completion, &op,
1154 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1155 completion->wait_for_complete();
7c673cae
FG		 1156 ASSERT_EQ(0, completion->get_return_value());
1157 completion->release();
1158 }
1159 {
1160 ObjectReadOperation op;
1161 op.cache_evict();
1162 librados::AioCompletion *completion = cluster.aio_create_completion();
1163 ASSERT_EQ(0, ioctx.aio_operate(
1164 "bar", completion, &op,
1165 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1166 completion->wait_for_complete();
7c673cae
FG		 1167 ASSERT_EQ(0, completion->get_return_value());
1168 completion->release();
1169 }
1170
1171 // cleanup
1172 ioctx.selfmanaged_snap_remove(my_snaps[0]);
1173}
1174
1175// this test case reproduces http://tracker.ceph.com/issues/8629
1176TEST_F(LibRadosTwoPoolsPP, EvictSnap2) {
1177 // create object
1178 {
1179 bufferlist bl;
1180 bl.append("hi there");
1181 ObjectWriteOperation op;
1182 op.write_full(bl);
1183 ASSERT_EQ(0, ioctx.operate("foo", &op));
1184 }
1185 // create a snapshot, clone
1186 vector<uint64_t> my_snaps(1);
1187 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
1188 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
1189 my_snaps));
1190 {
1191 bufferlist bl;
1192 bl.append("ciao!");
1193 ObjectWriteOperation op;
1194 op.write_full(bl);
1195 ASSERT_EQ(0, ioctx.operate("foo", &op));
1196 }
1197 // configure cache
1198 bufferlist inbl;
1199 ASSERT_EQ(0, cluster.mon_command(
1200 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1201 "\", \"tierpool\": \"" + cache_pool_name +
1202 "\", \"force_nonempty\": \"--force-nonempty\" }",
1203 inbl, NULL, NULL));
1204 ASSERT_EQ(0, cluster.mon_command(
1205 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1206 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1207 inbl, NULL, NULL));
1208 ASSERT_EQ(0, cluster.mon_command(
1209 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1210 "\", \"mode\": \"writeback\"}",
1211 inbl, NULL, NULL));
1212
1213 // wait for maps to settle
1214 cluster.wait_for_latest_osdmap();
1215
1216 // read, trigger a promote on the head
1217 {
1218 bufferlist bl;
1219 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1220 ASSERT_EQ('c', bl[0]);
1221 }
1222
1223 // evict
1224 {
1225 ObjectReadOperation op;
1226 op.cache_evict();
1227 librados::AioCompletion *completion = cluster.aio_create_completion();
1228 ASSERT_EQ(0, cache_ioctx.aio_operate(
1229 "foo", completion, &op,
1230 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1231 completion->wait_for_complete();
7c673cae
FG		 1232 ASSERT_EQ(0, completion->get_return_value());
1233 completion->release();
1234 }
1235
1236 // verify the snapdir is not present in the cache pool
1237 {
1238 ObjectReadOperation op;
1239 librados::snap_set_t snapset;
1240 op.list_snaps(&snapset, NULL);
1241 ioctx.snap_set_read(librados::SNAP_DIR);
1242 librados::AioCompletion *completion = cluster.aio_create_completion();
1243 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op,
1244 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1245 completion->wait_for_complete();
7c673cae
FG		 1246 ASSERT_EQ(-ENOENT, completion->get_return_value());
1247 completion->release();
1248 }
1249}
1250
11fdf7f2
TL		 1251//This test case reproduces http://tracker.ceph.com/issues/17445
1252TEST_F(LibRadosTwoPoolsPP, ListSnap){
1253 // Create object
1254 {
1255 bufferlist bl;
1256 bl.append("hi there");
1257 ObjectWriteOperation op;
1258 op.write_full(bl);
1259 ASSERT_EQ(0, ioctx.operate("foo", &op));
1260 }
1261 {
1262 bufferlist bl;
1263 bl.append("hi there");
1264 ObjectWriteOperation op;
1265 op.write_full(bl);
1266 ASSERT_EQ(0, ioctx.operate("bar", &op));
1267 }
1268 {
1269 bufferlist bl;
1270 bl.append("hi there");
1271 ObjectWriteOperation op;
1272 op.write_full(bl);
1273 ASSERT_EQ(0, ioctx.operate("baz", &op));
1274 }
1275 {
1276 bufferlist bl;
1277 bl.append("hi there");
1278 ObjectWriteOperation op;
1279 op.write_full(bl);
1280 ASSERT_EQ(0, ioctx.operate("bam", &op));
1281 }
1282
1283 // Create a snapshot, clone
1284 vector<uint64_t> my_snaps(1);
1285 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
1286 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
1287 my_snaps));
1288 {
1289 bufferlist bl;
1290 bl.append("ciao!");
1291 ObjectWriteOperation op;
1292 op.write_full(bl);
1293 ASSERT_EQ(0, ioctx.operate("foo", &op));
1294 }
1295 {
1296 bufferlist bl;
1297 bl.append("ciao!");
1298 ObjectWriteOperation op;
1299 op.write_full(bl);
1300 ASSERT_EQ(0, ioctx.operate("bar", &op));
1301 }
1302 {
1303 ObjectWriteOperation op;
1304 op.remove();
1305 ASSERT_EQ(0, ioctx.operate("baz", &op));
1306 }
1307 {
1308 bufferlist bl;
1309 bl.append("ciao!");
1310 ObjectWriteOperation op;
1311 op.write_full(bl);
1312 ASSERT_EQ(0, ioctx.operate("bam", &op));
1313 }
1314
1315 // Configure cache
1316 bufferlist inbl;
1317 ASSERT_EQ(0, cluster.mon_command(
1318 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1319 "\", \"tierpool\": \"" + cache_pool_name +
1320 "\", \"force_nonempty\": \"--force-nonempty\" }",
1321 inbl, NULL, NULL));
1322 ASSERT_EQ(0, cluster.mon_command(
1323 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1324 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1325 inbl, NULL, NULL));
1326 ASSERT_EQ(0, cluster.mon_command(
1327 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1328 "\", \"mode\": \"writeback\"}",
1329 inbl, NULL, NULL));
1330
1331 // Wait for maps to settle
1332 cluster.wait_for_latest_osdmap();
1333
1334 // Read, trigger a promote on the head
1335 {
1336 bufferlist bl;
1337 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1338 ASSERT_EQ('c', bl[0]);
1339 }
1340
1341 // Read foo snap
1342 ioctx.snap_set_read(my_snaps[0]);
1343 {
1344 bufferlist bl;
1345 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1346 ASSERT_EQ('h', bl[0]);
1347 }
1348
1349 // Evict foo snap
1350 {
1351 ObjectReadOperation op;
1352 op.cache_evict();
1353 librados::AioCompletion *completion = cluster.aio_create_completion();
1354 ASSERT_EQ(0, ioctx.aio_operate(
1355 "foo", completion, &op,
1356 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1357 completion->wait_for_complete();
11fdf7f2
TL		 1358 ASSERT_EQ(0, completion->get_return_value());
1359 completion->release();
1360 }
1361 // Snap is gone...
1362 {
1363 bufferlist bl;
1364 ObjectReadOperation op;
1365 op.read(1, 0, &bl, NULL);
1366 librados::AioCompletion *completion = cluster.aio_create_completion();
1367 ASSERT_EQ(0, ioctx.aio_operate(
1368 "foo", completion, &op,
1369 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1370 completion->wait_for_complete();
11fdf7f2
TL		 1371 ASSERT_EQ(-ENOENT, completion->get_return_value());
1372 completion->release();
1373 }
1374
1375 // Do list-snaps
1376 ioctx.snap_set_read(CEPH_SNAPDIR);
1377 {
1378 snap_set_t snap_set;
1379 int snap_ret;
1380 ObjectReadOperation op;
1381 op.list_snaps(&snap_set, &snap_ret);
1382 librados::AioCompletion *completion = cluster.aio_create_completion();
1383 ASSERT_EQ(0, ioctx.aio_operate(
1384 "foo", completion, &op,
1385 0, NULL));
9f95a23c 1386 completion->wait_for_complete();
11fdf7f2
TL		 1387 ASSERT_EQ(0, snap_ret);
1388 ASSERT_LT(0u, snap_set.clones.size());
1389 for (vector<librados::clone_info_t>::const_iterator r = snap_set.clones.begin();
1390 r != snap_set.clones.end();
1391 ++r) {
1392 if (r->cloneid != librados::SNAP_HEAD) {
1393 ASSERT_LT(0u, r->snaps.size());
1394 }
1395 }
1396 }
1397
1398 // Cleanup
1399 ioctx.selfmanaged_snap_remove(my_snaps[0]);
1400}
1401
7c673cae
FG		 1402TEST_F(LibRadosTwoPoolsPP, TryFlush) {
1403 // configure cache
1404 bufferlist inbl;
1405 ASSERT_EQ(0, cluster.mon_command(
1406 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1407 "\", \"tierpool\": \"" + cache_pool_name +
1408 "\", \"force_nonempty\": \"--force-nonempty\" }",
1409 inbl, NULL, NULL));
1410 ASSERT_EQ(0, cluster.mon_command(
1411 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1412 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1413 inbl, NULL, NULL));
1414 ASSERT_EQ(0, cluster.mon_command(
1415 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1416 "\", \"mode\": \"writeback\"}",
1417 inbl, NULL, NULL));
1418
1419 // wait for maps to settle
1420 cluster.wait_for_latest_osdmap();
1421
1422 // create object
1423 {
1424 bufferlist bl;
1425 bl.append("hi there");
1426 ObjectWriteOperation op;
1427 op.write_full(bl);
1428 ASSERT_EQ(0, ioctx.operate("foo", &op));
1429 }
1430
1431 // verify the object is present in the cache tier
1432 {
1433 NObjectIterator it = cache_ioctx.nobjects_begin();
1434 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
1435 ASSERT_TRUE(it->get_oid() == string("foo"));
1436 ++it;
1437 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
1438 }
1439
1440 // verify the object is NOT present in the base tier
1441 {
1442 NObjectIterator it = ioctx.nobjects_begin();
1443 ASSERT_TRUE(it == ioctx.nobjects_end());
1444 }
1445
1446 // verify dirty
1447 {
1448 bool dirty = false;
1449 int r = -1;
1450 ObjectReadOperation op;
1451 op.is_dirty(&dirty, &r);
1452 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
1453 ASSERT_TRUE(dirty);
1454 ASSERT_EQ(0, r);
1455 }
1456
1457 // pin
1458 {
1459 ObjectWriteOperation op;
1460 op.cache_pin();
1461 librados::AioCompletion *completion = cluster.aio_create_completion();
1462 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 1463 completion->wait_for_complete();
7c673cae
FG		 1464 ASSERT_EQ(0, completion->get_return_value());
1465 completion->release();
1466 }
1467
1468 // flush the pinned object with -EPERM
1469 {
1470 ObjectReadOperation op;
1471 op.cache_try_flush();
1472 librados::AioCompletion *completion = cluster.aio_create_completion();
1473 ASSERT_EQ(0, cache_ioctx.aio_operate(
1474 "foo", completion, &op,
1475 librados::OPERATION_IGNORE_OVERLAY |
1476 librados::OPERATION_SKIPRWLOCKS, NULL));
9f95a23c 1477 completion->wait_for_complete();
7c673cae
FG		 1478 ASSERT_EQ(-EPERM, completion->get_return_value());
1479 completion->release();
1480 }
1481
1482 // unpin
1483 {
1484 ObjectWriteOperation op;
1485 op.cache_unpin();
1486 librados::AioCompletion *completion = cluster.aio_create_completion();
1487 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 1488 completion->wait_for_complete();
7c673cae
FG		 1489 ASSERT_EQ(0, completion->get_return_value());
1490 completion->release();
1491 }
1492
1493 // flush
1494 {
1495 ObjectReadOperation op;
1496 op.cache_try_flush();
1497 librados::AioCompletion *completion = cluster.aio_create_completion();
1498 ASSERT_EQ(0, cache_ioctx.aio_operate(
1499 "foo", completion, &op,
1500 librados::OPERATION_IGNORE_OVERLAY |
1501 librados::OPERATION_SKIPRWLOCKS, NULL));
9f95a23c 1502 completion->wait_for_complete();
7c673cae
FG		 1503 ASSERT_EQ(0, completion->get_return_value());
1504 completion->release();
1505 }
1506
1507 // verify clean
1508 {
1509 bool dirty = false;
1510 int r = -1;
1511 ObjectReadOperation op;
1512 op.is_dirty(&dirty, &r);
1513 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
1514 ASSERT_FALSE(dirty);
1515 ASSERT_EQ(0, r);
1516 }
1517
1518 // verify in base tier
1519 {
1520 NObjectIterator it = ioctx.nobjects_begin();
1521 ASSERT_TRUE(it != ioctx.nobjects_end());
1522 ASSERT_TRUE(it->get_oid() == string("foo"));
1523 ++it;
1524 ASSERT_TRUE(it == ioctx.nobjects_end());
1525 }
1526
1527 // evict it
1528 {
1529 ObjectReadOperation op;
1530 op.cache_evict();
1531 librados::AioCompletion *completion = cluster.aio_create_completion();
1532 ASSERT_EQ(0, cache_ioctx.aio_operate(
1533 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1534 completion->wait_for_complete();
7c673cae
FG		 1535 ASSERT_EQ(0, completion->get_return_value());
1536 completion->release();
1537 }
1538
1539 // verify no longer in cache tier
1540 {
1541 NObjectIterator it = cache_ioctx.nobjects_begin();
1542 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
1543 }
1544}
1545
1546TEST_F(LibRadosTwoPoolsPP, Flush) {
1547 // configure cache
1548 bufferlist inbl;
1549 ASSERT_EQ(0, cluster.mon_command(
1550 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1551 "\", \"tierpool\": \"" + cache_pool_name +
1552 "\", \"force_nonempty\": \"--force-nonempty\" }",
1553 inbl, NULL, NULL));
1554 ASSERT_EQ(0, cluster.mon_command(
1555 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1556 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1557 inbl, NULL, NULL));
1558 ASSERT_EQ(0, cluster.mon_command(
1559 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1560 "\", \"mode\": \"writeback\"}",
1561 inbl, NULL, NULL));
1562
1563 // wait for maps to settle
1564 cluster.wait_for_latest_osdmap();
1565
1566 uint64_t user_version = 0;
1567
1568 // create object
1569 {
1570 bufferlist bl;
1571 bl.append("hi there");
1572 ObjectWriteOperation op;
1573 op.write_full(bl);
1574 ASSERT_EQ(0, ioctx.operate("foo", &op));
1575 }
1576
1577 // verify the object is present in the cache tier
1578 {
1579 NObjectIterator it = cache_ioctx.nobjects_begin();
1580 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
1581 ASSERT_TRUE(it->get_oid() == string("foo"));
1582 ++it;
1583 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
1584 }
1585
1586 // verify the object is NOT present in the base tier
1587 {
1588 NObjectIterator it = ioctx.nobjects_begin();
1589 ASSERT_TRUE(it == ioctx.nobjects_end());
1590 }
1591
1592 // verify dirty
1593 {
1594 bool dirty = false;
1595 int r = -1;
1596 ObjectReadOperation op;
1597 op.is_dirty(&dirty, &r);
1598 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
1599 ASSERT_TRUE(dirty);
1600 ASSERT_EQ(0, r);
1601 user_version = cache_ioctx.get_last_version();
1602 }
1603
1604 // pin
1605 {
1606 ObjectWriteOperation op;
1607 op.cache_pin();
1608 librados::AioCompletion *completion = cluster.aio_create_completion();
1609 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 1610 completion->wait_for_complete();
7c673cae
FG		 1611 ASSERT_EQ(0, completion->get_return_value());
1612 completion->release();
1613 }
1614
1615 // flush the pinned object with -EPERM
1616 {
1617 ObjectReadOperation op;
1618 op.cache_try_flush();
1619 librados::AioCompletion *completion = cluster.aio_create_completion();
1620 ASSERT_EQ(0, cache_ioctx.aio_operate(
1621 "foo", completion, &op,
1622 librados::OPERATION_IGNORE_OVERLAY |
1623 librados::OPERATION_SKIPRWLOCKS, NULL));
9f95a23c 1624 completion->wait_for_complete();
7c673cae
FG		 1625 ASSERT_EQ(-EPERM, completion->get_return_value());
1626 completion->release();
1627 }
1628
1629 // unpin
1630 {
1631 ObjectWriteOperation op;
1632 op.cache_unpin();
1633 librados::AioCompletion *completion = cluster.aio_create_completion();
1634 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 1635 completion->wait_for_complete();
7c673cae
FG		 1636 ASSERT_EQ(0, completion->get_return_value());
1637 completion->release();
1638 }
1639
1640 // flush
1641 {
1642 ObjectReadOperation op;
1643 op.cache_flush();
1644 librados::AioCompletion *completion = cluster.aio_create_completion();
1645 ASSERT_EQ(0, cache_ioctx.aio_operate(
1646 "foo", completion, &op,
1647 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 1648 completion->wait_for_complete();
7c673cae
FG		 1649 ASSERT_EQ(0, completion->get_return_value());
1650 completion->release();
1651 }
1652
1653 // verify clean
1654 {
1655 bool dirty = false;
1656 int r = -1;
1657 ObjectReadOperation op;
1658 op.is_dirty(&dirty, &r);
1659 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
1660 ASSERT_FALSE(dirty);
1661 ASSERT_EQ(0, r);
1662 }
1663
1664 // verify in base tier
1665 {
1666 NObjectIterator it = ioctx.nobjects_begin();
1667 ASSERT_TRUE(it != ioctx.nobjects_end());
1668 ASSERT_TRUE(it->get_oid() == string("foo"));
1669 ++it;
1670 ASSERT_TRUE(it == ioctx.nobjects_end());
1671 }
1672
1673 // evict it
1674 {
1675 ObjectReadOperation op;
1676 op.cache_evict();
1677 librados::AioCompletion *completion = cluster.aio_create_completion();
1678 ASSERT_EQ(0, cache_ioctx.aio_operate(
1679 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1680 completion->wait_for_complete();
7c673cae
FG		 1681 ASSERT_EQ(0, completion->get_return_value());
1682 completion->release();
1683 }
1684
1685 // verify no longer in cache tier
1686 {
1687 NObjectIterator it = cache_ioctx.nobjects_begin();
1688 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
1689 }
1690
1691 // read it again and verify the version is consistent
1692 {
1693 bufferlist bl;
1694 ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
1695 ASSERT_EQ(user_version, cache_ioctx.get_last_version());
1696 }
1697
1698 // erase it
1699 {
1700 ObjectWriteOperation op;
1701 op.remove();
1702 ASSERT_EQ(0, ioctx.operate("foo", &op));
1703 }
1704
1705 // flush whiteout
1706 {
1707 ObjectReadOperation op;
1708 op.cache_flush();
1709 librados::AioCompletion *completion = cluster.aio_create_completion();
1710 ASSERT_EQ(0, cache_ioctx.aio_operate(
1711 "foo", completion, &op,
1712 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 1713 completion->wait_for_complete();
7c673cae
FG		 1714 ASSERT_EQ(0, completion->get_return_value());
1715 completion->release();
1716 }
1717
1718 // evict
1719 {
1720 ObjectReadOperation op;
1721 op.cache_evict();
1722 librados::AioCompletion *completion = cluster.aio_create_completion();
1723 ASSERT_EQ(0, cache_ioctx.aio_operate(
1724 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1725 completion->wait_for_complete();
7c673cae
FG		 1726 ASSERT_EQ(0, completion->get_return_value());
1727 completion->release();
1728 }
1729
1730 // verify no longer in cache tier
1731 {
1732 NObjectIterator it = cache_ioctx.nobjects_begin();
1733 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
1734 }
1735 // or base tier
1736 {
1737 NObjectIterator it = ioctx.nobjects_begin();
1738 ASSERT_TRUE(it == ioctx.nobjects_end());
1739 }
1740}
1741
1742TEST_F(LibRadosTwoPoolsPP, FlushSnap) {
1743 // configure cache
1744 bufferlist inbl;
1745 ASSERT_EQ(0, cluster.mon_command(
1746 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1747 "\", \"tierpool\": \"" + cache_pool_name +
1748 "\", \"force_nonempty\": \"--force-nonempty\" }",
1749 inbl, NULL, NULL));
1750 ASSERT_EQ(0, cluster.mon_command(
1751 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1752 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1753 inbl, NULL, NULL));
1754 ASSERT_EQ(0, cluster.mon_command(
1755 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1756 "\", \"mode\": \"writeback\"}",
1757 inbl, NULL, NULL));
1758
1759 // wait for maps to settle
1760 cluster.wait_for_latest_osdmap();
1761
1762 // create object
1763 {
1764 bufferlist bl;
1765 bl.append("a");
1766 ObjectWriteOperation op;
1767 op.write_full(bl);
1768 ASSERT_EQ(0, ioctx.operate("foo", &op));
1769 }
1770
1771 // create a snapshot, clone
1772 vector<uint64_t> my_snaps(1);
1773 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
1774 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
1775 my_snaps));
1776 {
1777 bufferlist bl;
1778 bl.append("b");
1779 ObjectWriteOperation op;
1780 op.write_full(bl);
1781 ASSERT_EQ(0, ioctx.operate("foo", &op));
1782 }
1783
1784 // and another
1785 my_snaps.resize(2);
1786 my_snaps[1] = my_snaps[0];
1787 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
1788 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
1789 my_snaps));
1790 {
1791 bufferlist bl;
1792 bl.append("c");
1793 ObjectWriteOperation op;
1794 op.write_full(bl);
1795 ASSERT_EQ(0, ioctx.operate("foo", &op));
1796 }
1797
1798 // verify the object is present in the cache tier
1799 {
1800 NObjectIterator it = cache_ioctx.nobjects_begin();
1801 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
1802 ASSERT_TRUE(it->get_oid() == string("foo"));
1803 ++it;
1804 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
1805 }
1806
1807 // verify the object is NOT present in the base tier
1808 {
1809 NObjectIterator it = ioctx.nobjects_begin();
1810 ASSERT_TRUE(it == ioctx.nobjects_end());
1811 }
1812
1813 // flush on head (should fail)
1814 ioctx.snap_set_read(librados::SNAP_HEAD);
1815 {
1816 ObjectReadOperation op;
1817 op.cache_flush();
1818 librados::AioCompletion *completion = cluster.aio_create_completion();
1819 ASSERT_EQ(0, ioctx.aio_operate(
1820 "foo", completion, &op,
1821 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1822 completion->wait_for_complete();
7c673cae
FG		 1823 ASSERT_EQ(-EBUSY, completion->get_return_value());
1824 completion->release();
1825 }
1826 // flush on recent snap (should fail)
1827 ioctx.snap_set_read(my_snaps[0]);
1828 {
1829 ObjectReadOperation op;
1830 op.cache_flush();
1831 librados::AioCompletion *completion = cluster.aio_create_completion();
1832 ASSERT_EQ(0, ioctx.aio_operate(
1833 "foo", completion, &op,
1834 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1835 completion->wait_for_complete();
7c673cae
FG		 1836 ASSERT_EQ(-EBUSY, completion->get_return_value());
1837 completion->release();
1838 }
1839 // flush on oldest snap
1840 ioctx.snap_set_read(my_snaps[1]);
1841 {
1842 ObjectReadOperation op;
1843 op.cache_flush();
1844 librados::AioCompletion *completion = cluster.aio_create_completion();
1845 ASSERT_EQ(0, ioctx.aio_operate(
1846 "foo", completion, &op,
1847 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1848 completion->wait_for_complete();
7c673cae
FG		 1849 ASSERT_EQ(0, completion->get_return_value());
1850 completion->release();
1851 }
1852 // flush on next oldest snap
1853 ioctx.snap_set_read(my_snaps[0]);
1854 {
1855 ObjectReadOperation op;
1856 op.cache_flush();
1857 librados::AioCompletion *completion = cluster.aio_create_completion();
1858 ASSERT_EQ(0, ioctx.aio_operate(
1859 "foo", completion, &op,
1860 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1861 completion->wait_for_complete();
7c673cae
FG		 1862 ASSERT_EQ(0, completion->get_return_value());
1863 completion->release();
1864 }
1865 // flush on head
1866 ioctx.snap_set_read(librados::SNAP_HEAD);
1867 {
1868 ObjectReadOperation op;
1869 op.cache_flush();
1870 librados::AioCompletion *completion = cluster.aio_create_completion();
1871 ASSERT_EQ(0, ioctx.aio_operate(
1872 "foo", completion, &op,
1873 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 1874 completion->wait_for_complete();
7c673cae
FG		 1875 ASSERT_EQ(0, completion->get_return_value());
1876 completion->release();
1877 }
1878
1879 // verify i can read the snaps from the cache pool
1880 ioctx.snap_set_read(librados::SNAP_HEAD);
1881 {
1882 bufferlist bl;
1883 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1884 ASSERT_EQ('c', bl[0]);
1885 }
1886 ioctx.snap_set_read(my_snaps[0]);
1887 {
1888 bufferlist bl;
1889 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1890 ASSERT_EQ('b', bl[0]);
1891 }
1892 ioctx.snap_set_read(my_snaps[1]);
1893 {
1894 bufferlist bl;
1895 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1896 ASSERT_EQ('a', bl[0]);
1897 }
1898
1899 // remove overlay
1900 ASSERT_EQ(0, cluster.mon_command(
1901 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
1902 "\"}",
1903 inbl, NULL, NULL));
1904
1905 // wait for maps to settle
1906 cluster.wait_for_latest_osdmap();
1907
1908 // verify i can read the snaps from the base pool
1909 ioctx.snap_set_read(librados::SNAP_HEAD);
1910 {
1911 bufferlist bl;
1912 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1913 ASSERT_EQ('c', bl[0]);
1914 }
1915 ioctx.snap_set_read(my_snaps[0]);
1916 {
1917 bufferlist bl;
1918 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1919 ASSERT_EQ('b', bl[0]);
1920 }
1921 ioctx.snap_set_read(my_snaps[1]);
1922 {
1923 bufferlist bl;
1924 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
1925 ASSERT_EQ('a', bl[0]);
1926 }
1927
1928 ASSERT_EQ(0, cluster.mon_command(
1929 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1930 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1931 inbl, NULL, NULL));
1932
1933 // cleanup
1934 ioctx.selfmanaged_snap_remove(my_snaps[0]);
1935}
1936
1937TEST_F(LibRadosTierPP, FlushWriteRaces) {
1938 Rados cluster;
1939 std::string pool_name = get_temp_pool_name();
1940 std::string cache_pool_name = pool_name + "-cache";
1941 ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
1942 ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
1943 IoCtx cache_ioctx;
1944 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
c07f9fc5 1945 cache_ioctx.application_enable("rados", true);
7c673cae
FG		 1946 IoCtx ioctx;
1947 ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
1948
1949 // configure cache
1950 bufferlist inbl;
1951 ASSERT_EQ(0, cluster.mon_command(
1952 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
1953 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
1954 inbl, NULL, NULL));
1955 ASSERT_EQ(0, cluster.mon_command(
1956 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
1957 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
1958 inbl, NULL, NULL));
1959 ASSERT_EQ(0, cluster.mon_command(
1960 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
1961 "\", \"mode\": \"writeback\"}",
1962 inbl, NULL, NULL));
1963
1964 // wait for maps to settle
1965 cluster.wait_for_latest_osdmap();
1966
1967 // create/dirty object
1968 bufferlist bl;
1969 bl.append("hi there");
1970 {
1971 ObjectWriteOperation op;
1972 op.write_full(bl);
1973 ASSERT_EQ(0, ioctx.operate("foo", &op));
1974 }
1975
1976 // flush + write
1977 {
1978 ObjectReadOperation op;
1979 op.cache_flush();
1980 librados::AioCompletion *completion = cluster.aio_create_completion();
1981 ASSERT_EQ(0, cache_ioctx.aio_operate(
1982 "foo", completion, &op,
1983 librados::OPERATION_IGNORE_OVERLAY, NULL));
1984
1985 ObjectWriteOperation op2;
1986 op2.write_full(bl);
1987 librados::AioCompletion *completion2 = cluster.aio_create_completion();
1988 ASSERT_EQ(0, ioctx.aio_operate(
1989 "foo", completion2, &op2, 0));
1990
9f95a23c
TL		 1991 completion->wait_for_complete();
1992 completion2->wait_for_complete();
7c673cae
FG		 1993 ASSERT_EQ(0, completion->get_return_value());
1994 ASSERT_EQ(0, completion2->get_return_value());
1995 completion->release();
1996 completion2->release();
1997 }
1998
1999 int tries = 1000;
2000 do {
2001 // create/dirty object
2002 {
2003 bufferlist bl;
2004 bl.append("hi there");
2005 ObjectWriteOperation op;
2006 op.write_full(bl);
2007 ASSERT_EQ(0, ioctx.operate("foo", &op));
2008 }
2009
2010 // try-flush + write
2011 {
2012 ObjectReadOperation op;
2013 op.cache_try_flush();
2014 librados::AioCompletion *completion = cluster.aio_create_completion();
2015 ASSERT_EQ(0, cache_ioctx.aio_operate(
2016 "foo", completion, &op,
2017 librados::OPERATION_IGNORE_OVERLAY |
2018 librados::OPERATION_SKIPRWLOCKS, NULL));
2019
2020 ObjectWriteOperation op2;
2021 op2.write_full(bl);
2022 librados::AioCompletion *completion2 = cluster.aio_create_completion();
2023 ASSERT_EQ(0, ioctx.aio_operate("foo", completion2, &op2, 0));
2024
9f95a23c
TL		 2025 completion->wait_for_complete();
2026 completion2->wait_for_complete();
7c673cae
FG		 2027 int r = completion->get_return_value();
2028 ASSERT_TRUE(r == -EBUSY || r == 0);
2029 ASSERT_EQ(0, completion2->get_return_value());
2030 completion->release();
2031 completion2->release();
2032 if (r == -EBUSY)
2033 break;
2034 cout << "didn't get EBUSY, trying again" << std::endl;
2035 }
2036 ASSERT_TRUE(--tries);
2037 } while (true);
2038
2039 // tear down tiers
2040 ASSERT_EQ(0, cluster.mon_command(
2041 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
2042 "\"}",
2043 inbl, NULL, NULL));
2044 ASSERT_EQ(0, cluster.mon_command(
2045 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
2046 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
2047 inbl, NULL, NULL));
2048
2049 // wait for maps to settle before next test
2050 cluster.wait_for_latest_osdmap();
2051
2052 ASSERT_EQ(0, cluster.pool_delete(cache_pool_name.c_str()));
2053 ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
2054}
2055
2056TEST_F(LibRadosTwoPoolsPP, FlushTryFlushRaces) {
2057 // configure cache
2058 bufferlist inbl;
2059 ASSERT_EQ(0, cluster.mon_command(
2060 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2061 "\", \"tierpool\": \"" + cache_pool_name +
2062 "\", \"force_nonempty\": \"--force-nonempty\" }",
2063 inbl, NULL, NULL));
2064 ASSERT_EQ(0, cluster.mon_command(
2065 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
2066 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
2067 inbl, NULL, NULL));
2068 ASSERT_EQ(0, cluster.mon_command(
2069 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
2070 "\", \"mode\": \"writeback\"}",
2071 inbl, NULL, NULL));
2072
2073 // wait for maps to settle
2074 cluster.wait_for_latest_osdmap();
2075
2076 // create/dirty object
2077 {
2078 bufferlist bl;
2079 bl.append("hi there");
2080 ObjectWriteOperation op;
2081 op.write_full(bl);
2082 ASSERT_EQ(0, ioctx.operate("foo", &op));
2083 }
2084
2085 // flush + flush
2086 {
2087 ObjectReadOperation op;
2088 op.cache_flush();
2089 librados::AioCompletion *completion = cluster.aio_create_completion();
2090 ASSERT_EQ(0, cache_ioctx.aio_operate(
2091 "foo", completion, &op,
2092 librados::OPERATION_IGNORE_OVERLAY, NULL));
2093
2094 ObjectReadOperation op2;
2095 op2.cache_flush();
2096 librados::AioCompletion *completion2 = cluster.aio_create_completion();
2097 ASSERT_EQ(0, cache_ioctx.aio_operate(
2098 "foo", completion2, &op2,
2099 librados::OPERATION_IGNORE_OVERLAY, NULL));
2100
9f95a23c
TL		 2101 completion->wait_for_complete();
2102 completion2->wait_for_complete();
7c673cae
FG		 2103 ASSERT_EQ(0, completion->get_return_value());
2104 ASSERT_EQ(0, completion2->get_return_value());
2105 completion->release();
2106 completion2->release();
2107 }
2108
2109 // create/dirty object
2110 {
2111 bufferlist bl;
2112 bl.append("hi there");
2113 ObjectWriteOperation op;
2114 op.write_full(bl);
2115 ASSERT_EQ(0, ioctx.operate("foo", &op));
2116 }
2117
2118 // flush + try-flush
2119 {
2120 ObjectReadOperation op;
2121 op.cache_flush();
2122 librados::AioCompletion *completion = cluster.aio_create_completion();
2123 ASSERT_EQ(0, cache_ioctx.aio_operate(
2124 "foo", completion, &op,
2125 librados::OPERATION_IGNORE_OVERLAY, NULL));
2126
2127 ObjectReadOperation op2;
2128 op2.cache_try_flush();
2129 librados::AioCompletion *completion2 = cluster.aio_create_completion();
2130 ASSERT_EQ(0, cache_ioctx.aio_operate(
2131 "foo", completion2, &op2,
2132 librados::OPERATION_IGNORE_OVERLAY |
2133 librados::OPERATION_SKIPRWLOCKS, NULL));
2134
9f95a23c
TL		 2135 completion->wait_for_complete();
2136 completion2->wait_for_complete();
7c673cae
FG		 2137 ASSERT_EQ(0, completion->get_return_value());
2138 ASSERT_EQ(0, completion2->get_return_value());
2139 completion->release();
2140 completion2->release();
2141 }
2142
2143 // create/dirty object
2144 int tries = 1000;
2145 do {
2146 {
2147 bufferlist bl;
2148 bl.append("hi there");
2149 ObjectWriteOperation op;
2150 op.write_full(bl);
2151 ASSERT_EQ(0, ioctx.operate("foo", &op));
2152 }
2153
2154 // try-flush + flush
2155 // (flush will not piggyback on try-flush)
2156 {
2157 ObjectReadOperation op;
2158 op.cache_try_flush();
2159 librados::AioCompletion *completion = cluster.aio_create_completion();
2160 ASSERT_EQ(0, cache_ioctx.aio_operate(
2161 "foo", completion, &op,
2162 librados::OPERATION_IGNORE_OVERLAY |
2163 librados::OPERATION_SKIPRWLOCKS, NULL));
2164
2165 ObjectReadOperation op2;
2166 op2.cache_flush();
2167 librados::AioCompletion *completion2 = cluster.aio_create_completion();
2168 ASSERT_EQ(0, cache_ioctx.aio_operate(
2169 "foo", completion2, &op2,
2170 librados::OPERATION_IGNORE_OVERLAY, NULL));
2171
9f95a23c
TL		 2172 completion->wait_for_complete();
2173 completion2->wait_for_complete();
7c673cae
FG		 2174 int r = completion->get_return_value();
2175 ASSERT_TRUE(r == -EBUSY || r == 0);
2176 ASSERT_EQ(0, completion2->get_return_value());
2177 completion->release();
2178 completion2->release();
2179 if (r == -EBUSY)
2180 break;
2181 cout << "didn't get EBUSY, trying again" << std::endl;
2182 }
2183 ASSERT_TRUE(--tries);
2184 } while (true);
2185
2186 // create/dirty object
2187 {
2188 bufferlist bl;
2189 bl.append("hi there");
2190 ObjectWriteOperation op;
2191 op.write_full(bl);
2192 ASSERT_EQ(0, ioctx.operate("foo", &op));
2193 }
2194
2195 // try-flush + try-flush
2196 {
2197 ObjectReadOperation op;
2198 op.cache_try_flush();
2199 librados::AioCompletion *completion = cluster.aio_create_completion();
2200 ASSERT_EQ(0, cache_ioctx.aio_operate(
2201 "foo", completion, &op,
2202 librados::OPERATION_IGNORE_OVERLAY |
2203 librados::OPERATION_SKIPRWLOCKS, NULL));
2204
2205 ObjectReadOperation op2;
2206 op2.cache_try_flush();
2207 librados::AioCompletion *completion2 = cluster.aio_create_completion();
2208 ASSERT_EQ(0, cache_ioctx.aio_operate(
2209 "foo", completion2, &op2,
2210 librados::OPERATION_IGNORE_OVERLAY |
2211 librados::OPERATION_SKIPRWLOCKS, NULL));
2212
9f95a23c
TL		 2213 completion->wait_for_complete();
2214 completion2->wait_for_complete();
7c673cae
FG		 2215 ASSERT_EQ(0, completion->get_return_value());
2216 ASSERT_EQ(0, completion2->get_return_value());
2217 completion->release();
2218 completion2->release();
2219 }
2220}
2221
2222
2223IoCtx *read_ioctx = 0;
9f95a23c
TL		 2224ceph::mutex test_lock = ceph::make_mutex("FlushReadRaces::lock");
2225ceph::condition_variable cond;
7c673cae
FG		 2226int max_reads = 100;
2227int num_reads = 0; // in progress
2228
2229void flush_read_race_cb(completion_t cb, void *arg);
2230
2231void start_flush_read()
2232{
2233 //cout << " starting read" << std::endl;
2234 ObjectReadOperation op;
2235 op.stat(NULL, NULL, NULL);
2236 librados::AioCompletion *completion = completions.getCompletion();
2237 completion->set_complete_callback(0, flush_read_race_cb);
2238 read_ioctx->aio_operate("foo", completion, &op, NULL);
2239}
2240
2241void flush_read_race_cb(completion_t cb, void *arg)
2242{
2243 //cout << " finished read" << std::endl;
9f95a23c 2244 std::lock_guard l{test_lock};
7c673cae
FG		 2245 if (num_reads > max_reads) {
2246 num_reads--;
9f95a23c 2247 cond.notify_all();
7c673cae
FG		 2248 } else {
2249 start_flush_read();
2250 }
7c673cae
FG		 2251}
2252
2253TEST_F(LibRadosTwoPoolsPP, TryFlushReadRace) {
2254 // configure cache
2255 bufferlist inbl;
2256 ASSERT_EQ(0, cluster.mon_command(
2257 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2258 "\", \"tierpool\": \"" + cache_pool_name +
2259 "\", \"force_nonempty\": \"--force-nonempty\" }",
2260 inbl, NULL, NULL));
2261 ASSERT_EQ(0, cluster.mon_command(
2262 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
2263 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
2264 inbl, NULL, NULL));
2265 ASSERT_EQ(0, cluster.mon_command(
2266 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
2267 "\", \"mode\": \"writeback\"}",
2268 inbl, NULL, NULL));
2269
2270 // wait for maps to settle
2271 cluster.wait_for_latest_osdmap();
2272
2273 // create/dirty object
2274 {
2275 bufferlist bl;
2276 bl.append("hi there");
2277 bufferptr bp(4000000); // make it big!
2278 bp.zero();
2279 bl.append(bp);
2280 ObjectWriteOperation op;
2281 op.write_full(bl);
2282 ASSERT_EQ(0, ioctx.operate("foo", &op));
2283 }
2284
2285 // start a continuous stream of reads
2286 read_ioctx = &ioctx;
9f95a23c 2287 test_lock.lock();
7c673cae
FG		 2288 for (int i = 0; i < max_reads; ++i) {
2289 start_flush_read();
2290 num_reads++;
2291 }
9f95a23c 2292 test_lock.unlock();
7c673cae
FG		 2293
2294 // try-flush
2295 ObjectReadOperation op;
2296 op.cache_try_flush();
2297 librados::AioCompletion *completion = cluster.aio_create_completion();
2298 ASSERT_EQ(0, cache_ioctx.aio_operate(
2299 "foo", completion, &op,
2300 librados::OPERATION_IGNORE_OVERLAY |
2301 librados::OPERATION_SKIPRWLOCKS, NULL));
2302
9f95a23c 2303 completion->wait_for_complete();
7c673cae
FG		 2304 ASSERT_EQ(0, completion->get_return_value());
2305 completion->release();
2306
2307 // stop reads
9f95a23c
TL		 2308 std::unique_lock locker{test_lock};
2309 max_reads = 0;
2310 cond.wait(locker, [] { return num_reads == 0;});
7c673cae
FG		 2311}
2312
2313TEST_F(LibRadosTierPP, HitSetNone) {
2314 {
2315 list< pair<time_t,time_t> > ls;
2316 AioCompletion *c = librados::Rados::aio_create_completion();
2317 ASSERT_EQ(0, ioctx.hit_set_list(123, c, &ls));
2318 c->wait_for_complete();
2319 ASSERT_EQ(0, c->get_return_value());
2320 ASSERT_TRUE(ls.empty());
2321 c->release();
2322 }
2323 {
2324 bufferlist bl;
2325 AioCompletion *c = librados::Rados::aio_create_completion();
2326 ASSERT_EQ(0, ioctx.hit_set_get(123, c, 12345, &bl));
2327 c->wait_for_complete();
2328 ASSERT_EQ(-ENOENT, c->get_return_value());
2329 c->release();
2330 }
2331}
2332
2333string set_pool_str(string pool, string var, string val)
2334{
2335 return string("{\"prefix\": \"osd pool set\",\"pool\":\"") + pool
2336 + string("\",\"var\": \"") + var + string("\",\"val\": \"")
2337 + val + string("\"}");
2338}
2339
2340string set_pool_str(string pool, string var, int val)
2341{
2342 return string("{\"prefix\": \"osd pool set\",\"pool\":\"") + pool
2343 + string("\",\"var\": \"") + var + string("\",\"val\": \"")
2344 + stringify(val) + string("\"}");
2345}
2346
2347TEST_F(LibRadosTwoPoolsPP, HitSetRead) {
2348 // make it a tier
2349 bufferlist inbl;
2350 ASSERT_EQ(0, cluster.mon_command(
2351 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2352 "\", \"tierpool\": \"" + cache_pool_name +
2353 "\", \"force_nonempty\": \"--force-nonempty\" }",
2354 inbl, NULL, NULL));
2355
2356 // enable hitset tracking for this pool
2357 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", 2),
2358 inbl, NULL, NULL));
2359 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", 600),
2360 inbl, NULL, NULL));
2361 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type",
2362 "explicit_object"),
2363 inbl, NULL, NULL));
2364
2365 // wait for maps to settle
2366 cluster.wait_for_latest_osdmap();
2367
2368 cache_ioctx.set_namespace("");
2369
2370 // keep reading until we see our object appear in the HitSet
2371 utime_t start = ceph_clock_now();
2372 utime_t hard_stop = start + utime_t(600, 0);
2373
2374 while (true) {
2375 utime_t now = ceph_clock_now();
2376 ASSERT_TRUE(now < hard_stop);
2377
2378 string name = "foo";
2379 uint32_t hash;
2380 ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(name, &hash));
2381 hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash,
2382 cluster.pool_lookup(cache_pool_name.c_str()), "");
2383
2384 bufferlist bl;
2385 ASSERT_EQ(-ENOENT, cache_ioctx.read("foo", bl, 1, 0));
2386
2387 bufferlist hbl;
2388 AioCompletion *c = librados::Rados::aio_create_completion();
2389 ASSERT_EQ(0, cache_ioctx.hit_set_get(hash, c, now.sec(), &hbl));
2390 c->wait_for_complete();
2391 c->release();
2392
2393 if (hbl.length()) {
11fdf7f2 2394 auto p = hbl.cbegin();
7c673cae 2395 HitSet hs;
11fdf7f2 2396 decode(hs, p);
7c673cae
FG		 2397 if (hs.contains(oid)) {
2398 cout << "ok, hit_set contains " << oid << std::endl;
2399 break;
2400 }
2401 cout << "hmm, not in HitSet yet" << std::endl;
2402 } else {
2403 cout << "hmm, no HitSet yet" << std::endl;
2404 }
2405
2406 sleep(1);
2407 }
2408}
2409
2410static int _get_pg_num(Rados& cluster, string pool_name)
2411{
2412 bufferlist inbl;
2413 string cmd = string("{\"prefix\": \"osd pool get\",\"pool\":\"")
2414 + pool_name
2415 + string("\",\"var\": \"pg_num\",\"format\": \"json\"}");
2416 bufferlist outbl;
2417 int r = cluster.mon_command(cmd, inbl, &outbl, NULL);
11fdf7f2 2418 ceph_assert(r >= 0);
7c673cae
FG		 2419 string outstr(outbl.c_str(), outbl.length());
2420 json_spirit::Value v;
2421 if (!json_spirit::read(outstr, v)) {
2422 cerr <<" unable to parse json " << outstr << std::endl;
2423 return -1;
2424 }
2425
2426 json_spirit::Object& o = v.get_obj();
2427 for (json_spirit::Object::size_type i=0; i<o.size(); i++) {
2428 json_spirit::Pair& p = o[i];
2429 if (p.name_ == "pg_num") {
2430 cout << "pg_num = " << p.value_.get_int() << std::endl;
2431 return p.value_.get_int();
2432 }
2433 }
2434 cerr << "didn't find pg_num in " << outstr << std::endl;
2435 return -1;
2436}
2437
7c673cae
FG		 2438TEST_F(LibRadosTwoPoolsPP, HitSetWrite) {
2439 int num_pg = _get_pg_num(cluster, pool_name);
11fdf7f2 2440 ceph_assert(num_pg > 0);
7c673cae
FG		 2441
2442 // make it a tier
2443 bufferlist inbl;
2444 ASSERT_EQ(0, cluster.mon_command(
2445 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2446 "\", \"tierpool\": \"" + cache_pool_name +
2447 "\", \"force_nonempty\": \"--force-nonempty\" }",
2448 inbl, NULL, NULL));
2449
2450 // enable hitset tracking for this pool
2451 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", 8),
2452 inbl, NULL, NULL));
2453 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", 600),
2454 inbl, NULL, NULL));
2455 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type",
2456 "explicit_hash"),
2457 inbl, NULL, NULL));
2458
2459 // wait for maps to settle
2460 cluster.wait_for_latest_osdmap();
2461
2462 cache_ioctx.set_namespace("");
2463
2464 int num = 200;
2465
2466 // do a bunch of writes
2467 for (int i=0; i<num; ++i) {
2468 bufferlist bl;
2469 bl.append("a");
2470 ASSERT_EQ(0, cache_ioctx.write(stringify(i), bl, 1, 0));
2471 }
2472
2473 // get HitSets
2474 std::map<int,HitSet> hitsets;
2475 for (int i=0; i<num_pg; ++i) {
2476 list< pair<time_t,time_t> > ls;
2477 AioCompletion *c = librados::Rados::aio_create_completion();
2478 ASSERT_EQ(0, cache_ioctx.hit_set_list(i, c, &ls));
2479 c->wait_for_complete();
2480 c->release();
2481 std::cout << "pg " << i << " ls " << ls << std::endl;
2482 ASSERT_FALSE(ls.empty());
2483
2484 // get the latest
2485 c = librados::Rados::aio_create_completion();
2486 bufferlist bl;
2487 ASSERT_EQ(0, cache_ioctx.hit_set_get(i, c, ls.back().first, &bl));
2488 c->wait_for_complete();
2489 c->release();
2490
2491 try {
11fdf7f2
TL		 2492 auto p = bl.cbegin();
2493 decode(hitsets[i], p);
7c673cae
FG		 2494 }
2495 catch (buffer::error& e) {
2496 std::cout << "failed to decode hit set; bl len is " << bl.length() << "\n";
2497 bl.hexdump(std::cout);
2498 std::cout << std::endl;
2499 throw e;
2500 }
2501
2502 // cope with racing splits by refreshing pg_num
2503 if (i == num_pg - 1)
2504 num_pg = _get_pg_num(cluster, cache_pool_name);
2505 }
2506
2507 for (int i=0; i<num; ++i) {
2508 string n = stringify(i);
2509 uint32_t hash;
2510 ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(n, &hash));
2511 hobject_t oid(sobject_t(n, CEPH_NOSNAP), "", hash,
2512 cluster.pool_lookup(cache_pool_name.c_str()), "");
2513 std::cout << "checking for " << oid << std::endl;
2514 bool found = false;
2515 for (int p=0; p<num_pg; ++p) {
2516 if (hitsets[p].contains(oid)) {
2517 found = true;
2518 break;
2519 }
2520 }
2521 ASSERT_TRUE(found);
2522 }
2523}
2524
2525TEST_F(LibRadosTwoPoolsPP, HitSetTrim) {
2526 unsigned count = 3;
2527 unsigned period = 3;
2528
2529 // make it a tier
2530 bufferlist inbl;
2531 ASSERT_EQ(0, cluster.mon_command(
2532 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2533 "\", \"tierpool\": \"" + cache_pool_name +
2534 "\", \"force_nonempty\": \"--force-nonempty\" }",
2535 inbl, NULL, NULL));
2536
2537 // enable hitset tracking for this pool
2538 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", count),
2539 inbl, NULL, NULL));
2540 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", period),
2541 inbl, NULL, NULL));
2542 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
2543 inbl, NULL, NULL));
2544 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_fpp", ".01"),
2545 inbl, NULL, NULL));
2546
2547 // wait for maps to settle
2548 cluster.wait_for_latest_osdmap();
2549
2550 cache_ioctx.set_namespace("");
2551
2552 // do a bunch of writes and make sure the hitsets rotate
2553 utime_t start = ceph_clock_now();
2554 utime_t hard_stop = start + utime_t(count * period * 50, 0);
2555
2556 time_t first = 0;
2557 while (true) {
2558 string name = "foo";
2559 uint32_t hash;
2560 ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(name, &hash));
2561 hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash, -1, "");
2562
2563 bufferlist bl;
2564 bl.append("f");
2565 ASSERT_EQ(0, cache_ioctx.write("foo", bl, 1, 0));
2566
2567 list<pair<time_t, time_t> > ls;
2568 AioCompletion *c = librados::Rados::aio_create_completion();
2569 ASSERT_EQ(0, cache_ioctx.hit_set_list(hash, c, &ls));
2570 c->wait_for_complete();
2571 c->release();
2572
2573 cout << " got ls " << ls << std::endl;
2574 if (!ls.empty()) {
2575 if (!first) {
2576 first = ls.front().first;
2577 cout << "first is " << first << std::endl;
2578 } else {
2579 if (ls.front().first != first) {
2580 cout << "first now " << ls.front().first << ", trimmed" << std::endl;
2581 break;
2582 }
2583 }
2584 }
2585
2586 utime_t now = ceph_clock_now();
2587 ASSERT_TRUE(now < hard_stop);
2588
2589 sleep(1);
2590 }
2591}
2592
2593TEST_F(LibRadosTwoPoolsPP, PromoteOn2ndRead) {
2594 // create object
2595 for (int i=0; i<20; ++i) {
2596 bufferlist bl;
2597 bl.append("hi there");
2598 ObjectWriteOperation op;
2599 op.write_full(bl);
2600 ASSERT_EQ(0, ioctx.operate("foo" + stringify(i), &op));
2601 }
2602
2603 // configure cache
2604 bufferlist inbl;
2605 ASSERT_EQ(0, cluster.mon_command(
2606 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2607 "\", \"tierpool\": \"" + cache_pool_name +
2608 "\", \"force_nonempty\": \"--force-nonempty\" }",
2609 inbl, NULL, NULL));
2610 ASSERT_EQ(0, cluster.mon_command(
2611 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
2612 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
2613 inbl, NULL, NULL));
2614 ASSERT_EQ(0, cluster.mon_command(
2615 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
2616 "\", \"mode\": \"writeback\"}",
2617 inbl, NULL, NULL));
2618
2619 // enable hitset tracking for this pool
2620 ASSERT_EQ(0, cluster.mon_command(
2621 set_pool_str(cache_pool_name, "hit_set_count", 2),
2622 inbl, NULL, NULL));
2623 ASSERT_EQ(0, cluster.mon_command(
2624 set_pool_str(cache_pool_name, "hit_set_period", 600),
2625 inbl, NULL, NULL));
2626 ASSERT_EQ(0, cluster.mon_command(
2627 set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
2628 inbl, NULL, NULL));
2629 ASSERT_EQ(0, cluster.mon_command(
2630 set_pool_str(cache_pool_name, "min_read_recency_for_promote", 1),
2631 inbl, NULL, NULL));
2632 ASSERT_EQ(0, cluster.mon_command(
2633 set_pool_str(cache_pool_name, "hit_set_grade_decay_rate", 20),
2634 inbl, NULL, NULL));
2635 ASSERT_EQ(0, cluster.mon_command(
2636 set_pool_str(cache_pool_name, "hit_set_search_last_n", 1),
2637 inbl, NULL, NULL));
2638
2639 // wait for maps to settle
2640 cluster.wait_for_latest_osdmap();
2641
2642 int fake = 0; // set this to non-zero to test spurious promotion,
2643 // e.g. from thrashing
2644 int attempt = 0;
2645 string obj;
2646 while (true) {
2647 // 1st read, don't trigger a promote
2648 obj = "foo" + stringify(attempt);
2649 cout << obj << std::endl;
2650 {
2651 bufferlist bl;
2652 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
2653 if (--fake >= 0) {
2654 sleep(1);
2655 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
2656 sleep(1);
2657 }
2658 }
2659
2660 // verify the object is NOT present in the cache tier
2661 {
2662 bool found = false;
2663 NObjectIterator it = cache_ioctx.nobjects_begin();
2664 while (it != cache_ioctx.nobjects_end()) {
2665 cout << " see " << it->get_oid() << std::endl;
2666 if (it->get_oid() == string(obj.c_str())) {
2667 found = true;
2668 break;
2669 }
2670 ++it;
2671 }
2672 if (!found)
2673 break;
2674 }
2675
2676 ++attempt;
2677 ASSERT_LE(attempt, 20);
2678 cout << "hrm, object is present in cache on attempt " << attempt
2679 << ", retrying" << std::endl;
2680 }
2681
2682 // Read until the object is present in the cache tier
2683 cout << "verifying " << obj << " is eventually promoted" << std::endl;
2684 while (true) {
2685 bufferlist bl;
2686 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
2687
2688 bool there = false;
2689 NObjectIterator it = cache_ioctx.nobjects_begin();
2690 while (it != cache_ioctx.nobjects_end()) {
2691 if (it->get_oid() == string(obj.c_str())) {
2692 there = true;
2693 break;
2694 }
2695 ++it;
2696 }
2697 if (there)
2698 break;
2699
2700 sleep(1);
2701 }
2702
2703 // tear down tiers
2704 ASSERT_EQ(0, cluster.mon_command(
2705 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
2706 "\"}",
2707 inbl, NULL, NULL));
2708 ASSERT_EQ(0, cluster.mon_command(
2709 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
2710 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
2711 inbl, NULL, NULL));
2712
2713 // wait for maps to settle before next test
2714 cluster.wait_for_latest_osdmap();
2715}
2716
2717TEST_F(LibRadosTwoPoolsPP, ProxyRead) {
2718 // create object
2719 {
2720 bufferlist bl;
2721 bl.append("hi there");
2722 ObjectWriteOperation op;
2723 op.write_full(bl);
2724 ASSERT_EQ(0, ioctx.operate("foo", &op));
2725 }
2726
2727 // configure cache
2728 bufferlist inbl;
2729 ASSERT_EQ(0, cluster.mon_command(
2730 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2731 "\", \"tierpool\": \"" + cache_pool_name +
2732 "\", \"force_nonempty\": \"--force-nonempty\" }",
2733 inbl, NULL, NULL));
2734 ASSERT_EQ(0, cluster.mon_command(
2735 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
2736 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
2737 inbl, NULL, NULL));
2738 ASSERT_EQ(0, cluster.mon_command(
2739 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
2740 "\", \"mode\": \"readproxy\"}",
2741 inbl, NULL, NULL));
2742
2743 // wait for maps to settle
2744 cluster.wait_for_latest_osdmap();
2745
2746 // read and verify the object
2747 {
2748 bufferlist bl;
2749 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
2750 ASSERT_EQ('h', bl[0]);
2751 }
2752
2753 // Verify 10 times the object is NOT present in the cache tier
2754 uint32_t i = 0;
2755 while (i++ < 10) {
2756 NObjectIterator it = cache_ioctx.nobjects_begin();
2757 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
2758 sleep(1);
2759 }
2760
2761 // tear down tiers
2762 ASSERT_EQ(0, cluster.mon_command(
2763 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
2764 "\"}",
2765 inbl, NULL, NULL));
2766 ASSERT_EQ(0, cluster.mon_command(
2767 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
2768 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
2769 inbl, NULL, NULL));
2770
2771 // wait for maps to settle before next test
2772 cluster.wait_for_latest_osdmap();
2773}
2774
2775TEST_F(LibRadosTwoPoolsPP, CachePin) {
2776 // create object
2777 {
2778 bufferlist bl;
2779 bl.append("hi there");
2780 ObjectWriteOperation op;
2781 op.write_full(bl);
2782 ASSERT_EQ(0, ioctx.operate("foo", &op));
2783 }
2784 {
2785 bufferlist bl;
2786 bl.append("hi there");
2787 ObjectWriteOperation op;
2788 op.write_full(bl);
2789 ASSERT_EQ(0, ioctx.operate("bar", &op));
2790 }
2791 {
2792 bufferlist bl;
2793 bl.append("hi there");
2794 ObjectWriteOperation op;
2795 op.write_full(bl);
2796 ASSERT_EQ(0, ioctx.operate("baz", &op));
2797 }
2798 {
2799 bufferlist bl;
2800 bl.append("hi there");
2801 ObjectWriteOperation op;
2802 op.write_full(bl);
2803 ASSERT_EQ(0, ioctx.operate("bam", &op));
2804 }
2805
2806 // configure cache
2807 bufferlist inbl;
2808 ASSERT_EQ(0, cluster.mon_command(
2809 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2810 "\", \"tierpool\": \"" + cache_pool_name +
2811 "\", \"force_nonempty\": \"--force-nonempty\" }",
2812 inbl, NULL, NULL));
2813 ASSERT_EQ(0, cluster.mon_command(
2814 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
2815 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
2816 inbl, NULL, NULL));
2817 ASSERT_EQ(0, cluster.mon_command(
2818 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
2819 "\", \"mode\": \"writeback\"}",
2820 inbl, NULL, NULL));
2821
2822 // wait for maps to settle
2823 cluster.wait_for_latest_osdmap();
2824
2825 // read, trigger promote
2826 {
2827 bufferlist bl;
2828 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
2829 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
2830 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
2831 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
2832 }
2833
2834 // verify the objects are present in the cache tier
2835 {
2836 NObjectIterator it = cache_ioctx.nobjects_begin();
2837 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
2838 for (uint32_t i = 0; i < 4; i++) {
2839 ASSERT_TRUE(it->get_oid() == string("foo") ||
2840 it->get_oid() == string("bar") ||
2841 it->get_oid() == string("baz") ||
2842 it->get_oid() == string("bam"));
2843 ++it;
2844 }
2845 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
2846 }
2847
2848 // pin objects
2849 {
2850 ObjectWriteOperation op;
2851 op.cache_pin();
2852 librados::AioCompletion *completion = cluster.aio_create_completion();
2853 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 2854 completion->wait_for_complete();
7c673cae
FG		 2855 ASSERT_EQ(0, completion->get_return_value());
2856 completion->release();
2857 }
2858 {
2859 ObjectWriteOperation op;
2860 op.cache_pin();
2861 librados::AioCompletion *completion = cluster.aio_create_completion();
2862 ASSERT_EQ(0, cache_ioctx.aio_operate("baz", completion, &op));
9f95a23c 2863 completion->wait_for_complete();
7c673cae
FG		 2864 ASSERT_EQ(0, completion->get_return_value());
2865 completion->release();
2866 }
2867
2868 // enable agent
2869 ASSERT_EQ(0, cluster.mon_command(
2870 set_pool_str(cache_pool_name, "hit_set_count", 2),
2871 inbl, NULL, NULL));
2872 ASSERT_EQ(0, cluster.mon_command(
2873 set_pool_str(cache_pool_name, "hit_set_period", 600),
2874 inbl, NULL, NULL));
2875 ASSERT_EQ(0, cluster.mon_command(
2876 set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
2877 inbl, NULL, NULL));
2878 ASSERT_EQ(0, cluster.mon_command(
2879 set_pool_str(cache_pool_name, "min_read_recency_for_promote", 1),
2880 inbl, NULL, NULL));
2881 ASSERT_EQ(0, cluster.mon_command(
2882 set_pool_str(cache_pool_name, "target_max_objects", 1),
2883 inbl, NULL, NULL));
2884
2885 sleep(10);
2886
2887 // Verify the pinned object 'foo' is not flushed/evicted
2888 uint32_t count = 0;
2889 while (true) {
2890 bufferlist bl;
2891 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
2892
2893 count = 0;
2894 NObjectIterator it = cache_ioctx.nobjects_begin();
2895 while (it != cache_ioctx.nobjects_end()) {
2896 ASSERT_TRUE(it->get_oid() == string("foo") ||
2897 it->get_oid() == string("bar") ||
2898 it->get_oid() == string("baz") ||
2899 it->get_oid() == string("bam"));
2900 ++count;
2901 ++it;
2902 }
2903 if (count == 2) {
2904 ASSERT_TRUE(it->get_oid() == string("foo") ||
2905 it->get_oid() == string("baz"));
2906 break;
2907 }
2908
2909 sleep(1);
2910 }
2911
2912 // tear down tiers
2913 ASSERT_EQ(0, cluster.mon_command(
2914 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
2915 "\"}",
2916 inbl, NULL, NULL));
2917 ASSERT_EQ(0, cluster.mon_command(
2918 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
2919 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
2920 inbl, NULL, NULL));
2921
2922 // wait for maps to settle before next test
2923 cluster.wait_for_latest_osdmap();
2924}
2925
11fdf7f2
TL		 2926TEST_F(LibRadosTwoPoolsPP, SetRedirectRead) {
2927 // create object
2928 {
2929 bufferlist bl;
2930 bl.append("hi there");
2931 ObjectWriteOperation op;
2932 op.write_full(bl);
2933 ASSERT_EQ(0, ioctx.operate("foo", &op));
2934 }
2935 {
2936 bufferlist bl;
2937 bl.append("there");
2938 ObjectWriteOperation op;
2939 op.write_full(bl);
2940 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
2941 }
2942
2943 // configure tier
2944 bufferlist inbl;
2945 ASSERT_EQ(0, cluster.mon_command(
2946 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
2947 "\", \"tierpool\": \"" + cache_pool_name +
2948 "\", \"force_nonempty\": \"--force-nonempty\" }",
2949 inbl, NULL, NULL));
2950
2951 // wait for maps to settle
2952 cluster.wait_for_latest_osdmap();
2953
2954 {
2955 ObjectWriteOperation op;
2956 op.set_redirect("bar", cache_ioctx, 0);
2957 librados::AioCompletion *completion = cluster.aio_create_completion();
2958 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 2959 completion->wait_for_complete();
11fdf7f2
TL		 2960 ASSERT_EQ(0, completion->get_return_value());
2961 completion->release();
2962 }
2963 // read and verify the object
2964 {
2965 bufferlist bl;
2966 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
2967 ASSERT_EQ('t', bl[0]);
2968 }
2969
2970 ASSERT_EQ(0, cluster.mon_command(
2971 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
2972 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
2973 inbl, NULL, NULL));
2974
2975 // wait for maps to settle before next test
2976 cluster.wait_for_latest_osdmap();
2977}
2978
2979TEST_F(LibRadosTwoPoolsPP, SetChunkRead) {
2980 // skip test if not yet mimic
9f95a23c
TL		 2981 if (_get_required_osd_release(cluster) < "mimic") {
2982 GTEST_SKIP() << "cluster is not yet mimic, skipping test";
11fdf7f2
TL		 2983 }
2984
2985 // create object
2986 {
2987 ObjectWriteOperation op;
2988 op.create(true);
2989 ASSERT_EQ(0, ioctx.operate("foo", &op));
2990 }
2991 {
2992 bufferlist bl;
2993 bl.append("hi there");
2994 ObjectWriteOperation op;
2995 op.write_full(bl);
2996 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
2997 }
2998
2999 // configure tier
3000 bufferlist inbl;
3001 ASSERT_EQ(0, cluster.mon_command(
3002 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
3003 "\", \"tierpool\": \"" + cache_pool_name +
3004 "\", \"force_nonempty\": \"--force-nonempty\" }",
3005 inbl, NULL, NULL));
3006
3007 // wait for maps to settle
3008 cluster.wait_for_latest_osdmap();
3009
3010 // set_chunk
3011 {
3012 ObjectWriteOperation op;
3013 int len = strlen("hi there");
3014 for (int i = 0; i < len; i+=2) {
3015 op.set_chunk(i, 2, cache_ioctx, "bar", i);
3016 }
3017 librados::AioCompletion *completion = cluster.aio_create_completion();
3018 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 3019 completion->wait_for_complete();
11fdf7f2
TL		 3020 ASSERT_EQ(0, completion->get_return_value());
3021 completion->release();
3022 }
3023
3024 // make all chunks dirty --> full flush --> all chunks are evicted
3025 {
3026 bufferlist bl;
3027 bl.append("There hi");
3028 ObjectWriteOperation op;
3029 op.write_full(bl);
3030 ASSERT_EQ(0, ioctx.operate("foo", &op));
3031 }
3032
3033 // read and verify the object
3034 {
3035 bufferlist bl;
3036 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
3037 ASSERT_EQ('T', bl[0]);
3038 }
3039
3040 ASSERT_EQ(0, cluster.mon_command(
3041 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
3042 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
3043 inbl, NULL, NULL));
3044
3045 // wait for maps to settle before next test
3046 cluster.wait_for_latest_osdmap();
3047}
3048
3049TEST_F(LibRadosTwoPoolsPP, ManifestPromoteRead) {
3050 // skip test if not yet mimic
9f95a23c
TL		 3051 if (_get_required_osd_release(cluster) < "mimic") {
3052 GTEST_SKIP() << "cluster is not yet mimic, skipping test";
11fdf7f2
TL		 3053 }
3054
3055 // create object
3056 {
3057 bufferlist bl;
3058 bl.append("hi there");
3059 ObjectWriteOperation op;
3060 op.write_full(bl);
3061 ASSERT_EQ(0, ioctx.operate("foo", &op));
3062 }
3063 {
3064 bufferlist bl;
3065 bl.append("base chunk");
3066 ObjectWriteOperation op;
3067 op.write_full(bl);
3068 ASSERT_EQ(0, ioctx.operate("foo-chunk", &op));
3069 }
3070 {
3071 bufferlist bl;
3072 bl.append("there");
3073 ObjectWriteOperation op;
3074 op.write_full(bl);
3075 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
3076 }
3077 {
3078 bufferlist bl;
3079 bl.append("CHUNK");
3080 ObjectWriteOperation op;
3081 op.write_full(bl);
3082 ASSERT_EQ(0, cache_ioctx.operate("bar-chunk", &op));
3083 }
3084
3085 // configure tier
3086 bufferlist inbl;
3087 ASSERT_EQ(0, cluster.mon_command(
3088 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
3089 "\", \"tierpool\": \"" + cache_pool_name +
3090 "\", \"force_nonempty\": \"--force-nonempty\" }",
3091 inbl, NULL, NULL));
3092
3093 // wait for maps to settle
3094 cluster.wait_for_latest_osdmap();
3095
3096 // set-redirect
3097 {
3098 ObjectWriteOperation op;
3099 op.set_redirect("bar", cache_ioctx, 0);
3100 librados::AioCompletion *completion = cluster.aio_create_completion();
3101 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 3102 completion->wait_for_complete();
11fdf7f2
TL		 3103 ASSERT_EQ(0, completion->get_return_value());
3104 completion->release();
3105 }
3106 // set-chunk
3107 {
3108 ObjectWriteOperation op;
3109 op.set_chunk(0, 2, cache_ioctx, "bar-chunk", 0);
3110 librados::AioCompletion *completion = cluster.aio_create_completion();
3111 ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
9f95a23c 3112 completion->wait_for_complete();
11fdf7f2
TL		 3113 ASSERT_EQ(0, completion->get_return_value());
3114 completion->release();
3115 }
3116 // promote
3117 {
3118 ObjectWriteOperation op;
3119 op.tier_promote();
3120 librados::AioCompletion *completion = cluster.aio_create_completion();
3121 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 3122 completion->wait_for_complete();
11fdf7f2
TL		 3123 ASSERT_EQ(0, completion->get_return_value());
3124 completion->release();
3125 }
3126 // read and verify the object (redirect)
3127 {
3128 bufferlist bl;
3129 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
3130 ASSERT_EQ('t', bl[0]);
3131 }
3132 // promote
3133 {
3134 ObjectWriteOperation op;
3135 op.tier_promote();
3136 librados::AioCompletion *completion = cluster.aio_create_completion();
3137 ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
9f95a23c 3138 completion->wait_for_complete();
11fdf7f2
TL		 3139 ASSERT_EQ(0, completion->get_return_value());
3140 completion->release();
3141 }
3142 // read and verify the object
3143 {
3144 bufferlist bl;
3145 ASSERT_EQ(1, ioctx.read("foo-chunk", bl, 1, 0));
3146 ASSERT_EQ('C', bl[0]);
3147 }
3148
3149 ASSERT_EQ(0, cluster.mon_command(
3150 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
3151 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
3152 inbl, NULL, NULL));
3153
3154 // wait for maps to settle before next test
3155 cluster.wait_for_latest_osdmap();
3156}
3157
3158TEST_F(LibRadosTwoPoolsPP, ManifestRefRead) {
3159 // note: require >= mimic
3160
3161 // create object
3162 {
3163 bufferlist bl;
3164 bl.append("hi there");
3165 ObjectWriteOperation op;
3166 op.write_full(bl);
3167 ASSERT_EQ(0, ioctx.operate("foo", &op));
3168 }
3169 {
3170 bufferlist bl;
3171 bl.append("base chunk");
3172 ObjectWriteOperation op;
3173 op.write_full(bl);
3174 ASSERT_EQ(0, ioctx.operate("foo-chunk", &op));
3175 }
3176 {
3177 bufferlist bl;
3178 bl.append("there");
3179 ObjectWriteOperation op;
3180 op.write_full(bl);
3181 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
3182 }
3183 {
3184 bufferlist bl;
3185 bl.append("CHUNK");
3186 ObjectWriteOperation op;
3187 op.write_full(bl);
3188 ASSERT_EQ(0, cache_ioctx.operate("bar-chunk", &op));
3189 }
3190
3191 // wait for maps to settle
3192 cluster.wait_for_latest_osdmap();
3193
3194 // set-redirect
3195 {
3196 ObjectWriteOperation op;
3197 op.set_redirect("bar", cache_ioctx, 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE);
3198 librados::AioCompletion *completion = cluster.aio_create_completion();
3199 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 3200 completion->wait_for_complete();
11fdf7f2
TL		 3201 ASSERT_EQ(0, completion->get_return_value());
3202 completion->release();
3203 }
3204 // set-chunk
3205 {
3206 ObjectWriteOperation op;
3207 op.set_chunk(0, 2, cache_ioctx, "bar-chunk", 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE);
3208 librados::AioCompletion *completion = cluster.aio_create_completion();
3209 ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
9f95a23c 3210 completion->wait_for_complete();
11fdf7f2
TL		 3211 ASSERT_EQ(0, completion->get_return_value());
3212 completion->release();
3213 }
3214 // redirect's refcount
3215 {
3216 bufferlist in, out;
3217 cache_ioctx.exec("bar", "cas", "chunk_read", in, out);
3218 cls_chunk_refcount_read_ret read_ret;
3219 try {
3220 auto iter = out.cbegin();
3221 decode(read_ret, iter);
3222 } catch (buffer::error& err) {
3223 ASSERT_TRUE(0);
3224 }
3225 ASSERT_EQ(1U, read_ret.refs.size());
3226 }
3227 // chunk's refcount
3228 {
3229 bufferlist in, out;
3230 cache_ioctx.exec("bar-chunk", "cas", "chunk_read", in, out);
3231 cls_chunk_refcount_read_ret read_ret;
3232 try {
3233 auto iter = out.cbegin();
3234 decode(read_ret, iter);
3235 } catch (buffer::error& err) {
3236 ASSERT_TRUE(0);
3237 }
3238 ASSERT_EQ(1u, read_ret.refs.size());
3239 }
3240
3241 // wait for maps to settle before next test
3242 cluster.wait_for_latest_osdmap();
3243}
3244
3245TEST_F(LibRadosTwoPoolsPP, ManifestUnset) {
3246 // skip test if not yet nautilus
9f95a23c
TL		 3247 if (_get_required_osd_release(cluster) < "nautilus") {
3248 GTEST_SKIP() << "cluster is not yet nautilus, skipping test";
11fdf7f2
TL		 3249 }
3250
3251 // create object
3252 {
3253 bufferlist bl;
3254 bl.append("hi there");
3255 ObjectWriteOperation op;
3256 op.write_full(bl);
3257 ASSERT_EQ(0, ioctx.operate("foo", &op));
3258 }
3259 {
3260 bufferlist bl;
3261 bl.append("base chunk");
3262 ObjectWriteOperation op;
3263 op.write_full(bl);
3264 ASSERT_EQ(0, ioctx.operate("foo-chunk", &op));
3265 }
3266 {
3267 bufferlist bl;
3268 bl.append("there");
3269 ObjectWriteOperation op;
3270 op.write_full(bl);
3271 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
3272 }
3273 {
3274 bufferlist bl;
3275 bl.append("CHUNK");
3276 ObjectWriteOperation op;
3277 op.write_full(bl);
3278 ASSERT_EQ(0, cache_ioctx.operate("bar-chunk", &op));
3279 }
3280
3281 // wait for maps to settle
3282 cluster.wait_for_latest_osdmap();
3283
3284 // set-redirect
3285 {
3286 ObjectWriteOperation op;
3287 op.set_redirect("bar", cache_ioctx, 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE);
3288 librados::AioCompletion *completion = cluster.aio_create_completion();
3289 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 3290 completion->wait_for_complete();
11fdf7f2
TL		 3291 ASSERT_EQ(0, completion->get_return_value());
3292 completion->release();
3293 }
3294 // set-chunk
3295 {
3296 ObjectWriteOperation op;
3297 op.set_chunk(0, 2, cache_ioctx, "bar-chunk", 0, CEPH_OSD_OP_FLAG_WITH_REFERENCE);
3298 librados::AioCompletion *completion = cluster.aio_create_completion();
3299 ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
9f95a23c 3300 completion->wait_for_complete();
11fdf7f2
TL		 3301 ASSERT_EQ(0, completion->get_return_value());
3302 completion->release();
3303 }
3304 // redirect's refcount
3305 {
3306 bufferlist in, out;
3307 cache_ioctx.exec("bar", "cas", "chunk_read", in, out);
3308 cls_chunk_refcount_read_ret read_ret;
3309 try {
3310 auto iter = out.cbegin();
3311 decode(read_ret, iter);
3312 } catch (buffer::error& err) {
3313 ASSERT_TRUE(0);
3314 }
3315 ASSERT_EQ(1u, read_ret.refs.size());
3316 }
3317 // chunk's refcount
3318 {
3319 bufferlist in, out;
3320 cache_ioctx.exec("bar-chunk", "cas", "chunk_read", in, out);
3321 cls_chunk_refcount_read_ret read_ret;
3322 try {
3323 auto iter = out.cbegin();
3324 decode(read_ret, iter);
3325 } catch (buffer::error& err) {
3326 ASSERT_TRUE(0);
3327 }
3328 ASSERT_EQ(1u, read_ret.refs.size());
3329 }
3330
3331 // unset-manifest for set-redirect
3332 {
3333 ObjectWriteOperation op;
3334 op.unset_manifest();
3335 librados::AioCompletion *completion = cluster.aio_create_completion();
3336 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 3337 completion->wait_for_complete();
11fdf7f2
TL		 3338 ASSERT_EQ(0, completion->get_return_value());
3339 completion->release();
3340 }
3341
3342 // unset-manifest for set-chunk
3343 {
3344 ObjectWriteOperation op;
3345 op.unset_manifest();
3346 librados::AioCompletion *completion = cluster.aio_create_completion();
3347 ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
9f95a23c 3348 completion->wait_for_complete();
11fdf7f2
TL		 3349 ASSERT_EQ(0, completion->get_return_value());
3350 completion->release();
3351 }
3352 // redirect's refcount
3353 {
3354 bufferlist in, out;
3355 cache_ioctx.exec("bar", "cas", "chunk_read", in, out);
3356 if (out.length() != 0U) {
3357 ObjectWriteOperation op;
3358 op.unset_manifest();
3359 librados::AioCompletion *completion = cluster.aio_create_completion();
3360 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 3361 completion->wait_for_complete();
11fdf7f2
TL		 3362 ASSERT_EQ(-EOPNOTSUPP, completion->get_return_value());
3363 completion->release();
3364 }
3365 }
3366 // chunk's refcount
3367 {
3368 bufferlist in, out;
3369 cache_ioctx.exec("bar-chunk", "cas", "chunk_read", in, out);
3370 if (out.length() != 0U) {
3371 ObjectWriteOperation op;
3372 op.unset_manifest();
3373 librados::AioCompletion *completion = cluster.aio_create_completion();
3374 ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
9f95a23c 3375 completion->wait_for_complete();
11fdf7f2
TL		 3376 ASSERT_EQ(-EOPNOTSUPP, completion->get_return_value());
3377 completion->release();
3378 }
3379 }
3380
3381 // wait for maps to settle before next test
3382 cluster.wait_for_latest_osdmap();
3383}
3384
3385#include "common/ceph_crypto.h"
3386using ceph::crypto::SHA1;
3387#include "rgw/rgw_common.h"
3388TEST_F(LibRadosTwoPoolsPP, ManifestDedupRefRead) {
3389 // skip test if not yet nautilus
9f95a23c
TL		 3390 if (_get_required_osd_release(cluster) < "nautilus") {
3391 GTEST_SKIP() << "cluster is not yet nautilus, skipping test";
31f18b77 3392 }
9f95a23c 3393
11fdf7f2
TL		 3394 bufferlist inbl;
3395 ASSERT_EQ(0, cluster.mon_command(
3396 set_pool_str(pool_name, "fingerprint_algorithm", "sha1"),
3397 inbl, NULL, NULL));
3398 cluster.wait_for_latest_osdmap();
31f18b77
FG		 3399
3400 // create object
3401 {
3402 bufferlist bl;
3403 bl.append("hi there");
3404 ObjectWriteOperation op;
3405 op.write_full(bl);
3406 ASSERT_EQ(0, ioctx.operate("foo", &op));
3407 }
11fdf7f2
TL		 3408 {
3409 bufferlist bl;
3410 bl.append("hi there");
3411 ObjectWriteOperation op;
3412 op.write_full(bl);
3413 ASSERT_EQ(0, ioctx.operate("foo-dedup", &op));
3414 }
31f18b77
FG		 3415 {
3416 bufferlist bl;
3417 bl.append("there");
3418 ObjectWriteOperation op;
3419 op.write_full(bl);
3420 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
3421 }
11fdf7f2
TL		 3422 {
3423 bufferlist bl;
3424 bl.append("CHUNK");
3425 ObjectWriteOperation op;
3426 op.write_full(bl);
3427 ASSERT_EQ(0, cache_ioctx.operate("bar-chunk", &op));
3428 }
31f18b77
FG		 3429
3430 // wait for maps to settle
3431 cluster.wait_for_latest_osdmap();
3432
11fdf7f2 3433 // set-chunk (dedup)
31f18b77
FG		 3434 {
3435 ObjectWriteOperation op;
11fdf7f2
TL		 3436 int len = strlen("hi there");
3437 op.set_chunk(0, len, cache_ioctx, "bar-chunk", 0,
3438 CEPH_OSD_OP_FLAG_WITH_REFERENCE);
3439 librados::AioCompletion *completion = cluster.aio_create_completion();
3440 ASSERT_EQ(0, ioctx.aio_operate("foo-dedup", completion, &op));
9f95a23c 3441 completion->wait_for_complete();
11fdf7f2
TL		 3442 ASSERT_EQ(0, completion->get_return_value());
3443 completion->release();
3444 }
3445 // set-chunk (dedup)
3446 {
3447 ObjectWriteOperation op;
3448 int len = strlen("hi there");
3449 op.set_chunk(0, len, cache_ioctx, "bar", 0,
3450 CEPH_OSD_OP_FLAG_WITH_REFERENCE);
31f18b77
FG		 3451 librados::AioCompletion *completion = cluster.aio_create_completion();
3452 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 3453 completion->wait_for_complete();
31f18b77
FG		 3454 ASSERT_EQ(0, completion->get_return_value());
3455 completion->release();
3456 }
11fdf7f2 3457 // make all chunks dirty --> flush
31f18b77 3458 {
11fdf7f2 3459 // make a dirty chunks
31f18b77 3460 bufferlist bl;
11fdf7f2
TL		 3461 bl.append("There hi");
3462 ObjectWriteOperation op;
3463 op.write_full(bl);
3464 ASSERT_EQ(0, ioctx.operate("foo-dedup", &op));
3465 }
3466 {
3467 // do flush
3468 bufferlist bl;
3469 bl.append("There hi");
3470 ObjectWriteOperation op;
3471 op.write_full(bl);
3472 ASSERT_EQ(0, ioctx.operate("foo-dedup", &op));
3473 }
3474 {
3475 // make a dirty chunks
3476 bufferlist bl;
3477 bl.append("There hi");
3478 ObjectWriteOperation op;
3479 op.write_full(bl);
3480 ASSERT_EQ(0, ioctx.operate("foo", &op));
3481 }
3482 {
3483 // do flush
3484 bufferlist bl;
3485 bl.append("There hi");
3486 ObjectWriteOperation op;
3487 op.write_full(bl);
3488 ASSERT_EQ(0, ioctx.operate("foo", &op));
3489 }
3490 // chunk's refcount
3491 {
3492 bufferlist in, out;
3493 SHA1 sha1_gen;
3494 int size = strlen("There hi");
3495 unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
3496 char p_str[CEPH_CRYPTO_SHA1_DIGESTSIZE*2+1] = {0};
3497 sha1_gen.Update((const unsigned char *)"There hi", size);
3498 sha1_gen.Final(fingerprint);
3499 buf_to_hex(fingerprint, CEPH_CRYPTO_SHA1_DIGESTSIZE, p_str);
3500 cache_ioctx.exec(p_str, "cas", "chunk_read", in, out);
3501 cls_chunk_refcount_read_ret read_ret;
3502 try {
3503 auto iter = out.cbegin();
3504 decode(read_ret, iter);
3505 } catch (buffer::error& err) {
3506 ASSERT_TRUE(0);
3507 }
3508 ASSERT_EQ(2u, read_ret.refs.size());
31f18b77 3509 }
31f18b77
FG		 3510
3511 // wait for maps to settle before next test
3512 cluster.wait_for_latest_osdmap();
3513}
3514
9f95a23c
TL		 3515TEST_F(LibRadosTwoPoolsPP, ManifestFlushRead) {
3516 // skip test if not yet octopus
3517 if (_get_required_osd_release(cluster) < "octopus") {
3518 GTEST_SKIP() << "cluster is not yet octopus, skipping test";
3519 }
3520
3521 // create object
3522 {
3523 bufferlist bl;
3524 bl.append("base chunk");
3525 ObjectWriteOperation op;
3526 op.write_full(bl);
3527 ASSERT_EQ(0, ioctx.operate("foo-chunk", &op));
3528 }
3529 {
3530 bufferlist bl;
3531 bl.append("CHUNKS");
3532 ObjectWriteOperation op;
3533 op.write_full(bl);
3534 ASSERT_EQ(0, cache_ioctx.operate("bar-chunk", &op));
3535 }
3536
3537 // configure tier
3538 bufferlist inbl;
3539 ASSERT_EQ(0, cluster.mon_command(
3540 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
3541 "\", \"tierpool\": \"" + cache_pool_name +
3542 "\", \"force_nonempty\": \"--force-nonempty\" }",
3543 inbl, NULL, NULL));
3544
3545 // wait for maps to settle
3546 cluster.wait_for_latest_osdmap();
3547
3548 // set-chunk
3549 {
3550 ObjectWriteOperation op;
3551 op.set_chunk(0, 2, cache_ioctx, "bar-chunk", 0);
3552 librados::AioCompletion *completion = cluster.aio_create_completion();
3553 ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
3554 completion->wait_for_complete();
3555 ASSERT_EQ(0, completion->get_return_value());
3556 completion->release();
3557 }
3558 // set-chunk
3559 {
3560 ObjectWriteOperation op;
3561 op.set_chunk(2, 2, cache_ioctx, "bar-chunk", 2);
3562 librados::AioCompletion *completion = cluster.aio_create_completion();
3563 ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
3564 completion->wait_for_complete();
3565 ASSERT_EQ(0, completion->get_return_value());
3566 completion->release();
3567 }
3568 // make chunked object dirty
3569 {
3570 bufferlist bl;
3571 bl.append("DD");
3572 ObjectWriteOperation op;
3573 op.write(0, bl);
3574 ASSERT_EQ(0, ioctx.operate("foo-chunk", &op));
3575 }
3576 // flush
3577 {
3578 ObjectWriteOperation op;
3579 op.tier_flush();
3580 librados::AioCompletion *completion = cluster.aio_create_completion();
3581 ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
3582 completion->wait_for_complete();
3583 ASSERT_EQ(0, completion->get_return_value());
3584 completion->release();
3585 }
3586 // read and verify the chunked object
3587 {
3588 bufferlist bl;
3589 ASSERT_EQ(1, cache_ioctx.read("bar-chunk", bl, 1, 0));
3590 ASSERT_EQ('D', bl[0]);
3591 }
3592
3593 ASSERT_EQ(0, cluster.mon_command(
3594 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
3595 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
3596 inbl, NULL, NULL));
3597
3598 // wait for maps to settle before next test
3599 cluster.wait_for_latest_osdmap();
3600}
3601
7c673cae
FG		 3602class LibRadosTwoPoolsECPP : public RadosTestECPP
3603{
3604public:
3605 LibRadosTwoPoolsECPP() {};
3606 ~LibRadosTwoPoolsECPP() override {};
3607protected:
3608 static void SetUpTestCase() {
3609 pool_name = get_temp_pool_name();
3610 ASSERT_EQ("", create_one_ec_pool_pp(pool_name, s_cluster));
3611 }
3612 static void TearDownTestCase() {
3613 ASSERT_EQ(0, destroy_one_ec_pool_pp(pool_name, s_cluster));
3614 }
3615 static std::string cache_pool_name;
3616
3617 void SetUp() override {
3618 cache_pool_name = get_temp_pool_name();
3619 ASSERT_EQ(0, s_cluster.pool_create(cache_pool_name.c_str()));
3620 RadosTestECPP::SetUp();
c07f9fc5 3621
7c673cae 3622 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
c07f9fc5 3623 cache_ioctx.application_enable("rados", true);
7c673cae
FG		 3624 cache_ioctx.set_namespace(nspace);
3625 }
3626 void TearDown() override {
3627 // flush + evict cache
3628 flush_evict_all(cluster, cache_ioctx);
3629
3630 bufferlist inbl;
3631 // tear down tiers
3632 ASSERT_EQ(0, cluster.mon_command(
3633 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
3634 "\"}",
3635 inbl, NULL, NULL));
3636 ASSERT_EQ(0, cluster.mon_command(
3637 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
3638 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
3639 inbl, NULL, NULL));
3640
3641 // wait for maps to settle before next test
3642 cluster.wait_for_latest_osdmap();
3643
3644 RadosTestECPP::TearDown();
3645
3646 cleanup_default_namespace(cache_ioctx);
3647 cleanup_namespace(cache_ioctx, nspace);
3648
3649 cache_ioctx.close();
3650 ASSERT_EQ(0, s_cluster.pool_delete(cache_pool_name.c_str()));
3651 }
3652
3653 librados::IoCtx cache_ioctx;
3654};
3655
3656std::string LibRadosTwoPoolsECPP::cache_pool_name;
3657
3658TEST_F(LibRadosTierECPP, Dirty) {
3659 {
3660 ObjectWriteOperation op;
3661 op.undirty();
3662 ASSERT_EQ(0, ioctx.operate("foo", &op)); // still get 0 if it dne
3663 }
3664 {
3665 ObjectWriteOperation op;
3666 op.create(true);
3667 ASSERT_EQ(0, ioctx.operate("foo", &op));
3668 }
3669 {
3670 bool dirty = false;
3671 int r = -1;
3672 ObjectReadOperation op;
3673 op.is_dirty(&dirty, &r);
3674 ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
3675 ASSERT_TRUE(dirty);
3676 ASSERT_EQ(0, r);
3677 }
3678 {
3679 ObjectWriteOperation op;
3680 op.undirty();
3681 ASSERT_EQ(0, ioctx.operate("foo", &op));
3682 }
3683 {
3684 ObjectWriteOperation op;
3685 op.undirty();
3686 ASSERT_EQ(0, ioctx.operate("foo", &op)); // still 0 if already clean
3687 }
3688 {
3689 bool dirty = false;
3690 int r = -1;
3691 ObjectReadOperation op;
3692 op.is_dirty(&dirty, &r);
3693 ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
3694 ASSERT_FALSE(dirty);
3695 ASSERT_EQ(0, r);
3696 }
3697 //{
3698 // ObjectWriteOperation op;
3699 // op.truncate(0); // still a write even tho it is a no-op
3700 // ASSERT_EQ(0, ioctx.operate("foo", &op));
3701 //}
3702 //{
3703 // bool dirty = false;
3704 // int r = -1;
3705 // ObjectReadOperation op;
3706 // op.is_dirty(&dirty, &r);
3707 // ASSERT_EQ(0, ioctx.operate("foo", &op, NULL));
3708 // ASSERT_TRUE(dirty);
3709 // ASSERT_EQ(0, r);
3710 //}
3711}
3712
3713TEST_F(LibRadosTwoPoolsECPP, Overlay) {
3714 // create objects
3715 {
3716 bufferlist bl;
3717 bl.append("base");
3718 ObjectWriteOperation op;
3719 op.write_full(bl);
3720 ASSERT_EQ(0, ioctx.operate("foo", &op));
3721 }
3722 {
3723 bufferlist bl;
3724 bl.append("cache");
3725 ObjectWriteOperation op;
3726 op.write_full(bl);
3727 ASSERT_EQ(0, cache_ioctx.operate("foo", &op));
3728 }
3729
3730 // configure cache
3731 bufferlist inbl;
3732 ASSERT_EQ(0, cluster.mon_command(
3733 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
3734 "\", \"tierpool\": \"" + cache_pool_name +
3735 "\", \"force_nonempty\": \"--force-nonempty\" }",
3736 inbl, NULL, NULL));
3737 ASSERT_EQ(0, cluster.mon_command(
3738 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
3739 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
3740 inbl, NULL, NULL));
3741
3742 // wait for maps to settle
3743 cluster.wait_for_latest_osdmap();
3744
3745 // by default, the overlay sends us to cache pool
3746 {
3747 bufferlist bl;
3748 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
3749 ASSERT_EQ('c', bl[0]);
3750 }
3751 {
3752 bufferlist bl;
3753 ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
3754 ASSERT_EQ('c', bl[0]);
3755 }
3756
3757 // unless we say otherwise
3758 {
3759 bufferlist bl;
3760 ObjectReadOperation op;
3761 op.read(0, 1, &bl, NULL);
3762 librados::AioCompletion *completion = cluster.aio_create_completion();
3763 ASSERT_EQ(0, ioctx.aio_operate(
3764 "foo", completion, &op,
3765 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 3766 completion->wait_for_complete();
7c673cae
FG		 3767 ASSERT_EQ(0, completion->get_return_value());
3768 completion->release();
3769 ASSERT_EQ('b', bl[0]);
3770 }
3771}
3772
3773TEST_F(LibRadosTwoPoolsECPP, Promote) {
3774 // create object
3775 {
3776 bufferlist bl;
3777 bl.append("hi there");
3778 ObjectWriteOperation op;
3779 op.write_full(bl);
3780 ASSERT_EQ(0, ioctx.operate("foo", &op));
3781 }
3782
3783 // configure cache
3784 bufferlist inbl;
3785 ASSERT_EQ(0, cluster.mon_command(
3786 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
3787 "\", \"tierpool\": \"" + cache_pool_name +
3788 "\", \"force_nonempty\": \"--force-nonempty\" }",
3789 inbl, NULL, NULL));
3790 ASSERT_EQ(0, cluster.mon_command(
3791 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
3792 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
3793 inbl, NULL, NULL));
3794 ASSERT_EQ(0, cluster.mon_command(
3795 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
3796 "\", \"mode\": \"writeback\"}",
3797 inbl, NULL, NULL));
3798
3799 // wait for maps to settle
3800 cluster.wait_for_latest_osdmap();
3801
3802 // read, trigger a promote
3803 {
3804 bufferlist bl;
3805 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
3806 }
3807
3808 // read, trigger a whiteout
3809 {
3810 bufferlist bl;
3811 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
3812 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
3813 }
3814
3815 // verify the object is present in the cache tier
3816 {
3817 NObjectIterator it = cache_ioctx.nobjects_begin();
3818 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
3819 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
3820 ++it;
3821 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
3822 ++it;
3823 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
3824 }
3825}
3826
3827TEST_F(LibRadosTwoPoolsECPP, PromoteSnap) {
3828 // create object
3829 {
3830 bufferlist bl;
3831 bl.append("hi there");
3832 ObjectWriteOperation op;
3833 op.write_full(bl);
3834 ASSERT_EQ(0, ioctx.operate("foo", &op));
3835 }
3836 {
3837 bufferlist bl;
3838 bl.append("hi there");
3839 ObjectWriteOperation op;
3840 op.write_full(bl);
3841 ASSERT_EQ(0, ioctx.operate("bar", &op));
3842 }
3843 {
3844 bufferlist bl;
3845 bl.append("hi there");
3846 ObjectWriteOperation op;
3847 op.write_full(bl);
3848 ASSERT_EQ(0, ioctx.operate("baz", &op));
3849 }
3850 {
3851 bufferlist bl;
3852 bl.append("hi there");
3853 ObjectWriteOperation op;
3854 op.write_full(bl);
3855 ASSERT_EQ(0, ioctx.operate("bam", &op));
3856 }
3857
3858 // create a snapshot, clone
3859 vector<uint64_t> my_snaps(1);
3860 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
3861 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
3862 my_snaps));
3863 {
3864 bufferlist bl;
3865 bl.append("ciao!");
3866 ObjectWriteOperation op;
3867 op.write_full(bl);
3868 ASSERT_EQ(0, ioctx.operate("foo", &op));
3869 }
3870 {
3871 bufferlist bl;
3872 bl.append("ciao!");
3873 ObjectWriteOperation op;
3874 op.write_full(bl);
3875 ASSERT_EQ(0, ioctx.operate("bar", &op));
3876 }
3877 {
3878 ObjectWriteOperation op;
3879 op.remove();
3880 ASSERT_EQ(0, ioctx.operate("baz", &op));
3881 }
3882 {
3883 bufferlist bl;
3884 bl.append("ciao!");
3885 ObjectWriteOperation op;
3886 op.write_full(bl);
3887 ASSERT_EQ(0, ioctx.operate("bam", &op));
3888 }
3889
3890 // configure cache
3891 bufferlist inbl;
3892 ASSERT_EQ(0, cluster.mon_command(
3893 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
3894 "\", \"tierpool\": \"" + cache_pool_name +
3895 "\", \"force_nonempty\": \"--force-nonempty\" }",
3896 inbl, NULL, NULL));
3897 ASSERT_EQ(0, cluster.mon_command(
3898 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
3899 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
3900 inbl, NULL, NULL));
3901 ASSERT_EQ(0, cluster.mon_command(
3902 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
3903 "\", \"mode\": \"writeback\"}",
3904 inbl, NULL, NULL));
3905
3906 // wait for maps to settle
3907 cluster.wait_for_latest_osdmap();
3908
3909 // read, trigger a promote on the head
3910 {
3911 bufferlist bl;
3912 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
3913 ASSERT_EQ('c', bl[0]);
3914 }
3915 {
3916 bufferlist bl;
3917 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
3918 ASSERT_EQ('c', bl[0]);
3919 }
3920
3921 ioctx.snap_set_read(my_snaps[0]);
3922
3923 // stop and scrub this pg (to make sure scrub can handle missing
3924 // clones in the cache tier)
3925 // This test requires cache tier and base tier to have the same pg_num/pgp_num
3926 {
3927 for (int tries = 0; tries < 5; ++tries) {
3928 IoCtx cache_ioctx;
3929 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
3930 uint32_t hash;
3931 ASSERT_EQ(0, ioctx.get_object_pg_hash_position2("foo", &hash));
3932 ostringstream ss;
3933 ss << "{\"prefix\": \"pg scrub\", \"pgid\": \""
3934 << cache_ioctx.get_id() << "."
3935 << hash
3936 << "\"}";
3937 int r = cluster.mon_command(ss.str(), inbl, NULL, NULL);
3938 if (r == -EAGAIN ||
3939 r == -ENOENT) { // in case mgr osdmap is a bit stale
3940 sleep(5);
3941 continue;
3942 }
3943 ASSERT_EQ(0, r);
3944 break;
3945 }
3946 // give it a few seconds to go. this is sloppy but is usually enough time
3947 cout << "waiting for scrub..." << std::endl;
3948 sleep(15);
3949 cout << "done waiting" << std::endl;
3950 }
3951
3952 // read foo snap
3953 {
3954 bufferlist bl;
3955 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
3956 ASSERT_EQ('h', bl[0]);
3957 }
3958
3959 // read bar snap
3960 {
3961 bufferlist bl;
3962 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
3963 ASSERT_EQ('h', bl[0]);
3964 }
3965
3966 // read baz snap
3967 {
3968 bufferlist bl;
3969 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
3970 ASSERT_EQ('h', bl[0]);
3971 }
3972
3973 ioctx.snap_set_read(librados::SNAP_HEAD);
3974
3975 // read foo
3976 {
3977 bufferlist bl;
3978 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
3979 ASSERT_EQ('c', bl[0]);
3980 }
3981
3982 // read bar
3983 {
3984 bufferlist bl;
3985 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
3986 ASSERT_EQ('c', bl[0]);
3987 }
3988
3989 // read baz
3990 {
3991 bufferlist bl;
3992 ASSERT_EQ(-ENOENT, ioctx.read("baz", bl, 1, 0));
3993 }
3994
3995 // cleanup
3996 ioctx.selfmanaged_snap_remove(my_snaps[0]);
3997}
3998
3999TEST_F(LibRadosTwoPoolsECPP, PromoteSnapTrimRace) {
4000 // create object
4001 {
4002 bufferlist bl;
4003 bl.append("hi there");
4004 ObjectWriteOperation op;
4005 op.write_full(bl);
4006 ASSERT_EQ(0, ioctx.operate("foo", &op));
4007 }
4008
4009 // create a snapshot, clone
4010 vector<uint64_t> my_snaps(1);
4011 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
4012 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
4013 my_snaps));
4014 {
4015 bufferlist bl;
4016 bl.append("ciao!");
4017 ObjectWriteOperation op;
4018 op.write_full(bl);
4019 ASSERT_EQ(0, ioctx.operate("foo", &op));
4020 }
4021
4022 // configure cache
4023 bufferlist inbl;
4024 ASSERT_EQ(0, cluster.mon_command(
4025 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
4026 "\", \"tierpool\": \"" + cache_pool_name +
4027 "\", \"force_nonempty\": \"--force-nonempty\" }",
4028 inbl, NULL, NULL));
4029 ASSERT_EQ(0, cluster.mon_command(
4030 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
4031 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
4032 inbl, NULL, NULL));
4033 ASSERT_EQ(0, cluster.mon_command(
4034 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
4035 "\", \"mode\": \"writeback\"}",
4036 inbl, NULL, NULL));
4037
4038 // wait for maps to settle
4039 cluster.wait_for_latest_osdmap();
4040
4041 // delete the snap
4042 ASSERT_EQ(0, ioctx.selfmanaged_snap_remove(my_snaps[0]));
4043
4044 ioctx.snap_set_read(my_snaps[0]);
4045
9f95a23c
TL		 4046 // read foo snap. the OSD may or may not realize that this snap has
4047 // been logically deleted; either response is valid.
7c673cae
FG		 4048 {
4049 bufferlist bl;
9f95a23c
TL		 4050 int r = ioctx.read("foo", bl, 1, 0);
4051 ASSERT_TRUE(r == 1 || r == -ENOENT);
7c673cae
FG		 4052 }
4053
4054 // cleanup
4055 ioctx.selfmanaged_snap_remove(my_snaps[0]);
4056}
4057
4058TEST_F(LibRadosTwoPoolsECPP, Whiteout) {
4059 // create object
4060 {
4061 bufferlist bl;
4062 bl.append("hi there");
4063 ObjectWriteOperation op;
4064 op.write_full(bl);
4065 ASSERT_EQ(0, ioctx.operate("foo", &op));
4066 }
4067
4068 // configure cache
4069 bufferlist inbl;
4070 ASSERT_EQ(0, cluster.mon_command(
4071 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
4072 "\", \"tierpool\": \"" + cache_pool_name +
4073 "\", \"force_nonempty\": \"--force-nonempty\" }",
4074 inbl, NULL, NULL));
4075 ASSERT_EQ(0, cluster.mon_command(
4076 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
4077 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
4078 inbl, NULL, NULL));
4079 ASSERT_EQ(0, cluster.mon_command(
4080 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
4081 "\", \"mode\": \"writeback\"}",
4082 inbl, NULL, NULL));
4083
4084 // wait for maps to settle
4085 cluster.wait_for_latest_osdmap();
4086
4087 // create some whiteouts, verify they behave
4088 {
4089 ObjectWriteOperation op;
4090 op.assert_exists();
4091 op.remove();
4092 ASSERT_EQ(0, ioctx.operate("foo", &op));
4093 }
4094
4095 {
4096 ObjectWriteOperation op;
4097 op.assert_exists();
4098 op.remove();
4099 ASSERT_EQ(-ENOENT, ioctx.operate("bar", &op));
4100 }
4101 {
4102 ObjectWriteOperation op;
4103 op.assert_exists();
4104 op.remove();
4105 ASSERT_EQ(-ENOENT, ioctx.operate("bar", &op));
4106 }
4107
4108 // verify the whiteouts are there in the cache tier
4109 {
4110 NObjectIterator it = cache_ioctx.nobjects_begin();
4111 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
4112 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
4113 ++it;
4114 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
4115 ++it;
4116 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
4117 }
4118
4119 // delete a whiteout and verify it goes away
4120 ASSERT_EQ(-ENOENT, ioctx.remove("foo"));
4121 {
4122 ObjectWriteOperation op;
4123 op.remove();
4124 librados::AioCompletion *completion = cluster.aio_create_completion();
4125 ASSERT_EQ(0, ioctx.aio_operate("bar", completion, &op,
4126 librados::OPERATION_IGNORE_CACHE));
9f95a23c 4127 completion->wait_for_complete();
7c673cae
FG		 4128 ASSERT_EQ(0, completion->get_return_value());
4129 completion->release();
4130
4131 NObjectIterator it = cache_ioctx.nobjects_begin();
4132 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
4133 ASSERT_TRUE(it->get_oid() == string("foo"));
4134 ++it;
4135 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
4136 }
4137
4138 // recreate an object and verify we can read it
4139 {
4140 bufferlist bl;
4141 bl.append("hi there");
4142 ObjectWriteOperation op;
4143 op.write_full(bl);
4144 ASSERT_EQ(0, ioctx.operate("foo", &op));
4145 }
4146 {
4147 bufferlist bl;
4148 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
4149 ASSERT_EQ('h', bl[0]);
4150 }
4151}
4152
4153TEST_F(LibRadosTwoPoolsECPP, Evict) {
4154 // create object
4155 {
4156 bufferlist bl;
4157 bl.append("hi there");
4158 ObjectWriteOperation op;
4159 op.write_full(bl);
4160 ASSERT_EQ(0, ioctx.operate("foo", &op));
4161 }
4162
4163 // configure cache
4164 bufferlist inbl;
4165 ASSERT_EQ(0, cluster.mon_command(
4166 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
4167 "\", \"tierpool\": \"" + cache_pool_name +
4168 "\", \"force_nonempty\": \"--force-nonempty\" }",
4169 inbl, NULL, NULL));
4170 ASSERT_EQ(0, cluster.mon_command(
4171 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
4172 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
4173 inbl, NULL, NULL));
4174 ASSERT_EQ(0, cluster.mon_command(
4175 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
4176 "\", \"mode\": \"writeback\"}",
4177 inbl, NULL, NULL));
4178
4179 // wait for maps to settle
4180 cluster.wait_for_latest_osdmap();
4181
4182 // read, trigger a promote
4183 {
4184 bufferlist bl;
4185 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
4186 }
4187
4188 // read, trigger a whiteout, and a dirty object
4189 {
4190 bufferlist bl;
4191 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
4192 ASSERT_EQ(-ENOENT, ioctx.read("bar", bl, 1, 0));
4193 ASSERT_EQ(0, ioctx.write("bar", bl, bl.length(), 0));
4194 }
4195
4196 // verify the object is present in the cache tier
4197 {
4198 NObjectIterator it = cache_ioctx.nobjects_begin();
4199 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
4200 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
4201 ++it;
4202 ASSERT_TRUE(it->get_oid() == string("foo") || it->get_oid() == string("bar"));
4203 ++it;
4204 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
4205 }
4206
4207 // pin
4208 {
4209 ObjectWriteOperation op;
4210 op.cache_pin();
4211 librados::AioCompletion *completion = cluster.aio_create_completion();
4212 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 4213 completion->wait_for_complete();
7c673cae
FG		 4214 ASSERT_EQ(0, completion->get_return_value());
4215 completion->release();
4216 }
4217
4218 // evict the pinned object with -EPERM
4219 {
4220 ObjectReadOperation op;
4221 op.cache_evict();
4222 librados::AioCompletion *completion = cluster.aio_create_completion();
4223 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
4224 librados::OPERATION_IGNORE_CACHE,
4225 NULL));
9f95a23c 4226 completion->wait_for_complete();
7c673cae
FG		 4227 ASSERT_EQ(-EPERM, completion->get_return_value());
4228 completion->release();
4229 }
4230
4231 // unpin
4232 {
4233 ObjectWriteOperation op;
4234 op.cache_unpin();
4235 librados::AioCompletion *completion = cluster.aio_create_completion();
4236 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 4237 completion->wait_for_complete();
7c673cae
FG		 4238 ASSERT_EQ(0, completion->get_return_value());
4239 completion->release();
4240 }
4241
4242 // flush
4243 {
4244 ObjectReadOperation op;
4245 op.cache_flush();
4246 librados::AioCompletion *completion = cluster.aio_create_completion();
4247 ASSERT_EQ(0, cache_ioctx.aio_operate(
4248 "foo", completion, &op,
4249 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 4250 completion->wait_for_complete();
7c673cae
FG		 4251 ASSERT_EQ(0, completion->get_return_value());
4252 completion->release();
4253 }
4254
4255 // verify clean
4256 {
4257 bool dirty = false;
4258 int r = -1;
4259 ObjectReadOperation op;
4260 op.is_dirty(&dirty, &r);
4261 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
4262 ASSERT_FALSE(dirty);
4263 ASSERT_EQ(0, r);
4264 }
4265
4266 // evict
4267 {
4268 ObjectReadOperation op;
4269 op.cache_evict();
4270 librados::AioCompletion *completion = cluster.aio_create_completion();
4271 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
4272 librados::OPERATION_IGNORE_CACHE,
4273 NULL));
9f95a23c 4274 completion->wait_for_complete();
7c673cae
FG		 4275 ASSERT_EQ(0, completion->get_return_value());
4276 completion->release();
4277 }
4278 {
4279 ObjectReadOperation op;
4280 op.cache_evict();
4281 librados::AioCompletion *completion = cluster.aio_create_completion();
4282 ASSERT_EQ(0, cache_ioctx.aio_operate(
4283 "foo", completion, &op,
4284 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 4285 completion->wait_for_complete();
7c673cae
FG		 4286 ASSERT_EQ(0, completion->get_return_value());
4287 completion->release();
4288 }
4289 {
4290 ObjectReadOperation op;
4291 op.cache_evict();
4292 librados::AioCompletion *completion = cluster.aio_create_completion();
4293 ASSERT_EQ(0, cache_ioctx.aio_operate(
4294 "bar", completion, &op,
4295 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 4296 completion->wait_for_complete();
7c673cae
FG		 4297 ASSERT_EQ(-EBUSY, completion->get_return_value());
4298 completion->release();
4299 }
4300}
4301
4302TEST_F(LibRadosTwoPoolsECPP, EvictSnap) {
4303 // create object
4304 {
4305 bufferlist bl;
4306 bl.append("hi there");
4307 ObjectWriteOperation op;
4308 op.write_full(bl);
4309 ASSERT_EQ(0, ioctx.operate("foo", &op));
4310 }
4311 {
4312 bufferlist bl;
4313 bl.append("hi there");
4314 ObjectWriteOperation op;
4315 op.write_full(bl);
4316 ASSERT_EQ(0, ioctx.operate("bar", &op));
4317 }
4318 {
4319 bufferlist bl;
4320 bl.append("hi there");
4321 ObjectWriteOperation op;
4322 op.write_full(bl);
4323 ASSERT_EQ(0, ioctx.operate("baz", &op));
4324 }
4325 {
4326 bufferlist bl;
4327 bl.append("hi there");
4328 ObjectWriteOperation op;
4329 op.write_full(bl);
4330 ASSERT_EQ(0, ioctx.operate("bam", &op));
4331 }
4332
4333 // create a snapshot, clone
4334 vector<uint64_t> my_snaps(1);
4335 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
4336 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
4337 my_snaps));
4338 {
4339 bufferlist bl;
4340 bl.append("ciao!");
4341 ObjectWriteOperation op;
4342 op.write_full(bl);
4343 ASSERT_EQ(0, ioctx.operate("foo", &op));
4344 }
4345 {
4346 bufferlist bl;
4347 bl.append("ciao!");
4348 ObjectWriteOperation op;
4349 op.write_full(bl);
4350 ASSERT_EQ(0, ioctx.operate("bar", &op));
4351 }
4352 {
4353 ObjectWriteOperation op;
4354 op.remove();
4355 ASSERT_EQ(0, ioctx.operate("baz", &op));
4356 }
4357 {
4358 bufferlist bl;
4359 bl.append("ciao!");
4360 ObjectWriteOperation op;
4361 op.write_full(bl);
4362 ASSERT_EQ(0, ioctx.operate("bam", &op));
4363 }
4364
4365 // configure cache
4366 bufferlist inbl;
4367 ASSERT_EQ(0, cluster.mon_command(
4368 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
4369 "\", \"tierpool\": \"" + cache_pool_name +
4370 "\", \"force_nonempty\": \"--force-nonempty\" }",
4371 inbl, NULL, NULL));
4372 ASSERT_EQ(0, cluster.mon_command(
4373 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
4374 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
4375 inbl, NULL, NULL));
4376 ASSERT_EQ(0, cluster.mon_command(
4377 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
4378 "\", \"mode\": \"writeback\"}",
4379 inbl, NULL, NULL));
4380
4381 // wait for maps to settle
4382 cluster.wait_for_latest_osdmap();
4383
4384 // read, trigger a promote on the head
4385 {
4386 bufferlist bl;
4387 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
4388 ASSERT_EQ('c', bl[0]);
4389 }
4390 {
4391 bufferlist bl;
4392 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
4393 ASSERT_EQ('c', bl[0]);
4394 }
4395
4396 // evict bam
4397 {
4398 ObjectReadOperation op;
4399 op.cache_evict();
4400 librados::AioCompletion *completion = cluster.aio_create_completion();
4401 ASSERT_EQ(0, cache_ioctx.aio_operate(
4402 "bam", completion, &op,
4403 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 4404 completion->wait_for_complete();
7c673cae
FG		 4405 ASSERT_EQ(0, completion->get_return_value());
4406 completion->release();
4407 }
4408 {
4409 bufferlist bl;
4410 ObjectReadOperation op;
4411 op.read(1, 0, &bl, NULL);
4412 librados::AioCompletion *completion = cluster.aio_create_completion();
4413 ASSERT_EQ(0, cache_ioctx.aio_operate(
4414 "bam", completion, &op,
4415 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 4416 completion->wait_for_complete();
7c673cae
FG		 4417 ASSERT_EQ(-ENOENT, completion->get_return_value());
4418 completion->release();
4419 }
4420
4421 // read foo snap
4422 ioctx.snap_set_read(my_snaps[0]);
4423 {
4424 bufferlist bl;
4425 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
4426 ASSERT_EQ('h', bl[0]);
4427 }
4428
4429 // evict foo snap
4430 {
4431 ObjectReadOperation op;
4432 op.cache_evict();
4433 librados::AioCompletion *completion = cluster.aio_create_completion();
4434 ASSERT_EQ(0, ioctx.aio_operate(
4435 "foo", completion, &op,
4436 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 4437 completion->wait_for_complete();
7c673cae
FG		 4438 ASSERT_EQ(0, completion->get_return_value());
4439 completion->release();
4440 }
4441 // snap is gone...
4442 {
4443 bufferlist bl;
4444 ObjectReadOperation op;
4445 op.read(1, 0, &bl, NULL);
4446 librados::AioCompletion *completion = cluster.aio_create_completion();
4447 ASSERT_EQ(0, ioctx.aio_operate(
4448 "foo", completion, &op,
4449 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 4450 completion->wait_for_complete();
7c673cae
FG		 4451 ASSERT_EQ(-ENOENT, completion->get_return_value());
4452 completion->release();
4453 }
4454 // head is still there...
4455 ioctx.snap_set_read(librados::SNAP_HEAD);
4456 {
4457 bufferlist bl;
4458 ObjectReadOperation op;
4459 op.read(1, 0, &bl, NULL);
4460 librados::AioCompletion *completion = cluster.aio_create_completion();
4461 ASSERT_EQ(0, ioctx.aio_operate(
4462 "foo", completion, &op,
4463 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 4464 completion->wait_for_complete();
7c673cae
FG		 4465 ASSERT_EQ(0, completion->get_return_value());
4466 completion->release();
4467 }
4468
4469 // promote head + snap of bar
4470 ioctx.snap_set_read(librados::SNAP_HEAD);
4471 {
4472 bufferlist bl;
4473 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
4474 ASSERT_EQ('c', bl[0]);
4475 }
4476 ioctx.snap_set_read(my_snaps[0]);
4477 {
4478 bufferlist bl;
4479 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
4480 ASSERT_EQ('h', bl[0]);
4481 }
4482
4483 // evict bar head (fail)
4484 ioctx.snap_set_read(librados::SNAP_HEAD);
4485 {
4486 ObjectReadOperation op;
4487 op.cache_evict();
4488 librados::AioCompletion *completion = cluster.aio_create_completion();
4489 ASSERT_EQ(0, ioctx.aio_operate(
4490 "bar", completion, &op,
4491 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 4492 completion->wait_for_complete();
7c673cae
FG		 4493 ASSERT_EQ(-EBUSY, completion->get_return_value());
4494 completion->release();
4495 }
4496
4497 // evict bar snap
4498 ioctx.snap_set_read(my_snaps[0]);
4499 {
4500 ObjectReadOperation op;
4501 op.cache_evict();
4502 librados::AioCompletion *completion = cluster.aio_create_completion();
4503 ASSERT_EQ(0, ioctx.aio_operate(
4504 "bar", completion, &op,
4505 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 4506 completion->wait_for_complete();
7c673cae
FG		 4507 ASSERT_EQ(0, completion->get_return_value());
4508 completion->release();
4509 }
4510 // ...and then head
4511 ioctx.snap_set_read(librados::SNAP_HEAD);
4512 {
4513 bufferlist bl;
4514 ObjectReadOperation op;
4515 op.read(1, 0, &bl, NULL);
4516 librados::AioCompletion *completion = cluster.aio_create_completion();
4517 ASSERT_EQ(0, ioctx.aio_operate(
4518 "bar", completion, &op,
4519 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 4520 completion->wait_for_complete();
7c673cae
FG		 4521 ASSERT_EQ(0, completion->get_return_value());
4522 completion->release();
4523 }
4524 {
4525 ObjectReadOperation op;
4526 op.cache_evict();
4527 librados::AioCompletion *completion = cluster.aio_create_completion();
4528 ASSERT_EQ(0, ioctx.aio_operate(
4529 "bar", completion, &op,
4530 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 4531 completion->wait_for_complete();
7c673cae
FG		 4532 ASSERT_EQ(0, completion->get_return_value());
4533 completion->release();
4534 }
4535
4536 // cleanup
4537 ioctx.selfmanaged_snap_remove(my_snaps[0]);
4538}
4539
4540TEST_F(LibRadosTwoPoolsECPP, TryFlush) {
4541 // configure cache
4542 bufferlist inbl;
4543 ASSERT_EQ(0, cluster.mon_command(
4544 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
4545 "\", \"tierpool\": \"" + cache_pool_name +
4546 "\", \"force_nonempty\": \"--force-nonempty\" }",
4547 inbl, NULL, NULL));
4548 ASSERT_EQ(0, cluster.mon_command(
4549 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
4550 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
4551 inbl, NULL, NULL));
4552 ASSERT_EQ(0, cluster.mon_command(
4553 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
4554 "\", \"mode\": \"writeback\"}",
4555 inbl, NULL, NULL));
4556
4557 // wait for maps to settle
4558 cluster.wait_for_latest_osdmap();
4559
4560 // create object
4561 {
4562 bufferlist bl;
4563 bl.append("hi there");
4564 ObjectWriteOperation op;
4565 op.write_full(bl);
4566 ASSERT_EQ(0, ioctx.operate("foo", &op));
4567 }
4568
4569 // verify the object is present in the cache tier
4570 {
4571 NObjectIterator it = cache_ioctx.nobjects_begin();
4572 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
4573 ASSERT_TRUE(it->get_oid() == string("foo"));
4574 ++it;
4575 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
4576 }
4577
4578 // verify the object is NOT present in the base tier
4579 {
4580 NObjectIterator it = ioctx.nobjects_begin();
4581 ASSERT_TRUE(it == ioctx.nobjects_end());
4582 }
4583
4584 // verify dirty
4585 {
4586 bool dirty = false;
4587 int r = -1;
4588 ObjectReadOperation op;
4589 op.is_dirty(&dirty, &r);
4590 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
4591 ASSERT_TRUE(dirty);
4592 ASSERT_EQ(0, r);
4593 }
4594
4595 // pin
4596 {
4597 ObjectWriteOperation op;
4598 op.cache_pin();
4599 librados::AioCompletion *completion = cluster.aio_create_completion();
4600 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 4601 completion->wait_for_complete();
7c673cae
FG		 4602 ASSERT_EQ(0, completion->get_return_value());
4603 completion->release();
4604 }
4605
4606 // flush the pinned object with -EPERM
4607 {
4608 ObjectReadOperation op;
4609 op.cache_try_flush();
4610 librados::AioCompletion *completion = cluster.aio_create_completion();
4611 ASSERT_EQ(0, cache_ioctx.aio_operate(
4612 "foo", completion, &op,
4613 librados::OPERATION_IGNORE_OVERLAY |
4614 librados::OPERATION_SKIPRWLOCKS, NULL));
9f95a23c 4615 completion->wait_for_complete();
7c673cae
FG		 4616 ASSERT_EQ(-EPERM, completion->get_return_value());
4617 completion->release();
4618 }
4619
4620 // unpin
4621 {
4622 ObjectWriteOperation op;
4623 op.cache_unpin();
4624 librados::AioCompletion *completion = cluster.aio_create_completion();
4625 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 4626 completion->wait_for_complete();
7c673cae
FG		 4627 ASSERT_EQ(0, completion->get_return_value());
4628 completion->release();
4629 }
4630
4631 // flush
4632 {
4633 ObjectReadOperation op;
4634 op.cache_try_flush();
4635 librados::AioCompletion *completion = cluster.aio_create_completion();
4636 ASSERT_EQ(0, cache_ioctx.aio_operate(
4637 "foo", completion, &op,
4638 librados::OPERATION_IGNORE_OVERLAY |
4639 librados::OPERATION_SKIPRWLOCKS, NULL));
9f95a23c 4640 completion->wait_for_complete();
7c673cae
FG		 4641 ASSERT_EQ(0, completion->get_return_value());
4642 completion->release();
4643 }
4644
4645 // verify clean
4646 {
4647 bool dirty = false;
4648 int r = -1;
4649 ObjectReadOperation op;
4650 op.is_dirty(&dirty, &r);
4651 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
4652 ASSERT_FALSE(dirty);
4653 ASSERT_EQ(0, r);
4654 }
4655
4656 // verify in base tier
4657 {
4658 NObjectIterator it = ioctx.nobjects_begin();
4659 ASSERT_TRUE(it != ioctx.nobjects_end());
4660 ASSERT_TRUE(it->get_oid() == string("foo"));
4661 ++it;
4662 ASSERT_TRUE(it == ioctx.nobjects_end());
4663 }
4664
4665 // evict it
4666 {
4667 ObjectReadOperation op;
4668 op.cache_evict();
4669 librados::AioCompletion *completion = cluster.aio_create_completion();
4670 ASSERT_EQ(0, cache_ioctx.aio_operate(
4671 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 4672 completion->wait_for_complete();
7c673cae
FG		 4673 ASSERT_EQ(0, completion->get_return_value());
4674 completion->release();
4675 }
4676
4677 // verify no longer in cache tier
4678 {
4679 NObjectIterator it = cache_ioctx.nobjects_begin();
4680 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
4681 }
4682}
4683
4684TEST_F(LibRadosTwoPoolsECPP, FailedFlush) {
4685 // configure cache
4686 bufferlist inbl;
4687 ASSERT_EQ(0, cluster.mon_command(
4688 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
4689 "\", \"tierpool\": \"" + cache_pool_name +
4690 "\", \"force_nonempty\": \"--force-nonempty\" }",
4691 inbl, NULL, NULL));
4692 ASSERT_EQ(0, cluster.mon_command(
4693 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
4694 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
4695 inbl, NULL, NULL));
4696 ASSERT_EQ(0, cluster.mon_command(
4697 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
4698 "\", \"mode\": \"writeback\"}",
4699 inbl, NULL, NULL));
4700
4701 // wait for maps to settle
4702 cluster.wait_for_latest_osdmap();
4703
4704 // create object
4705 {
4706 bufferlist bl;
4707 bl.append("hi there");
4708 ObjectWriteOperation op;
4709 op.write_full(bl);
4710 ASSERT_EQ(0, ioctx.operate("foo", &op));
4711 }
4712
4713 // verify the object is present in the cache tier
4714 {
4715 NObjectIterator it = cache_ioctx.nobjects_begin();
4716 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
4717 ASSERT_TRUE(it->get_oid() == string("foo"));
4718 ++it;
4719 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
4720 }
4721
4722 // verify the object is NOT present in the base tier
4723 {
4724 NObjectIterator it = ioctx.nobjects_begin();
4725 ASSERT_TRUE(it == ioctx.nobjects_end());
4726 }
4727
4728 // set omap
4729 {
4730 ObjectWriteOperation op;
4731 std::map<std::string, bufferlist> omap;
4732 omap["somekey"] = bufferlist();
4733 op.omap_set(omap);
4734 librados::AioCompletion *completion = cluster.aio_create_completion();
4735 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 4736 completion->wait_for_complete();
7c673cae
FG		 4737 ASSERT_EQ(0, completion->get_return_value());
4738 completion->release();
4739 }
4740
4741 // flush
4742 {
4743 ObjectReadOperation op;
4744 op.cache_flush();
4745 librados::AioCompletion *completion = cluster.aio_create_completion();
4746 ASSERT_EQ(0, cache_ioctx.aio_operate(
4747 "foo", completion, &op,
4748 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 4749 completion->wait_for_complete();
7c673cae
FG		 4750 ASSERT_NE(0, completion->get_return_value());
4751 completion->release();
4752 }
4753
4754 // get omap
4755 {
4756 ObjectReadOperation op;
4757 bufferlist bl;
4758 int prval = 0;
4759 std::set<std::string> keys;
4760 keys.insert("somekey");
4761 std::map<std::string, bufferlist> map;
4762
4763 op.omap_get_vals_by_keys(keys, &map, &prval);
4764 librados::AioCompletion *completion = cluster.aio_create_completion();
4765 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op, &bl));
4766 sleep(5);
4767 bool completed = completion->is_complete();
4768 if( !completed ) {
4769 cache_ioctx.aio_cancel(completion);
4770 std::cerr << "Most probably test case will hang here, please reset manually" << std::endl;
4771 ASSERT_TRUE(completed); //in fact we are locked forever at test case shutdown unless fix for http://tracker.ceph.com/issues/14511 is applied. Seems there is no workaround for that
4772 }
4773 completion->release();
4774 }
4775 // verify still not in base tier
4776 {
4777 ASSERT_TRUE(ioctx.nobjects_begin() == ioctx.nobjects_end());
4778 }
4779 // erase it
4780 {
4781 ObjectWriteOperation op;
4782 op.remove();
4783 ASSERT_EQ(0, ioctx.operate("foo", &op));
4784 }
4785 // flush whiteout
4786 {
4787 ObjectReadOperation op;
4788 op.cache_flush();
4789 librados::AioCompletion *completion = cluster.aio_create_completion();
4790 ASSERT_EQ(0, cache_ioctx.aio_operate(
4791 "foo", completion, &op,
4792 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 4793 completion->wait_for_complete();
7c673cae
FG		 4794 ASSERT_EQ(0, completion->get_return_value());
4795 completion->release();
4796 }
4797 // evict
4798 {
4799 ObjectReadOperation op;
4800 op.cache_evict();
4801 librados::AioCompletion *completion = cluster.aio_create_completion();
4802 ASSERT_EQ(0, cache_ioctx.aio_operate(
4803 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 4804 completion->wait_for_complete();
7c673cae
FG		 4805 ASSERT_EQ(0, completion->get_return_value());
4806 completion->release();
4807 }
4808
4809 // verify no longer in cache tier
4810 {
4811 NObjectIterator it = cache_ioctx.nobjects_begin();
4812 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
4813 }
4814 // or base tier
4815 {
4816 NObjectIterator it = ioctx.nobjects_begin();
4817 ASSERT_TRUE(it == ioctx.nobjects_end());
4818 }
4819}
4820
4821TEST_F(LibRadosTwoPoolsECPP, Flush) {
4822 // configure cache
4823 bufferlist inbl;
4824 ASSERT_EQ(0, cluster.mon_command(
4825 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
4826 "\", \"tierpool\": \"" + cache_pool_name +
4827 "\", \"force_nonempty\": \"--force-nonempty\" }",
4828 inbl, NULL, NULL));
4829 ASSERT_EQ(0, cluster.mon_command(
4830 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
4831 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
4832 inbl, NULL, NULL));
4833 ASSERT_EQ(0, cluster.mon_command(
4834 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
4835 "\", \"mode\": \"writeback\"}",
4836 inbl, NULL, NULL));
4837
4838 // wait for maps to settle
4839 cluster.wait_for_latest_osdmap();
4840
4841 uint64_t user_version = 0;
4842
4843 // create object
4844 {
4845 bufferlist bl;
4846 bl.append("hi there");
4847 ObjectWriteOperation op;
4848 op.write_full(bl);
4849 ASSERT_EQ(0, ioctx.operate("foo", &op));
4850 }
4851
4852 // verify the object is present in the cache tier
4853 {
4854 NObjectIterator it = cache_ioctx.nobjects_begin();
4855 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
4856 ASSERT_TRUE(it->get_oid() == string("foo"));
4857 ++it;
4858 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
4859 }
4860
4861 // verify the object is NOT present in the base tier
4862 {
4863 NObjectIterator it = ioctx.nobjects_begin();
4864 ASSERT_TRUE(it == ioctx.nobjects_end());
4865 }
4866
4867 // verify dirty
4868 {
4869 bool dirty = false;
4870 int r = -1;
4871 ObjectReadOperation op;
4872 op.is_dirty(&dirty, &r);
4873 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
4874 ASSERT_TRUE(dirty);
4875 ASSERT_EQ(0, r);
4876 user_version = cache_ioctx.get_last_version();
4877 }
4878
4879 // pin
4880 {
4881 ObjectWriteOperation op;
4882 op.cache_pin();
4883 librados::AioCompletion *completion = cluster.aio_create_completion();
4884 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 4885 completion->wait_for_complete();
7c673cae
FG		 4886 ASSERT_EQ(0, completion->get_return_value());
4887 completion->release();
4888 }
4889
4890 // flush the pinned object with -EPERM
4891 {
4892 ObjectReadOperation op;
4893 op.cache_try_flush();
4894 librados::AioCompletion *completion = cluster.aio_create_completion();
4895 ASSERT_EQ(0, cache_ioctx.aio_operate(
4896 "foo", completion, &op,
4897 librados::OPERATION_IGNORE_OVERLAY |
4898 librados::OPERATION_SKIPRWLOCKS, NULL));
9f95a23c 4899 completion->wait_for_complete();
7c673cae
FG		 4900 ASSERT_EQ(-EPERM, completion->get_return_value());
4901 completion->release();
4902 }
4903
4904 // unpin
4905 {
4906 ObjectWriteOperation op;
4907 op.cache_unpin();
4908 librados::AioCompletion *completion = cluster.aio_create_completion();
4909 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 4910 completion->wait_for_complete();
7c673cae
FG		 4911 ASSERT_EQ(0, completion->get_return_value());
4912 completion->release();
4913 }
4914
4915 // flush
4916 {
4917 ObjectReadOperation op;
4918 op.cache_flush();
4919 librados::AioCompletion *completion = cluster.aio_create_completion();
4920 ASSERT_EQ(0, cache_ioctx.aio_operate(
4921 "foo", completion, &op,
4922 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 4923 completion->wait_for_complete();
7c673cae
FG		 4924 ASSERT_EQ(0, completion->get_return_value());
4925 completion->release();
4926 }
4927
4928 // verify clean
4929 {
4930 bool dirty = false;
4931 int r = -1;
4932 ObjectReadOperation op;
4933 op.is_dirty(&dirty, &r);
4934 ASSERT_EQ(0, cache_ioctx.operate("foo", &op, NULL));
4935 ASSERT_FALSE(dirty);
4936 ASSERT_EQ(0, r);
4937 }
4938
4939 // verify in base tier
4940 {
4941 NObjectIterator it = ioctx.nobjects_begin();
4942 ASSERT_TRUE(it != ioctx.nobjects_end());
4943 ASSERT_TRUE(it->get_oid() == string("foo"));
4944 ++it;
4945 ASSERT_TRUE(it == ioctx.nobjects_end());
4946 }
4947
4948 // evict it
4949 {
4950 ObjectReadOperation op;
4951 op.cache_evict();
4952 librados::AioCompletion *completion = cluster.aio_create_completion();
4953 ASSERT_EQ(0, cache_ioctx.aio_operate(
4954 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 4955 completion->wait_for_complete();
7c673cae
FG		 4956 ASSERT_EQ(0, completion->get_return_value());
4957 completion->release();
4958 }
4959
4960 // verify no longer in cache tier
4961 {
4962 NObjectIterator it = cache_ioctx.nobjects_begin();
4963 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
4964 }
4965
4966 // read it again and verify the version is consistent
4967 {
4968 bufferlist bl;
4969 ASSERT_EQ(1, cache_ioctx.read("foo", bl, 1, 0));
4970 ASSERT_EQ(user_version, cache_ioctx.get_last_version());
4971 }
4972
4973 // erase it
4974 {
4975 ObjectWriteOperation op;
4976 op.remove();
4977 ASSERT_EQ(0, ioctx.operate("foo", &op));
4978 }
4979
4980 // flush whiteout
4981 {
4982 ObjectReadOperation op;
4983 op.cache_flush();
4984 librados::AioCompletion *completion = cluster.aio_create_completion();
4985 ASSERT_EQ(0, cache_ioctx.aio_operate(
4986 "foo", completion, &op,
4987 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 4988 completion->wait_for_complete();
7c673cae
FG		 4989 ASSERT_EQ(0, completion->get_return_value());
4990 completion->release();
4991 }
4992
4993 // evict
4994 {
4995 ObjectReadOperation op;
4996 op.cache_evict();
4997 librados::AioCompletion *completion = cluster.aio_create_completion();
4998 ASSERT_EQ(0, cache_ioctx.aio_operate(
4999 "foo", completion, &op, librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 5000 completion->wait_for_complete();
7c673cae
FG		 5001 ASSERT_EQ(0, completion->get_return_value());
5002 completion->release();
5003 }
5004
5005 // verify no longer in cache tier
5006 {
5007 NObjectIterator it = cache_ioctx.nobjects_begin();
5008 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
5009 }
5010 // or base tier
5011 {
5012 NObjectIterator it = ioctx.nobjects_begin();
5013 ASSERT_TRUE(it == ioctx.nobjects_end());
5014 }
5015}
5016
5017TEST_F(LibRadosTwoPoolsECPP, FlushSnap) {
5018 // configure cache
5019 bufferlist inbl;
5020 ASSERT_EQ(0, cluster.mon_command(
5021 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
5022 "\", \"tierpool\": \"" + cache_pool_name +
5023 "\", \"force_nonempty\": \"--force-nonempty\" }",
5024 inbl, NULL, NULL));
5025 ASSERT_EQ(0, cluster.mon_command(
5026 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
5027 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
5028 inbl, NULL, NULL));
5029 ASSERT_EQ(0, cluster.mon_command(
5030 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
5031 "\", \"mode\": \"writeback\"}",
5032 inbl, NULL, NULL));
5033
5034 // wait for maps to settle
5035 cluster.wait_for_latest_osdmap();
5036
5037 // create object
5038 {
5039 bufferlist bl;
5040 bl.append("a");
5041 ObjectWriteOperation op;
5042 op.write_full(bl);
5043 ASSERT_EQ(0, ioctx.operate("foo", &op));
5044 }
5045
5046 // create a snapshot, clone
5047 vector<uint64_t> my_snaps(1);
5048 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
5049 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
5050 my_snaps));
5051 {
5052 bufferlist bl;
5053 bl.append("b");
5054 ObjectWriteOperation op;
5055 op.write_full(bl);
5056 ASSERT_EQ(0, ioctx.operate("foo", &op));
5057 }
5058
5059 // and another
5060 my_snaps.resize(2);
5061 my_snaps[1] = my_snaps[0];
5062 ASSERT_EQ(0, ioctx.selfmanaged_snap_create(&my_snaps[0]));
5063 ASSERT_EQ(0, ioctx.selfmanaged_snap_set_write_ctx(my_snaps[0],
5064 my_snaps));
5065 {
5066 bufferlist bl;
5067 bl.append("c");
5068 ObjectWriteOperation op;
5069 op.write_full(bl);
5070 ASSERT_EQ(0, ioctx.operate("foo", &op));
5071 }
5072
5073 // verify the object is present in the cache tier
5074 {
5075 NObjectIterator it = cache_ioctx.nobjects_begin();
5076 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
5077 ASSERT_TRUE(it->get_oid() == string("foo"));
5078 ++it;
5079 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
5080 }
5081
5082 // verify the object is NOT present in the base tier
5083 {
5084 NObjectIterator it = ioctx.nobjects_begin();
5085 ASSERT_TRUE(it == ioctx.nobjects_end());
5086 }
5087
5088 // flush on head (should fail)
5089 ioctx.snap_set_read(librados::SNAP_HEAD);
5090 {
5091 ObjectReadOperation op;
5092 op.cache_flush();
5093 librados::AioCompletion *completion = cluster.aio_create_completion();
5094 ASSERT_EQ(0, ioctx.aio_operate(
5095 "foo", completion, &op,
5096 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 5097 completion->wait_for_complete();
7c673cae
FG		 5098 ASSERT_EQ(-EBUSY, completion->get_return_value());
5099 completion->release();
5100 }
5101 // flush on recent snap (should fail)
5102 ioctx.snap_set_read(my_snaps[0]);
5103 {
5104 ObjectReadOperation op;
5105 op.cache_flush();
5106 librados::AioCompletion *completion = cluster.aio_create_completion();
5107 ASSERT_EQ(0, ioctx.aio_operate(
5108 "foo", completion, &op,
5109 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 5110 completion->wait_for_complete();
7c673cae
FG		 5111 ASSERT_EQ(-EBUSY, completion->get_return_value());
5112 completion->release();
5113 }
5114 // flush on oldest snap
5115 ioctx.snap_set_read(my_snaps[1]);
5116 {
5117 ObjectReadOperation op;
5118 op.cache_flush();
5119 librados::AioCompletion *completion = cluster.aio_create_completion();
5120 ASSERT_EQ(0, ioctx.aio_operate(
5121 "foo", completion, &op,
5122 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 5123 completion->wait_for_complete();
7c673cae
FG		 5124 ASSERT_EQ(0, completion->get_return_value());
5125 completion->release();
5126 }
5127 // flush on next oldest snap
5128 ioctx.snap_set_read(my_snaps[0]);
5129 {
5130 ObjectReadOperation op;
5131 op.cache_flush();
5132 librados::AioCompletion *completion = cluster.aio_create_completion();
5133 ASSERT_EQ(0, ioctx.aio_operate(
5134 "foo", completion, &op,
5135 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 5136 completion->wait_for_complete();
7c673cae
FG		 5137 ASSERT_EQ(0, completion->get_return_value());
5138 completion->release();
5139 }
5140 // flush on head
5141 ioctx.snap_set_read(librados::SNAP_HEAD);
5142 {
5143 ObjectReadOperation op;
5144 op.cache_flush();
5145 librados::AioCompletion *completion = cluster.aio_create_completion();
5146 ASSERT_EQ(0, ioctx.aio_operate(
5147 "foo", completion, &op,
5148 librados::OPERATION_IGNORE_CACHE, NULL));
9f95a23c 5149 completion->wait_for_complete();
7c673cae
FG		 5150 ASSERT_EQ(0, completion->get_return_value());
5151 completion->release();
5152 }
5153
5154 // verify i can read the snaps from the cache pool
5155 ioctx.snap_set_read(librados::SNAP_HEAD);
5156 {
5157 bufferlist bl;
5158 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
5159 ASSERT_EQ('c', bl[0]);
5160 }
5161 ioctx.snap_set_read(my_snaps[0]);
5162 {
5163 bufferlist bl;
5164 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
5165 ASSERT_EQ('b', bl[0]);
5166 }
5167 ioctx.snap_set_read(my_snaps[1]);
5168 {
5169 bufferlist bl;
5170 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
5171 ASSERT_EQ('a', bl[0]);
5172 }
5173
5174 // tear down tiers
5175 ASSERT_EQ(0, cluster.mon_command(
5176 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
5177 "\"}",
5178 inbl, NULL, NULL));
5179
5180 // wait for maps to settle
5181 cluster.wait_for_latest_osdmap();
5182
5183 // verify i can read the snaps from the base pool
5184 ioctx.snap_set_read(librados::SNAP_HEAD);
5185 {
5186 bufferlist bl;
5187 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
5188 ASSERT_EQ('c', bl[0]);
5189 }
5190 ioctx.snap_set_read(my_snaps[0]);
5191 {
5192 bufferlist bl;
5193 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
5194 ASSERT_EQ('b', bl[0]);
5195 }
5196 ioctx.snap_set_read(my_snaps[1]);
5197 {
5198 bufferlist bl;
5199 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
5200 ASSERT_EQ('a', bl[0]);
5201 }
5202
5203 ASSERT_EQ(0, cluster.mon_command(
5204 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
5205 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
5206 inbl, NULL, NULL));
5207 cluster.wait_for_latest_osdmap();
5208
5209 // cleanup
5210 ioctx.selfmanaged_snap_remove(my_snaps[0]);
5211}
5212
5213TEST_F(LibRadosTierECPP, FlushWriteRaces) {
5214 Rados cluster;
5215 std::string pool_name = get_temp_pool_name();
5216 std::string cache_pool_name = pool_name + "-cache";
5217 ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
5218 ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
5219 IoCtx cache_ioctx;
5220 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
c07f9fc5 5221 cache_ioctx.application_enable("rados", true);
7c673cae
FG		 5222 IoCtx ioctx;
5223 ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
5224
5225 // configure cache
5226 bufferlist inbl;
5227 ASSERT_EQ(0, cluster.mon_command(
5228 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
5229 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
5230 inbl, NULL, NULL));
5231 ASSERT_EQ(0, cluster.mon_command(
5232 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
5233 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
5234 inbl, NULL, NULL));
5235 ASSERT_EQ(0, cluster.mon_command(
5236 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
5237 "\", \"mode\": \"writeback\"}",
5238 inbl, NULL, NULL));
5239
5240 // wait for maps to settle
5241 cluster.wait_for_latest_osdmap();
5242
5243 // create/dirty object
5244 bufferlist bl;
5245 bl.append("hi there");
5246 {
5247 ObjectWriteOperation op;
5248 op.write_full(bl);
5249 ASSERT_EQ(0, ioctx.operate("foo", &op));
5250 }
5251
5252 // flush + write
5253 {
5254 ObjectReadOperation op;
5255 op.cache_flush();
5256 librados::AioCompletion *completion = cluster.aio_create_completion();
5257 ASSERT_EQ(0, cache_ioctx.aio_operate(
5258 "foo", completion, &op,
5259 librados::OPERATION_IGNORE_OVERLAY, NULL));
5260
5261 ObjectWriteOperation op2;
5262 op2.write_full(bl);
5263 librados::AioCompletion *completion2 = cluster.aio_create_completion();
5264 ASSERT_EQ(0, ioctx.aio_operate(
5265 "foo", completion2, &op2, 0));
5266
9f95a23c
TL		 5267 completion->wait_for_complete();
5268 completion2->wait_for_complete();
7c673cae
FG		 5269 ASSERT_EQ(0, completion->get_return_value());
5270 ASSERT_EQ(0, completion2->get_return_value());
5271 completion->release();
5272 completion2->release();
5273 }
5274
5275 int tries = 1000;
5276 do {
5277 // create/dirty object
5278 {
5279 bufferlist bl;
5280 bl.append("hi there");
5281 ObjectWriteOperation op;
5282 op.write_full(bl);
5283 ASSERT_EQ(0, ioctx.operate("foo", &op));
5284 }
5285
5286 // try-flush + write
5287 {
5288 ObjectReadOperation op;
5289 op.cache_try_flush();
5290 librados::AioCompletion *completion = cluster.aio_create_completion();
5291 ASSERT_EQ(0, cache_ioctx.aio_operate(
5292 "foo", completion, &op,
5293 librados::OPERATION_IGNORE_OVERLAY |
5294 librados::OPERATION_SKIPRWLOCKS, NULL));
5295
5296 ObjectWriteOperation op2;
5297 op2.write_full(bl);
5298 librados::AioCompletion *completion2 = cluster.aio_create_completion();
5299 ASSERT_EQ(0, ioctx.aio_operate("foo", completion2, &op2, 0));
5300
9f95a23c
TL		 5301 completion->wait_for_complete();
5302 completion2->wait_for_complete();
7c673cae
FG		 5303 int r = completion->get_return_value();
5304 ASSERT_TRUE(r == -EBUSY || r == 0);
5305 ASSERT_EQ(0, completion2->get_return_value());
5306 completion->release();
5307 completion2->release();
5308 if (r == -EBUSY)
5309 break;
5310 cout << "didn't get EBUSY, trying again" << std::endl;
5311 }
5312 ASSERT_TRUE(--tries);
5313 } while (true);
5314
5315 // tear down tiers
5316 ASSERT_EQ(0, cluster.mon_command(
5317 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
5318 "\"}",
5319 inbl, NULL, NULL));
5320 ASSERT_EQ(0, cluster.mon_command(
5321 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
5322 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
5323 inbl, NULL, NULL));
5324
5325 // wait for maps to settle before next test
5326 cluster.wait_for_latest_osdmap();
5327
5328 ASSERT_EQ(0, cluster.pool_delete(cache_pool_name.c_str()));
5329 ASSERT_EQ(0, destroy_one_pool_pp(pool_name, cluster));
5330}
5331
5332TEST_F(LibRadosTwoPoolsECPP, FlushTryFlushRaces) {
5333 // configure cache
5334 bufferlist inbl;
5335 ASSERT_EQ(0, cluster.mon_command(
5336 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
5337 "\", \"tierpool\": \"" + cache_pool_name +
5338 "\", \"force_nonempty\": \"--force-nonempty\" }",
5339 inbl, NULL, NULL));
5340 ASSERT_EQ(0, cluster.mon_command(
5341 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
5342 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
5343 inbl, NULL, NULL));
5344 ASSERT_EQ(0, cluster.mon_command(
5345 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
5346 "\", \"mode\": \"writeback\"}",
5347 inbl, NULL, NULL));
5348
5349 // wait for maps to settle
5350 cluster.wait_for_latest_osdmap();
5351
5352 // create/dirty object
5353 {
5354 bufferlist bl;
5355 bl.append("hi there");
5356 ObjectWriteOperation op;
5357 op.write_full(bl);
5358 ASSERT_EQ(0, ioctx.operate("foo", &op));
5359 }
5360
5361 // flush + flush
5362 {
5363 ObjectReadOperation op;
5364 op.cache_flush();
5365 librados::AioCompletion *completion = cluster.aio_create_completion();
5366 ASSERT_EQ(0, cache_ioctx.aio_operate(
5367 "foo", completion, &op,
5368 librados::OPERATION_IGNORE_OVERLAY, NULL));
5369
5370 ObjectReadOperation op2;
5371 op2.cache_flush();
5372 librados::AioCompletion *completion2 = cluster.aio_create_completion();
5373 ASSERT_EQ(0, cache_ioctx.aio_operate(
5374 "foo", completion2, &op2,
5375 librados::OPERATION_IGNORE_OVERLAY, NULL));
5376
9f95a23c
TL		 5377 completion->wait_for_complete();
5378 completion2->wait_for_complete();
7c673cae
FG		 5379 ASSERT_EQ(0, completion->get_return_value());
5380 ASSERT_EQ(0, completion2->get_return_value());
5381 completion->release();
5382 completion2->release();
5383 }
5384
5385 // create/dirty object
5386 {
5387 bufferlist bl;
5388 bl.append("hi there");
5389 ObjectWriteOperation op;
5390 op.write_full(bl);
5391 ASSERT_EQ(0, ioctx.operate("foo", &op));
5392 }
5393
5394 // flush + try-flush
5395 {
5396 ObjectReadOperation op;
5397 op.cache_flush();
5398 librados::AioCompletion *completion = cluster.aio_create_completion();
5399 ASSERT_EQ(0, cache_ioctx.aio_operate(
5400 "foo", completion, &op,
5401 librados::OPERATION_IGNORE_OVERLAY, NULL));
5402
5403 ObjectReadOperation op2;
5404 op2.cache_try_flush();
5405 librados::AioCompletion *completion2 = cluster.aio_create_completion();
5406 ASSERT_EQ(0, cache_ioctx.aio_operate(
5407 "foo", completion2, &op2,
5408 librados::OPERATION_IGNORE_OVERLAY |
5409 librados::OPERATION_SKIPRWLOCKS, NULL));
5410
9f95a23c
TL		 5411 completion->wait_for_complete();
5412 completion2->wait_for_complete();
7c673cae
FG		 5413 ASSERT_EQ(0, completion->get_return_value());
5414 ASSERT_EQ(0, completion2->get_return_value());
5415 completion->release();
5416 completion2->release();
5417 }
5418
5419 // create/dirty object
5420 int tries = 1000;
5421 do {
5422 {
5423 bufferlist bl;
5424 bl.append("hi there");
5425 ObjectWriteOperation op;
5426 op.write_full(bl);
5427 ASSERT_EQ(0, ioctx.operate("foo", &op));
5428 }
5429
5430 // try-flush + flush
5431 // (flush will not piggyback on try-flush)
5432 {
5433 ObjectReadOperation op;
5434 op.cache_try_flush();
5435 librados::AioCompletion *completion = cluster.aio_create_completion();
5436 ASSERT_EQ(0, cache_ioctx.aio_operate(
5437 "foo", completion, &op,
5438 librados::OPERATION_IGNORE_OVERLAY |
5439 librados::OPERATION_SKIPRWLOCKS, NULL));
5440
5441 ObjectReadOperation op2;
5442 op2.cache_flush();
5443 librados::AioCompletion *completion2 = cluster.aio_create_completion();
5444 ASSERT_EQ(0, cache_ioctx.aio_operate(
5445 "foo", completion2, &op2,
5446 librados::OPERATION_IGNORE_OVERLAY, NULL));
5447
9f95a23c
TL		 5448 completion->wait_for_complete();
5449 completion2->wait_for_complete();
7c673cae
FG		 5450 int r = completion->get_return_value();
5451 ASSERT_TRUE(r == -EBUSY || r == 0);
5452 ASSERT_EQ(0, completion2->get_return_value());
5453 completion->release();
5454 completion2->release();
5455 if (r == -EBUSY)
5456 break;
5457 cout << "didn't get EBUSY, trying again" << std::endl;
5458 }
5459 ASSERT_TRUE(--tries);
5460 } while (true);
5461
5462 // create/dirty object
5463 {
5464 bufferlist bl;
5465 bl.append("hi there");
5466 ObjectWriteOperation op;
5467 op.write_full(bl);
5468 ASSERT_EQ(0, ioctx.operate("foo", &op));
5469 }
5470
5471 // try-flush + try-flush
5472 {
5473 ObjectReadOperation op;
5474 op.cache_try_flush();
5475 librados::AioCompletion *completion = cluster.aio_create_completion();
5476 ASSERT_EQ(0, cache_ioctx.aio_operate(
5477 "foo", completion, &op,
5478 librados::OPERATION_IGNORE_OVERLAY |
5479 librados::OPERATION_SKIPRWLOCKS, NULL));
5480
5481 ObjectReadOperation op2;
5482 op2.cache_try_flush();
5483 librados::AioCompletion *completion2 = cluster.aio_create_completion();
5484 ASSERT_EQ(0, cache_ioctx.aio_operate(
5485 "foo", completion2, &op2,
5486 librados::OPERATION_IGNORE_OVERLAY |
5487 librados::OPERATION_SKIPRWLOCKS, NULL));
5488
9f95a23c
TL		 5489 completion->wait_for_complete();
5490 completion2->wait_for_complete();
7c673cae
FG		 5491 ASSERT_EQ(0, completion->get_return_value());
5492 ASSERT_EQ(0, completion2->get_return_value());
5493 completion->release();
5494 completion2->release();
5495 }
5496}
5497
5498TEST_F(LibRadosTwoPoolsECPP, TryFlushReadRace) {
5499 // configure cache
5500 bufferlist inbl;
5501 ASSERT_EQ(0, cluster.mon_command(
5502 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
5503 "\", \"tierpool\": \"" + cache_pool_name +
5504 "\", \"force_nonempty\": \"--force-nonempty\" }",
5505 inbl, NULL, NULL));
5506 ASSERT_EQ(0, cluster.mon_command(
5507 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
5508 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
5509 inbl, NULL, NULL));
5510 ASSERT_EQ(0, cluster.mon_command(
5511 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
5512 "\", \"mode\": \"writeback\"}",
5513 inbl, NULL, NULL));
5514
5515 // wait for maps to settle
5516 cluster.wait_for_latest_osdmap();
5517
5518 // create/dirty object
5519 {
5520 bufferlist bl;
5521 bl.append("hi there");
5522 bufferptr bp(4000000); // make it big!
5523 bp.zero();
5524 bl.append(bp);
5525 ObjectWriteOperation op;
5526 op.write_full(bl);
5527 ASSERT_EQ(0, ioctx.operate("foo", &op));
5528 }
5529
5530 // start a continuous stream of reads
5531 read_ioctx = &ioctx;
9f95a23c 5532 test_lock.lock();
7c673cae
FG		 5533 for (int i = 0; i < max_reads; ++i) {
5534 start_flush_read();
5535 num_reads++;
5536 }
9f95a23c 5537 test_lock.unlock();
7c673cae
FG		 5538
5539 // try-flush
5540 ObjectReadOperation op;
5541 op.cache_try_flush();
5542 librados::AioCompletion *completion = cluster.aio_create_completion();
5543 ASSERT_EQ(0, cache_ioctx.aio_operate(
5544 "foo", completion, &op,
5545 librados::OPERATION_IGNORE_OVERLAY |
5546 librados::OPERATION_SKIPRWLOCKS, NULL));
5547
9f95a23c 5548 completion->wait_for_complete();
7c673cae
FG		 5549 ASSERT_EQ(0, completion->get_return_value());
5550 completion->release();
5551
5552 // stop reads
9f95a23c
TL		 5553 std::unique_lock locker{test_lock};
5554 max_reads = 0;
5555 cond.wait(locker, [] { return num_reads == 0;});
7c673cae
FG		 5556}
5557
5558TEST_F(LibRadosTierECPP, CallForcesPromote) {
5559 Rados cluster;
5560 std::string pool_name = get_temp_pool_name();
5561 std::string cache_pool_name = pool_name + "-cache";
5562 ASSERT_EQ("", create_one_ec_pool_pp(pool_name, cluster));
5563 ASSERT_EQ(0, cluster.pool_create(cache_pool_name.c_str()));
5564 IoCtx cache_ioctx;
5565 ASSERT_EQ(0, cluster.ioctx_create(cache_pool_name.c_str(), cache_ioctx));
c07f9fc5 5566 cache_ioctx.application_enable("rados", true);
7c673cae
FG		 5567 IoCtx ioctx;
5568 ASSERT_EQ(0, cluster.ioctx_create(pool_name.c_str(), ioctx));
5569
5570 // configure cache
5571 bufferlist inbl;
5572 ASSERT_EQ(0, cluster.mon_command(
5573 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
5574 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
5575 inbl, NULL, NULL));
5576 ASSERT_EQ(0, cluster.mon_command(
5577 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
5578 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
5579 inbl, NULL, NULL));
5580 ASSERT_EQ(0, cluster.mon_command(
5581 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
5582 "\", \"mode\": \"writeback\"}",
5583 inbl, NULL, NULL));
5584
5585 // set things up such that the op would normally be proxied
5586 ASSERT_EQ(0, cluster.mon_command(
5587 set_pool_str(cache_pool_name, "hit_set_count", 2),
5588 inbl, NULL, NULL));
5589 ASSERT_EQ(0, cluster.mon_command(
5590 set_pool_str(cache_pool_name, "hit_set_period", 600),
5591 inbl, NULL, NULL));
5592 ASSERT_EQ(0, cluster.mon_command(
5593 set_pool_str(cache_pool_name, "hit_set_type",
5594 "explicit_object"),
5595 inbl, NULL, NULL));
5596 ASSERT_EQ(0, cluster.mon_command(
5597 set_pool_str(cache_pool_name, "min_read_recency_for_promote",
5598 "4"),
5599 inbl, NULL, NULL));
5600
5601 // wait for maps to settle
5602 cluster.wait_for_latest_osdmap();
5603
5604 // create/dirty object
5605 bufferlist bl;
5606 bl.append("hi there");
5607 {
5608 ObjectWriteOperation op;
5609 op.write_full(bl);
5610 ASSERT_EQ(0, ioctx.operate("foo", &op));
5611 }
5612
5613 // flush
5614 {
5615 ObjectReadOperation op;
5616 op.cache_flush();
5617 librados::AioCompletion *completion = cluster.aio_create_completion();
5618 ASSERT_EQ(0, cache_ioctx.aio_operate(
5619 "foo", completion, &op,
5620 librados::OPERATION_IGNORE_OVERLAY, NULL));
9f95a23c 5621 completion->wait_for_complete();
7c673cae
FG		 5622 ASSERT_EQ(0, completion->get_return_value());
5623 completion->release();
5624 }
5625
5626 // evict
5627 {
5628 ObjectReadOperation op;
5629 op.cache_evict();
5630 librados::AioCompletion *completion = cluster.aio_create_completion();
5631 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op,
5632 librados::OPERATION_IGNORE_CACHE,
5633 NULL));
9f95a23c 5634 completion->wait_for_complete();
7c673cae
FG		 5635 ASSERT_EQ(0, completion->get_return_value());
5636 completion->release();
5637 }
5638
5639 // call
5640 {
5641 ObjectReadOperation op;
5642 bufferlist bl;
5643 op.exec("rbd", "get_id", bl);
5644 bufferlist out;
5645 // should get EIO (not an rbd object), not -EOPNOTSUPP (we didn't promote)
5646 ASSERT_EQ(-5, ioctx.operate("foo", &op, &out));
5647 }
5648
5649 // make sure foo is back in the cache tier
5650 {
5651 NObjectIterator it = cache_ioctx.nobjects_begin();
5652 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
5653 ASSERT_TRUE(it->get_oid() == string("foo"));
5654 ++it;
5655 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
5656 }
5657
5658 // tear down tiers
5659 ASSERT_EQ(0, cluster.mon_command(
5660 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
5661 "\"}",
5662 inbl, NULL, NULL));
5663 ASSERT_EQ(0, cluster.mon_command(
5664 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
5665 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
5666 inbl, NULL, NULL));
5667
5668 // wait for maps to settle before next test
5669 cluster.wait_for_latest_osdmap();
5670
5671 ASSERT_EQ(0, cluster.pool_delete(cache_pool_name.c_str()));
9f95a23c 5672 ASSERT_EQ(0, destroy_one_ec_pool_pp(pool_name, cluster));
7c673cae
FG		 5673}
5674
5675TEST_F(LibRadosTierECPP, HitSetNone) {
5676 {
5677 list< pair<time_t,time_t> > ls;
5678 AioCompletion *c = librados::Rados::aio_create_completion();
5679 ASSERT_EQ(0, ioctx.hit_set_list(123, c, &ls));
5680 c->wait_for_complete();
5681 ASSERT_EQ(0, c->get_return_value());
5682 ASSERT_TRUE(ls.empty());
5683 c->release();
5684 }
5685 {
5686 bufferlist bl;
5687 AioCompletion *c = librados::Rados::aio_create_completion();
5688 ASSERT_EQ(0, ioctx.hit_set_get(123, c, 12345, &bl));
5689 c->wait_for_complete();
5690 ASSERT_EQ(-ENOENT, c->get_return_value());
5691 c->release();
5692 }
5693}
5694
5695TEST_F(LibRadosTwoPoolsECPP, HitSetRead) {
5696 // make it a tier
5697 bufferlist inbl;
5698 ASSERT_EQ(0, cluster.mon_command(
5699 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
5700 "\", \"tierpool\": \"" + cache_pool_name +
5701 "\", \"force_nonempty\": \"--force-nonempty\" }",
5702 inbl, NULL, NULL));
5703
5704 // enable hitset tracking for this pool
5705 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", 2),
5706 inbl, NULL, NULL));
5707 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", 600),
5708 inbl, NULL, NULL));
5709 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type",
5710 "explicit_object"),
5711 inbl, NULL, NULL));
5712
5713 // wait for maps to settle
5714 cluster.wait_for_latest_osdmap();
5715
5716 cache_ioctx.set_namespace("");
5717
5718 // keep reading until we see our object appear in the HitSet
5719 utime_t start = ceph_clock_now();
5720 utime_t hard_stop = start + utime_t(600, 0);
5721
5722 while (true) {
5723 utime_t now = ceph_clock_now();
5724 ASSERT_TRUE(now < hard_stop);
5725
5726 string name = "foo";
5727 uint32_t hash;
5728 ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(name, &hash));
5729 hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash,
5730 cluster.pool_lookup(cache_pool_name.c_str()), "");
5731
5732 bufferlist bl;
5733 ASSERT_EQ(-ENOENT, cache_ioctx.read("foo", bl, 1, 0));
5734
5735 bufferlist hbl;
5736 AioCompletion *c = librados::Rados::aio_create_completion();
5737 ASSERT_EQ(0, cache_ioctx.hit_set_get(hash, c, now.sec(), &hbl));
5738 c->wait_for_complete();
5739 c->release();
5740
5741 if (hbl.length()) {
11fdf7f2 5742 auto p = hbl.cbegin();
7c673cae 5743 HitSet hs;
11fdf7f2 5744 decode(hs, p);
7c673cae
FG		 5745 if (hs.contains(oid)) {
5746 cout << "ok, hit_set contains " << oid << std::endl;
5747 break;
5748 }
5749 cout << "hmm, not in HitSet yet" << std::endl;
5750 } else {
5751 cout << "hmm, no HitSet yet" << std::endl;
5752 }
5753
5754 sleep(1);
5755 }
5756}
5757
5758// disable this test until hitset-get reliably works on EC pools
5759#if 0
5760TEST_F(LibRadosTierECPP, HitSetWrite) {
5761 int num_pg = _get_pg_num(cluster, pool_name);
11fdf7f2 5762 ceph_assert(num_pg > 0);
7c673cae
FG		 5763
5764 // enable hitset tracking for this pool
5765 bufferlist inbl;
5766 ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_count", 8),
5767 inbl, NULL, NULL));
5768 ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_period", 600),
5769 inbl, NULL, NULL));
5770 ASSERT_EQ(0, cluster.mon_command(set_pool_str(pool_name, "hit_set_type",
5771 "explicit_hash"),
5772 inbl, NULL, NULL));
5773
5774 // wait for maps to settle
5775 cluster.wait_for_latest_osdmap();
5776
5777 ioctx.set_namespace("");
5778
5779 // do a bunch of writes
5780 for (int i=0; i<1000; ++i) {
5781 bufferlist bl;
5782 bl.append("a");
5783 ASSERT_EQ(0, ioctx.write(stringify(i), bl, 1, 0));
5784 }
5785
5786 // get HitSets
5787 std::map<int,HitSet> hitsets;
5788 for (int i=0; i<num_pg; ++i) {
5789 list< pair<time_t,time_t> > ls;
5790 AioCompletion *c = librados::Rados::aio_create_completion();
5791 ASSERT_EQ(0, ioctx.hit_set_list(i, c, &ls));
5792 c->wait_for_complete();
5793 c->release();
5794 std::cout << "pg " << i << " ls " << ls << std::endl;
5795 ASSERT_FALSE(ls.empty());
5796
5797 // get the latest
5798 c = librados::Rados::aio_create_completion();
5799 bufferlist bl;
5800 ASSERT_EQ(0, ioctx.hit_set_get(i, c, ls.back().first, &bl));
5801 c->wait_for_complete();
5802 c->release();
5803
5804 //std::cout << "bl len is " << bl.length() << "\n";
5805 //bl.hexdump(std::cout);
5806 //std::cout << std::endl;
5807
11fdf7f2
TL		 5808 auto p = bl.cbegin();
5809 decode(hitsets[i], p);
7c673cae
FG		 5810
5811 // cope with racing splits by refreshing pg_num
5812 if (i == num_pg - 1)
5813 num_pg = _get_pg_num(cluster, pool_name);
5814 }
5815
5816 for (int i=0; i<1000; ++i) {
5817 string n = stringify(i);
5818 uint32_t hash = ioctx.get_object_hash_position(n);
5819 hobject_t oid(sobject_t(n, CEPH_NOSNAP), "", hash,
5820 cluster.pool_lookup(pool_name.c_str()), "");
5821 std::cout << "checking for " << oid << std::endl;
5822 bool found = false;
5823 for (int p=0; p<num_pg; ++p) {
5824 if (hitsets[p].contains(oid)) {
5825 found = true;
5826 break;
5827 }
5828 }
5829 ASSERT_TRUE(found);
5830 }
5831}
5832#endif
5833
5834TEST_F(LibRadosTwoPoolsECPP, HitSetTrim) {
5835 unsigned count = 3;
5836 unsigned period = 3;
5837
5838 // make it a tier
5839 bufferlist inbl;
5840 ASSERT_EQ(0, cluster.mon_command(
5841 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
5842 "\", \"tierpool\": \"" + cache_pool_name +
5843 "\", \"force_nonempty\": \"--force-nonempty\" }",
5844 inbl, NULL, NULL));
5845
5846 // enable hitset tracking for this pool
5847 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_count", count),
5848 inbl, NULL, NULL));
5849 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_period", period),
5850 inbl, NULL, NULL));
5851 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
5852 inbl, NULL, NULL));
5853 ASSERT_EQ(0, cluster.mon_command(set_pool_str(cache_pool_name, "hit_set_fpp", ".01"),
5854 inbl, NULL, NULL));
5855
5856 // wait for maps to settle
5857 cluster.wait_for_latest_osdmap();
5858
5859 cache_ioctx.set_namespace("");
5860
5861 // do a bunch of writes and make sure the hitsets rotate
5862 utime_t start = ceph_clock_now();
5863 utime_t hard_stop = start + utime_t(count * period * 50, 0);
5864
5865 time_t first = 0;
5866 int bsize = alignment;
5867 char *buf = (char *)new char[bsize];
5868 memset(buf, 'f', bsize);
5869
5870 while (true) {
5871 string name = "foo";
5872 uint32_t hash;
5873 ASSERT_EQ(0, cache_ioctx.get_object_hash_position2(name, &hash));
5874 hobject_t oid(sobject_t(name, CEPH_NOSNAP), "", hash, -1, "");
5875
5876 bufferlist bl;
5877 bl.append(buf, bsize);
5878 ASSERT_EQ(0, cache_ioctx.append("foo", bl, bsize));
5879
5880 list<pair<time_t, time_t> > ls;
5881 AioCompletion *c = librados::Rados::aio_create_completion();
5882 ASSERT_EQ(0, cache_ioctx.hit_set_list(hash, c, &ls));
5883 c->wait_for_complete();
5884 c->release();
5885
5886 cout << " got ls " << ls << std::endl;
5887 if (!ls.empty()) {
5888 if (!first) {
5889 first = ls.front().first;
5890 cout << "first is " << first << std::endl;
5891 } else {
5892 if (ls.front().first != first) {
5893 cout << "first now " << ls.front().first << ", trimmed" << std::endl;
5894 break;
5895 }
5896 }
5897 }
5898
5899 utime_t now = ceph_clock_now();
5900 ASSERT_TRUE(now < hard_stop);
5901
5902 sleep(1);
5903 }
5904 delete[] buf;
5905}
5906
5907TEST_F(LibRadosTwoPoolsECPP, PromoteOn2ndRead) {
5908 // create object
5909 for (int i=0; i<20; ++i) {
5910 bufferlist bl;
5911 bl.append("hi there");
5912 ObjectWriteOperation op;
5913 op.write_full(bl);
5914 ASSERT_EQ(0, ioctx.operate("foo" + stringify(i), &op));
5915 }
5916
5917 // configure cache
5918 bufferlist inbl;
5919 ASSERT_EQ(0, cluster.mon_command(
5920 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
5921 "\", \"tierpool\": \"" + cache_pool_name +
5922 "\", \"force_nonempty\": \"--force-nonempty\" }",
5923 inbl, NULL, NULL));
5924 ASSERT_EQ(0, cluster.mon_command(
5925 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
5926 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
5927 inbl, NULL, NULL));
5928 ASSERT_EQ(0, cluster.mon_command(
5929 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
5930 "\", \"mode\": \"writeback\"}",
5931 inbl, NULL, NULL));
5932
5933 // enable hitset tracking for this pool
5934 ASSERT_EQ(0, cluster.mon_command(
5935 set_pool_str(cache_pool_name, "hit_set_count", 2),
5936 inbl, NULL, NULL));
5937 ASSERT_EQ(0, cluster.mon_command(
5938 set_pool_str(cache_pool_name, "hit_set_period", 600),
5939 inbl, NULL, NULL));
5940 ASSERT_EQ(0, cluster.mon_command(
5941 set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
5942 inbl, NULL, NULL));
5943 ASSERT_EQ(0, cluster.mon_command(
5944 set_pool_str(cache_pool_name, "min_read_recency_for_promote", 1),
5945 inbl, NULL, NULL));
5946 ASSERT_EQ(0, cluster.mon_command(
5947 set_pool_str(cache_pool_name, "hit_set_grade_decay_rate", 20),
5948 inbl, NULL, NULL));
5949 ASSERT_EQ(0, cluster.mon_command(
5950 set_pool_str(cache_pool_name, "hit_set_search_last_n", 1),
5951 inbl, NULL, NULL));
5952
5953 // wait for maps to settle
5954 cluster.wait_for_latest_osdmap();
5955
5956 int fake = 0; // set this to non-zero to test spurious promotion,
5957 // e.g. from thrashing
5958 int attempt = 0;
5959 string obj;
5960 while (true) {
5961 // 1st read, don't trigger a promote
5962 obj = "foo" + stringify(attempt);
5963 cout << obj << std::endl;
5964 {
5965 bufferlist bl;
5966 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
5967 if (--fake >= 0) {
5968 sleep(1);
5969 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
5970 sleep(1);
5971 }
5972 }
5973
5974 // verify the object is NOT present in the cache tier
5975 {
5976 bool found = false;
5977 NObjectIterator it = cache_ioctx.nobjects_begin();
5978 while (it != cache_ioctx.nobjects_end()) {
5979 cout << " see " << it->get_oid() << std::endl;
5980 if (it->get_oid() == string(obj.c_str())) {
5981 found = true;
5982 break;
5983 }
5984 ++it;
5985 }
5986 if (!found)
5987 break;
5988 }
5989
5990 ++attempt;
5991 ASSERT_LE(attempt, 20);
5992 cout << "hrm, object is present in cache on attempt " << attempt
5993 << ", retrying" << std::endl;
5994 }
5995
5996 // Read until the object is present in the cache tier
5997 cout << "verifying " << obj << " is eventually promoted" << std::endl;
5998 while (true) {
5999 bufferlist bl;
6000 ASSERT_EQ(1, ioctx.read(obj.c_str(), bl, 1, 0));
6001
6002 bool there = false;
6003 NObjectIterator it = cache_ioctx.nobjects_begin();
6004 while (it != cache_ioctx.nobjects_end()) {
6005 if (it->get_oid() == string(obj.c_str())) {
6006 there = true;
6007 break;
6008 }
6009 ++it;
6010 }
6011 if (there)
6012 break;
6013
6014 sleep(1);
6015 }
6016
6017 // tear down tiers
6018 ASSERT_EQ(0, cluster.mon_command(
6019 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
6020 "\"}",
6021 inbl, NULL, NULL));
6022 ASSERT_EQ(0, cluster.mon_command(
6023 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
6024 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
6025 inbl, NULL, NULL));
6026
6027 // wait for maps to settle before next test
6028 cluster.wait_for_latest_osdmap();
6029}
6030
6031TEST_F(LibRadosTwoPoolsECPP, ProxyRead) {
6032 // create object
6033 {
6034 bufferlist bl;
6035 bl.append("hi there");
6036 ObjectWriteOperation op;
6037 op.write_full(bl);
6038 ASSERT_EQ(0, ioctx.operate("foo", &op));
6039 }
6040
6041 // configure cache
6042 bufferlist inbl;
6043 ASSERT_EQ(0, cluster.mon_command(
6044 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
6045 "\", \"tierpool\": \"" + cache_pool_name +
6046 "\", \"force_nonempty\": \"--force-nonempty\" }",
6047 inbl, NULL, NULL));
6048 ASSERT_EQ(0, cluster.mon_command(
6049 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
6050 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
6051 inbl, NULL, NULL));
6052 ASSERT_EQ(0, cluster.mon_command(
6053 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
6054 "\", \"mode\": \"readproxy\"}",
6055 inbl, NULL, NULL));
6056
6057 // wait for maps to settle
6058 cluster.wait_for_latest_osdmap();
6059
6060 // read and verify the object
6061 {
6062 bufferlist bl;
6063 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
6064 ASSERT_EQ('h', bl[0]);
6065 }
6066
6067 // Verify 10 times the object is NOT present in the cache tier
6068 uint32_t i = 0;
6069 while (i++ < 10) {
6070 NObjectIterator it = cache_ioctx.nobjects_begin();
6071 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
6072 sleep(1);
6073 }
6074
6075 // tear down tiers
6076 ASSERT_EQ(0, cluster.mon_command(
6077 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
6078 "\"}",
6079 inbl, NULL, NULL));
6080 ASSERT_EQ(0, cluster.mon_command(
6081 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
6082 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
6083 inbl, NULL, NULL));
6084
6085 // wait for maps to settle before next test
6086 cluster.wait_for_latest_osdmap();
6087}
6088
6089TEST_F(LibRadosTwoPoolsECPP, CachePin) {
6090 // create object
6091 {
6092 bufferlist bl;
6093 bl.append("hi there");
6094 ObjectWriteOperation op;
6095 op.write_full(bl);
6096 ASSERT_EQ(0, ioctx.operate("foo", &op));
6097 }
6098 {
6099 bufferlist bl;
6100 bl.append("hi there");
6101 ObjectWriteOperation op;
6102 op.write_full(bl);
6103 ASSERT_EQ(0, ioctx.operate("bar", &op));
6104 }
6105 {
6106 bufferlist bl;
6107 bl.append("hi there");
6108 ObjectWriteOperation op;
6109 op.write_full(bl);
6110 ASSERT_EQ(0, ioctx.operate("baz", &op));
6111 }
6112 {
6113 bufferlist bl;
6114 bl.append("hi there");
6115 ObjectWriteOperation op;
6116 op.write_full(bl);
6117 ASSERT_EQ(0, ioctx.operate("bam", &op));
6118 }
6119
6120 // configure cache
6121 bufferlist inbl;
6122 ASSERT_EQ(0, cluster.mon_command(
6123 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
6124 "\", \"tierpool\": \"" + cache_pool_name +
6125 "\", \"force_nonempty\": \"--force-nonempty\" }",
6126 inbl, NULL, NULL));
6127 ASSERT_EQ(0, cluster.mon_command(
6128 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
6129 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
6130 inbl, NULL, NULL));
6131 ASSERT_EQ(0, cluster.mon_command(
6132 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
6133 "\", \"mode\": \"writeback\"}",
6134 inbl, NULL, NULL));
6135
6136 // wait for maps to settle
6137 cluster.wait_for_latest_osdmap();
6138
6139 // read, trigger promote
6140 {
6141 bufferlist bl;
6142 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
6143 ASSERT_EQ(1, ioctx.read("bar", bl, 1, 0));
6144 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
6145 ASSERT_EQ(1, ioctx.read("bam", bl, 1, 0));
6146 }
6147
6148 // verify the objects are present in the cache tier
6149 {
6150 NObjectIterator it = cache_ioctx.nobjects_begin();
6151 ASSERT_TRUE(it != cache_ioctx.nobjects_end());
6152 for (uint32_t i = 0; i < 4; i++) {
6153 ASSERT_TRUE(it->get_oid() == string("foo") ||
6154 it->get_oid() == string("bar") ||
6155 it->get_oid() == string("baz") ||
6156 it->get_oid() == string("bam"));
6157 ++it;
6158 }
6159 ASSERT_TRUE(it == cache_ioctx.nobjects_end());
6160 }
6161
6162 // pin objects
6163 {
6164 ObjectWriteOperation op;
6165 op.cache_pin();
6166 librados::AioCompletion *completion = cluster.aio_create_completion();
6167 ASSERT_EQ(0, cache_ioctx.aio_operate("foo", completion, &op));
9f95a23c 6168 completion->wait_for_complete();
7c673cae
FG		 6169 ASSERT_EQ(0, completion->get_return_value());
6170 completion->release();
6171 }
6172 {
6173 ObjectWriteOperation op;
6174 op.cache_pin();
6175 librados::AioCompletion *completion = cluster.aio_create_completion();
6176 ASSERT_EQ(0, cache_ioctx.aio_operate("baz", completion, &op));
9f95a23c 6177 completion->wait_for_complete();
7c673cae
FG		 6178 ASSERT_EQ(0, completion->get_return_value());
6179 completion->release();
6180 }
6181
6182 // enable agent
6183 ASSERT_EQ(0, cluster.mon_command(
6184 set_pool_str(cache_pool_name, "hit_set_count", 2),
6185 inbl, NULL, NULL));
6186 ASSERT_EQ(0, cluster.mon_command(
6187 set_pool_str(cache_pool_name, "hit_set_period", 600),
6188 inbl, NULL, NULL));
6189 ASSERT_EQ(0, cluster.mon_command(
6190 set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
6191 inbl, NULL, NULL));
6192 ASSERT_EQ(0, cluster.mon_command(
6193 set_pool_str(cache_pool_name, "min_read_recency_for_promote", 1),
6194 inbl, NULL, NULL));
6195 ASSERT_EQ(0, cluster.mon_command(
6196 set_pool_str(cache_pool_name, "target_max_objects", 1),
6197 inbl, NULL, NULL));
6198
6199 sleep(10);
6200
6201 // Verify the pinned object 'foo' is not flushed/evicted
6202 uint32_t count = 0;
6203 while (true) {
6204 bufferlist bl;
6205 ASSERT_EQ(1, ioctx.read("baz", bl, 1, 0));
6206
6207 count = 0;
6208 NObjectIterator it = cache_ioctx.nobjects_begin();
6209 while (it != cache_ioctx.nobjects_end()) {
6210 ASSERT_TRUE(it->get_oid() == string("foo") ||
6211 it->get_oid() == string("bar") ||
6212 it->get_oid() == string("baz") ||
6213 it->get_oid() == string("bam"));
6214 ++count;
6215 ++it;
6216 }
6217 if (count == 2) {
6218 ASSERT_TRUE(it->get_oid() == string("foo") ||
6219 it->get_oid() == string("baz"));
6220 break;
6221 }
6222
6223 sleep(1);
6224 }
6225
6226 // tear down tiers
6227 ASSERT_EQ(0, cluster.mon_command(
6228 "{\"prefix\": \"osd tier remove-overlay\", \"pool\": \"" + pool_name +
6229 "\"}",
6230 inbl, NULL, NULL));
6231 ASSERT_EQ(0, cluster.mon_command(
6232 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
6233 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
6234 inbl, NULL, NULL));
6235
6236 // wait for maps to settle before next test
6237 cluster.wait_for_latest_osdmap();
6238}
31f18b77 6239TEST_F(LibRadosTwoPoolsECPP, SetRedirectRead) {
31f18b77
FG		 6240 // create object
6241 {
6242 bufferlist bl;
6243 bl.append("hi there");
6244 ObjectWriteOperation op;
6245 op.write_full(bl);
6246 ASSERT_EQ(0, ioctx.operate("foo", &op));
6247 }
6248 {
6249 bufferlist bl;
6250 bl.append("there");
6251 ObjectWriteOperation op;
6252 op.write_full(bl);
6253 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
6254 }
6255
6256 // configure tier
6257 bufferlist inbl;
6258 ASSERT_EQ(0, cluster.mon_command(
6259 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
6260 "\", \"tierpool\": \"" + cache_pool_name +
6261 "\", \"force_nonempty\": \"--force-nonempty\" }",
6262 inbl, NULL, NULL));
6263
6264 // wait for maps to settle
6265 cluster.wait_for_latest_osdmap();
6266
6267 {
6268 ObjectWriteOperation op;
6269 op.set_redirect("bar", cache_ioctx, 0);
6270 librados::AioCompletion *completion = cluster.aio_create_completion();
6271 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 6272 completion->wait_for_complete();
31f18b77
FG		 6273 ASSERT_EQ(0, completion->get_return_value());
6274 completion->release();
6275 }
6276 // read and verify the object
6277 {
6278 bufferlist bl;
6279 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
6280 ASSERT_EQ('t', bl[0]);
6281 }
6282
6283 ASSERT_EQ(0, cluster.mon_command(
6284 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
6285 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
6286 inbl, NULL, NULL));
6287
6288 // wait for maps to settle before next test
6289 cluster.wait_for_latest_osdmap();
6290}
11fdf7f2
TL		 6291
6292TEST_F(LibRadosTwoPoolsECPP, SetChunkRead) {
6293 // note: require >= mimic
6294
6295 // create object
6296 {
6297 ObjectWriteOperation op;
6298 op.create(true);
6299 ASSERT_EQ(0, ioctx.operate("foo", &op));
6300 }
6301 {
6302 bufferlist bl;
6303 bl.append("hi there");
6304 ObjectWriteOperation op;
6305 op.write_full(bl);
6306 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
6307 }
6308
6309 // configure tier
6310 bufferlist inbl;
6311 ASSERT_EQ(0, cluster.mon_command(
6312 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
6313 "\", \"tierpool\": \"" + cache_pool_name +
6314 "\", \"force_nonempty\": \"--force-nonempty\" }",
6315 inbl, NULL, NULL));
6316
6317 // wait for maps to settle
6318 cluster.wait_for_latest_osdmap();
6319
6320 // set_chunk
6321 {
6322 ObjectWriteOperation op;
6323 op.set_chunk(0, 8, cache_ioctx, "bar", 0);
6324 librados::AioCompletion *completion = cluster.aio_create_completion();
6325 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 6326 completion->wait_for_complete();
11fdf7f2
TL		 6327 ASSERT_EQ(0, completion->get_return_value());
6328 completion->release();
6329 }
6330
6331 // make all chunks dirty --> full flush --> all chunks are evicted
6332 {
6333 bufferlist bl;
6334 bl.append("There hi");
6335 ObjectWriteOperation op;
6336 op.write_full(bl);
6337 ASSERT_EQ(0, ioctx.operate("foo", &op));
6338 }
6339
6340 // read and verify the object
6341 {
6342 bufferlist bl;
6343 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
6344 ASSERT_EQ('T', bl[0]);
6345 }
6346
6347 ASSERT_EQ(0, cluster.mon_command(
6348 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
6349 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
6350 inbl, NULL, NULL));
6351
6352 // wait for maps to settle before next test
6353 cluster.wait_for_latest_osdmap();
6354}
6355
6356TEST_F(LibRadosTwoPoolsECPP, ManifestPromoteRead) {
6357 // note: require >= mimic
6358
6359 // create object
6360 {
6361 bufferlist bl;
6362 bl.append("hi there");
6363 ObjectWriteOperation op;
6364 op.write_full(bl);
6365 ASSERT_EQ(0, ioctx.operate("foo", &op));
6366 }
6367 {
6368 ObjectWriteOperation op;
6369 op.create(true);
6370 ASSERT_EQ(0, ioctx.operate("foo-chunk", &op));
6371 }
6372 {
6373 bufferlist bl;
6374 bl.append("HI there");
6375 ObjectWriteOperation op;
6376 op.write_full(bl);
6377 ASSERT_EQ(0, cache_ioctx.operate("bar", &op));
6378 }
6379 {
6380 bufferlist bl;
6381 bl.append("BASE CHUNK");
6382 ObjectWriteOperation op;
6383 op.write_full(bl);
6384 ASSERT_EQ(0, cache_ioctx.operate("bar-chunk", &op));
6385 }
6386
6387 // configure tier
6388 bufferlist inbl;
6389 ASSERT_EQ(0, cluster.mon_command(
6390 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
6391 "\", \"tierpool\": \"" + cache_pool_name +
6392 "\", \"force_nonempty\": \"--force-nonempty\" }",
6393 inbl, NULL, NULL));
6394
6395 // wait for maps to settle
6396 cluster.wait_for_latest_osdmap();
6397
6398 // set-redirect
6399 {
6400 ObjectWriteOperation op;
6401 op.set_redirect("bar", cache_ioctx, 0);
6402 librados::AioCompletion *completion = cluster.aio_create_completion();
6403 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 6404 completion->wait_for_complete();
11fdf7f2
TL		 6405 ASSERT_EQ(0, completion->get_return_value());
6406 completion->release();
6407 }
6408 // set-chunk
6409 {
6410 ObjectWriteOperation op;
6411 op.set_chunk(0, 10, cache_ioctx, "bar-chunk", 0);
6412 librados::AioCompletion *completion = cluster.aio_create_completion();
6413 ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
9f95a23c 6414 completion->wait_for_complete();
11fdf7f2
TL		 6415 ASSERT_EQ(0, completion->get_return_value());
6416 completion->release();
6417 }
6418 // promote
6419 {
6420 ObjectWriteOperation op;
6421 op.tier_promote();
6422 librados::AioCompletion *completion = cluster.aio_create_completion();
6423 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &op));
9f95a23c 6424 completion->wait_for_complete();
11fdf7f2
TL		 6425 ASSERT_EQ(0, completion->get_return_value());
6426 completion->release();
6427 }
6428 // read and verify the object (redirect)
6429 {
6430 bufferlist bl;
6431 ASSERT_EQ(1, ioctx.read("foo", bl, 1, 0));
6432 ASSERT_EQ('H', bl[0]);
6433 }
6434 // promote
6435 {
6436 ObjectWriteOperation op;
6437 op.tier_promote();
6438 librados::AioCompletion *completion = cluster.aio_create_completion();
6439 ASSERT_EQ(0, ioctx.aio_operate("foo-chunk", completion, &op));
9f95a23c 6440 completion->wait_for_complete();
11fdf7f2
TL		 6441 ASSERT_EQ(0, completion->get_return_value());
6442 completion->release();
6443 }
6444 // read and verify the object
6445 {
6446 bufferlist bl;
6447 ASSERT_EQ(1, ioctx.read("foo-chunk", bl, 1, 0));
6448 ASSERT_EQ('B', bl[0]);
6449 }
6450
6451 ASSERT_EQ(0, cluster.mon_command(
6452 "{\"prefix\": \"osd tier remove\", \"pool\": \"" + pool_name +
6453 "\", \"tierpool\": \"" + cache_pool_name + "\"}",
6454 inbl, NULL, NULL));
6455
6456 // wait for maps to settle before next test
6457 cluster.wait_for_latest_osdmap();
6458}
6459
6460TEST_F(LibRadosTwoPoolsPP, PropagateBaseTierError) {
6461 // write object to base tier
6462 bufferlist omap_bl;
6463 encode(static_cast<uint32_t>(0U), omap_bl);
6464
6465 ObjectWriteOperation op1;
6466 op1.omap_set({{"somekey", omap_bl}});
6467 ASSERT_EQ(0, ioctx.operate("propagate-base-tier-error", &op1));
6468
6469 // configure cache
6470 bufferlist inbl;
6471 ASSERT_EQ(0, cluster.mon_command(
6472 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
6473 "\", \"tierpool\": \"" + cache_pool_name +
6474 "\", \"force_nonempty\": \"--force-nonempty\" }",
6475 inbl, NULL, NULL));
6476 ASSERT_EQ(0, cluster.mon_command(
6477 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
6478 "\", \"mode\": \"writeback\"}",
6479 inbl, NULL, NULL));
6480 ASSERT_EQ(0, cluster.mon_command(
6481 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
6482 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
6483 inbl, NULL, NULL));
6484
6485 ASSERT_EQ(0, cluster.mon_command(
6486 set_pool_str(cache_pool_name, "hit_set_type", "bloom"),
6487 inbl, NULL, NULL));
6488 ASSERT_EQ(0, cluster.mon_command(
6489 set_pool_str(cache_pool_name, "hit_set_count", 1),
6490 inbl, NULL, NULL));
6491 ASSERT_EQ(0, cluster.mon_command(
6492 set_pool_str(cache_pool_name, "hit_set_period", 600),
6493 inbl, NULL, NULL));
6494 ASSERT_EQ(0, cluster.mon_command(
6495 set_pool_str(cache_pool_name, "target_max_objects", 250),
6496 inbl, NULL, NULL));
6497
6498 // wait for maps to settle
6499 cluster.wait_for_latest_osdmap();
6500
6501 // guarded op should fail so expect error to propagate to cache tier
6502 bufferlist test_omap_bl;
6503 encode(static_cast<uint32_t>(1U), test_omap_bl);
6504
6505 ObjectWriteOperation op2;
6506 op2.omap_cmp({{"somekey", {test_omap_bl, CEPH_OSD_CMPXATTR_OP_EQ}}}, nullptr);
6507 op2.omap_set({{"somekey", test_omap_bl}});
6508
6509 ASSERT_EQ(-ECANCELED, ioctx.operate("propagate-base-tier-error", &op2));
6510}
9f95a23c
TL		 6511
6512TEST_F(LibRadosTwoPoolsPP, HelloWriteReturn) {
6513 // configure cache
6514 bufferlist inbl;
6515 ASSERT_EQ(0, cluster.mon_command(
6516 "{\"prefix\": \"osd tier add\", \"pool\": \"" + pool_name +
6517 "\", \"tierpool\": \"" + cache_pool_name +
6518 "\", \"force_nonempty\": \"--force-nonempty\" }",
6519 inbl, NULL, NULL));
6520 ASSERT_EQ(0, cluster.mon_command(
6521 "{\"prefix\": \"osd tier set-overlay\", \"pool\": \"" + pool_name +
6522 "\", \"overlaypool\": \"" + cache_pool_name + "\"}",
6523 inbl, NULL, NULL));
6524 ASSERT_EQ(0, cluster.mon_command(
6525 "{\"prefix\": \"osd tier cache-mode\", \"pool\": \"" + cache_pool_name +
6526 "\", \"mode\": \"writeback\"}",
6527 inbl, NULL, NULL));
6528
6529 // set things up such that the op would normally be proxied
6530 ASSERT_EQ(0, cluster.mon_command(
6531 set_pool_str(cache_pool_name, "hit_set_count", 2),
6532 inbl, NULL, NULL));
6533 ASSERT_EQ(0, cluster.mon_command(
6534 set_pool_str(cache_pool_name, "hit_set_period", 600),
6535 inbl, NULL, NULL));
6536 ASSERT_EQ(0, cluster.mon_command(
6537 set_pool_str(cache_pool_name, "hit_set_type",
6538 "explicit_object"),
6539 inbl, NULL, NULL));
6540 ASSERT_EQ(0, cluster.mon_command(
6541 set_pool_str(cache_pool_name, "min_read_recency_for_promote",
6542 "10000"),
6543 inbl, NULL, NULL));
6544
6545 // wait for maps to settle
6546 cluster.wait_for_latest_osdmap();
6547
6548 // this *will* return data due to the RETURNVEC flag
6549 {
6550 bufferlist in, out;
6551 int rval;
6552 ObjectWriteOperation o;
6553 o.exec("hello", "write_return_data", in, &out, &rval);
6554 librados::AioCompletion *completion = cluster.aio_create_completion();
6555 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &o,
6556 librados::OPERATION_RETURNVEC));
6557 completion->wait_for_complete();
6558 ASSERT_EQ(42, completion->get_return_value());
6559 ASSERT_EQ(42, rval);
6560 out.hexdump(std::cout);
6561 ASSERT_EQ("you might see this", std::string(out.c_str(), out.length()));
6562 }
6563
6564 // this will overflow because the return data is too big
6565 {
6566 bufferlist in, out;
6567 int rval;
6568 ObjectWriteOperation o;
6569 o.exec("hello", "write_too_much_return_data", in, &out, &rval);
6570 librados::AioCompletion *completion = cluster.aio_create_completion();
6571 ASSERT_EQ(0, ioctx.aio_operate("foo", completion, &o,
6572 librados::OPERATION_RETURNVEC));
6573 completion->wait_for_complete();
6574 ASSERT_EQ(-EOVERFLOW, completion->get_return_value());
6575 ASSERT_EQ(-EOVERFLOW, rval);
6576 ASSERT_EQ("", std::string(out.c_str(), out.length()));
6577 }
6578}
Ceph