1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
20 #include <sys/mount.h>
21 #include <boost/scoped_ptr.hpp>
22 #include <boost/random/mersenne_twister.hpp>
23 #include <boost/random/uniform_int.hpp>
24 #include <boost/random/binomial_distribution.hpp>
25 #include <gtest/gtest.h>
27 #include "os/ObjectStore.h"
28 #include "os/filestore/FileStore.h"
29 #if defined(HAVE_LIBAIO)
30 #include "os/bluestore/BlueStore.h"
32 #include "include/Context.h"
33 #include "common/ceph_argparse.h"
34 #include "global/global_init.h"
35 #include "common/Mutex.h"
36 #include "common/Cond.h"
37 #include "common/errno.h"
38 #include "include/stringify.h"
39 #include "include/coredumpctl.h"
41 #include "include/unordered_map.h"
42 #include "store_test_fixture.h"
45 typedef boost::mt11213b gen_type
;
47 #define dout_context g_ceph_context
49 #if GTEST_HAS_PARAM_TEST
51 static bool bl_eq(bufferlist
& expected
, bufferlist
& actual
)
53 if (expected
.contents_equal(actual
))
57 if(expected
.length() != actual
.length()) {
58 cout
<< "--- buffer lengths mismatch " << std::hex
59 << "expected 0x" << expected
.length() << " != actual 0x"
60 << actual
.length() << std::dec
<< std::endl
;
61 derr
<< "--- buffer lengths mismatch " << std::hex
62 << "expected 0x" << expected
.length() << " != actual 0x"
63 << actual
.length() << std::dec
<< dendl
;
65 auto len
= MIN(expected
.length(), actual
.length());
66 while ( first
<len
&& expected
[first
] == actual
[first
])
69 while (last
> 0 && expected
[last
-1] == actual
[last
-1])
72 cout
<< "--- buffer mismatch between offset 0x" << std::hex
<< first
73 << " and 0x" << last
<< ", total 0x" << len
<< std::dec
75 derr
<< "--- buffer mismatch between offset 0x" << std::hex
<< first
76 << " and 0x" << last
<< ", total 0x" << len
<< std::dec
78 cout
<< "--- expected:\n";
79 expected
.hexdump(cout
);
80 cout
<< "--- actual:\n";
89 int apply_transaction(
91 ObjectStore::Sequencer
*osr
,
92 ObjectStore::Transaction
&&t
) {
94 ObjectStore::Transaction t2
;
96 return store
->apply_transaction(osr
, std::move(t2
));
98 return store
->apply_transaction(osr
, std::move(t
));
103 bool sorted(const vector
<ghobject_t
> &in
) {
105 for (vector
<ghobject_t
>::const_iterator i
= in
.begin();
109 cout
<< start
<< " should follow " << *i
<< std::endl
;
117 class StoreTest
: public StoreTestFixture
,
118 public ::testing::WithParamInterface
<const char*> {
121 : StoreTestFixture(GetParam())
125 class StoreTestDeferredSetup
: public StoreTest
{
126 void SetUp() override
{
131 void DeferredSetup() {
138 class StoreTestSpecificAUSize
: public StoreTestDeferredSetup
{
143 boost::scoped_ptr
<ObjectStore
>& store
,
147 uint64_t align
)> MatrixTest
;
149 void StartDeferred(size_t min_alloc_size
) {
150 g_conf
->set_val("bluestore_min_alloc_size", stringify(min_alloc_size
));
154 void TearDown() override
{
155 g_conf
->set_val("bluestore_min_alloc_size", "0");
156 StoreTestDeferredSetup::TearDown();
160 // bluestore matrix testing
161 uint64_t max_write
= 40 * 1024;
162 uint64_t max_size
= 400 * 1024;
163 uint64_t alignment
= 0;
164 uint64_t num_ops
= 10000;
167 string
matrix_get(const char *k
) {
168 if (string(k
) == "max_write") {
169 return stringify(max_write
);
170 } else if (string(k
) == "max_size") {
171 return stringify(max_size
);
172 } else if (string(k
) == "alignment") {
173 return stringify(alignment
);
174 } else if (string(k
) == "num_ops") {
175 return stringify(num_ops
);
178 g_conf
->get_val(k
, &buf
, -1);
185 void matrix_set(const char *k
, const char *v
) {
186 if (string(k
) == "max_write") {
187 max_write
= atoll(v
);
188 } else if (string(k
) == "max_size") {
190 } else if (string(k
) == "alignment") {
191 alignment
= atoll(v
);
192 } else if (string(k
) == "num_ops") {
195 g_conf
->set_val(k
, v
);
199 void do_matrix_choose(const char *matrix
[][10],
200 int i
, int pos
, int num
,
201 boost::scoped_ptr
<ObjectStore
>& store
,
205 for (count
= 0; matrix
[i
][count
+1]; ++count
) ;
206 for (int j
= 1; matrix
[i
][j
]; ++j
) {
207 matrix_set(matrix
[i
][0], matrix
[i
][j
]);
208 do_matrix_choose(matrix
,
216 cout
<< "---------------------- " << (pos
+ 1) << " / " << num
217 << " ----------------------" << std::endl
;
218 for (unsigned k
=0; matrix
[k
][0]; ++k
) {
219 cout
<< " " << matrix
[k
][0] << " = " << matrix_get(matrix
[k
][0])
222 g_ceph_context
->_conf
->apply_changes(NULL
);
223 fn(store
, num_ops
, max_size
, max_write
, alignment
);
227 void do_matrix(const char *matrix
[][10],
228 boost::scoped_ptr
<ObjectStore
>& store
,
230 map
<string
,string
> old
;
231 for (unsigned i
=0; matrix
[i
][0]; ++i
) {
232 old
[matrix
[i
][0]] = matrix_get(matrix
[i
][0]);
234 cout
<< "saved config options " << old
<< std::endl
;
236 if (strcmp(matrix
[0][0], "bluestore_min_alloc_size") == 0) {
238 for (count
= 0; matrix
[0][count
+1]; ++count
) ;
239 for (size_t j
= 1; matrix
[0][j
]; ++j
) {
243 StartDeferred(strtoll(matrix
[0][j
], NULL
, 10));
244 do_matrix_choose(matrix
, 1, j
- 1, count
, store
, fn
);
248 do_matrix_choose(matrix
, 0, 0, 1, store
, fn
);
251 cout
<< "restoring config options " << old
<< std::endl
;
253 cout
<< " " << p
.first
<< " = " << p
.second
<< std::endl
;
254 matrix_set(p
.first
.c_str(), p
.second
.c_str());
256 g_ceph_context
->_conf
->apply_changes(NULL
);
261 TEST_P(StoreTest
, collect_metadata
) {
262 map
<string
,string
> pm
;
263 store
->collect_metadata(&pm
);
264 if (GetParam() == string("filestore")) {
265 ASSERT_NE(pm
.count("filestore_backend"), 0u);
266 ASSERT_NE(pm
.count("filestore_f_type"), 0u);
267 ASSERT_NE(pm
.count("backend_filestore_partition_path"), 0u);
268 ASSERT_NE(pm
.count("backend_filestore_dev_node"), 0u);
272 TEST_P(StoreTest
, Trivial
) {
275 TEST_P(StoreTest
, TrivialRemount
) {
276 int r
= store
->umount();
282 TEST_P(StoreTest
, SimpleRemount
) {
283 ObjectStore::Sequencer
osr("test");
285 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
286 ghobject_t
hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP
)));
288 bl
.append("1234512345");
291 cerr
<< "create collection + write" << std::endl
;
292 ObjectStore::Transaction t
;
293 t
.create_collection(cid
, 0);
294 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
295 r
= apply_transaction(store
, &osr
, std::move(t
));
303 ObjectStore::Transaction t
;
304 t
.write(cid
, hoid2
, 0, bl
.length(), bl
);
305 r
= apply_transaction(store
, &osr
, std::move(t
));
309 ObjectStore::Transaction t
;
311 t
.remove(cid
, hoid2
);
312 t
.remove_collection(cid
);
313 cerr
<< "remove collection" << std::endl
;
314 r
= apply_transaction(store
, &osr
, std::move(t
));
322 ObjectStore::Transaction t
;
323 t
.create_collection(cid
, 0);
324 r
= apply_transaction(store
, &osr
, std::move(t
));
326 bool exists
= store
->exists(cid
, hoid
);
327 ASSERT_TRUE(!exists
);
330 ObjectStore::Transaction t
;
331 t
.remove_collection(cid
);
332 cerr
<< "remove collection" << std::endl
;
333 r
= apply_transaction(store
, &osr
, std::move(t
));
338 TEST_P(StoreTest
, IORemount
) {
339 ObjectStore::Sequencer
osr("test");
342 bl
.append("1234512345");
345 cerr
<< "create collection + objects" << std::endl
;
346 ObjectStore::Transaction t
;
347 t
.create_collection(cid
, 0);
348 for (int n
=1; n
<=100; ++n
) {
349 ghobject_t
hoid(hobject_t(sobject_t("Object " + stringify(n
), CEPH_NOSNAP
)));
350 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
352 r
= apply_transaction(store
, &osr
, std::move(t
));
357 cout
<< "overwrites" << std::endl
;
358 for (int n
=1; n
<=100; ++n
) {
359 ObjectStore::Transaction t
;
360 ghobject_t
hoid(hobject_t(sobject_t("Object " + stringify(n
), CEPH_NOSNAP
)));
361 t
.write(cid
, hoid
, 1, bl
.length(), bl
);
362 r
= apply_transaction(store
, &osr
, std::move(t
));
371 ObjectStore::Transaction t
;
372 for (int n
=1; n
<=100; ++n
) {
373 ghobject_t
hoid(hobject_t(sobject_t("Object " + stringify(n
), CEPH_NOSNAP
)));
376 t
.remove_collection(cid
);
377 r
= apply_transaction(store
, &osr
, std::move(t
));
382 TEST_P(StoreTest
, UnprintableCharsName
) {
383 ObjectStore::Sequencer
osr("test");
385 string name
= "funnychars_";
386 for (unsigned i
= 0; i
< 256; ++i
) {
389 ghobject_t
oid(hobject_t(sobject_t(name
, CEPH_NOSNAP
)));
392 cerr
<< "create collection + object" << std::endl
;
393 ObjectStore::Transaction t
;
394 t
.create_collection(cid
, 0);
396 r
= apply_transaction(store
, &osr
, std::move(t
));
404 cout
<< "removing" << std::endl
;
405 ObjectStore::Transaction t
;
407 t
.remove_collection(cid
);
408 r
= apply_transaction(store
, &osr
, std::move(t
));
413 TEST_P(StoreTest
, FiemapEmpty
) {
414 ObjectStore::Sequencer
osr("test");
417 ghobject_t
oid(hobject_t(sobject_t("fiemap_object", CEPH_NOSNAP
)));
419 ObjectStore::Transaction t
;
420 t
.create_collection(cid
, 0);
422 t
.truncate(cid
, oid
, 100000);
423 r
= apply_transaction(store
, &osr
, std::move(t
));
428 store
->fiemap(cid
, oid
, 0, 100000, bl
);
429 map
<uint64_t,uint64_t> m
, e
;
430 bufferlist::iterator p
= bl
.begin();
432 cout
<< " got " << m
<< std::endl
;
434 EXPECT_TRUE(m
== e
|| m
.empty());
437 ObjectStore::Transaction t
;
439 t
.remove_collection(cid
);
440 cerr
<< "remove collection" << std::endl
;
441 r
= apply_transaction(store
, &osr
, std::move(t
));
446 TEST_P(StoreTest
, FiemapHoles
) {
447 ObjectStore::Sequencer
osr("test");
448 const uint64_t MAX_EXTENTS
= 4000;
449 const uint64_t SKIP_STEP
= 65536;
452 ghobject_t
oid(hobject_t(sobject_t("fiemap_object", CEPH_NOSNAP
)));
456 ObjectStore::Transaction t
;
457 t
.create_collection(cid
, 0);
459 for (uint64_t i
= 0; i
< MAX_EXTENTS
; i
++)
460 t
.write(cid
, oid
, SKIP_STEP
* i
, 3, bl
);
461 r
= apply_transaction(store
, &osr
, std::move(t
));
465 //fiemap test from 0 to SKIP_STEP * (MAX_EXTENTS - 1) + 3
467 store
->fiemap(cid
, oid
, 0, SKIP_STEP
* (MAX_EXTENTS
- 1) + 3, bl
);
468 map
<uint64_t,uint64_t> m
, e
;
469 bufferlist::iterator p
= bl
.begin();
471 cout
<< " got " << m
<< std::endl
;
472 ASSERT_TRUE(!m
.empty());
474 bool extents_exist
= true;
475 if (m
.size() == MAX_EXTENTS
) {
476 for (uint64_t i
= 0; i
< MAX_EXTENTS
; i
++)
477 extents_exist
= extents_exist
&& m
.count(SKIP_STEP
*i
);
479 ASSERT_TRUE((m
.size() == 1 &&
480 m
[0] > SKIP_STEP
* (MAX_EXTENTS
- 1)) ||
481 (m
.size() == MAX_EXTENTS
&& extents_exist
));
483 // fiemap test from SKIP_STEP to SKIP_STEP * (MAX_EXTENTS - 2) + 3
484 // reset bufferlist and map
488 store
->fiemap(cid
, oid
, SKIP_STEP
, SKIP_STEP
* (MAX_EXTENTS
- 2) + 3, bl
);
491 cout
<< " got " << m
<< std::endl
;
492 ASSERT_TRUE(!m
.empty());
493 ASSERT_GE(m
[SKIP_STEP
], 3u);
494 extents_exist
= true;
495 if (m
.size() == (MAX_EXTENTS
- 2)) {
496 for (uint64_t i
= 1; i
< MAX_EXTENTS
- 1; i
++)
497 extents_exist
= extents_exist
&& m
.count(SKIP_STEP
*i
);
499 ASSERT_TRUE((m
.size() == 1 &&
500 m
[SKIP_STEP
] > SKIP_STEP
* (MAX_EXTENTS
- 2)) ||
501 (m
.size() == (MAX_EXTENTS
- 1) && extents_exist
));
504 ObjectStore::Transaction t
;
506 t
.remove_collection(cid
);
507 cerr
<< "remove collection" << std::endl
;
508 r
= apply_transaction(store
, &osr
, std::move(t
));
513 TEST_P(StoreTest
, SimpleMetaColTest
) {
514 ObjectStore::Sequencer
osr("test");
518 ObjectStore::Transaction t
;
519 t
.create_collection(cid
, 0);
520 cerr
<< "create collection" << std::endl
;
521 r
= apply_transaction(store
, &osr
, std::move(t
));
525 ObjectStore::Transaction t
;
526 t
.remove_collection(cid
);
527 cerr
<< "remove collection" << std::endl
;
528 r
= apply_transaction(store
, &osr
, std::move(t
));
532 ObjectStore::Transaction t
;
533 t
.create_collection(cid
, 0);
534 cerr
<< "add collection" << std::endl
;
535 r
= apply_transaction(store
, &osr
, std::move(t
));
539 ObjectStore::Transaction t
;
540 t
.remove_collection(cid
);
541 cerr
<< "remove collection" << std::endl
;
542 r
= apply_transaction(store
, &osr
, std::move(t
));
547 TEST_P(StoreTest
, SimplePGColTest
) {
548 ObjectStore::Sequencer
osr("test");
549 coll_t
cid(spg_t(pg_t(1,2), shard_id_t::NO_SHARD
));
552 ObjectStore::Transaction t
;
553 t
.create_collection(cid
, 4);
554 cerr
<< "create collection" << std::endl
;
555 r
= apply_transaction(store
, &osr
, std::move(t
));
559 ObjectStore::Transaction t
;
560 t
.remove_collection(cid
);
561 cerr
<< "remove collection" << std::endl
;
562 r
= apply_transaction(store
, &osr
, std::move(t
));
566 ObjectStore::Transaction t
;
567 t
.create_collection(cid
, 4);
568 cerr
<< "add collection" << std::endl
;
569 r
= apply_transaction(store
, &osr
, std::move(t
));
573 ObjectStore::Transaction t
;
574 t
.remove_collection(cid
);
575 cerr
<< "remove collection" << std::endl
;
576 r
= apply_transaction(store
, &osr
, std::move(t
));
581 TEST_P(StoreTest
, SimpleColPreHashTest
) {
582 ObjectStore::Sequencer
osr("test");
583 // Firstly we will need to revert the value making sure
584 // collection hint actually works
585 int merge_threshold
= g_ceph_context
->_conf
->filestore_merge_threshold
;
586 std::ostringstream oss
;
587 if (merge_threshold
> 0) {
588 oss
<< "-" << merge_threshold
;
589 g_ceph_context
->_conf
->set_val("filestore_merge_threshold", oss
.str().c_str());
592 uint32_t pg_num
= 128;
594 boost::uniform_int
<> pg_id_range(0, pg_num
);
595 gen_type
rng(time(NULL
));
596 int pg_id
= pg_id_range(rng
);
598 int objs_per_folder
= abs(merge_threshold
) * 16 * g_ceph_context
->_conf
->filestore_split_multiple
;
599 boost::uniform_int
<> folders_range(5, 256);
600 uint64_t expected_num_objs
= (uint64_t)objs_per_folder
* (uint64_t)folders_range(rng
);
602 coll_t
cid(spg_t(pg_t(pg_id
, 15), shard_id_t::NO_SHARD
));
605 // Create a collection along with a hint
606 ObjectStore::Transaction t
;
607 t
.create_collection(cid
, 5);
608 cerr
<< "create collection" << std::endl
;
610 ::encode(pg_num
, hint
);
611 ::encode(expected_num_objs
, hint
);
612 t
.collection_hint(cid
, ObjectStore::Transaction::COLL_HINT_EXPECTED_NUM_OBJECTS
, hint
);
613 cerr
<< "collection hint" << std::endl
;
614 r
= apply_transaction(store
, &osr
, std::move(t
));
618 // Remove the collection
619 ObjectStore::Transaction t
;
620 t
.remove_collection(cid
);
621 cerr
<< "remove collection" << std::endl
;
622 r
= apply_transaction(store
, &osr
, std::move(t
));
625 // Revert the config change so that it does not affect the split/merge tests
626 if (merge_threshold
> 0) {
628 oss
<< merge_threshold
;
629 g_ceph_context
->_conf
->set_val("filestore_merge_threshold", oss
.str().c_str());
633 TEST_P(StoreTest
, SmallBlockWrites
) {
634 ObjectStore::Sequencer
osr("test");
637 ghobject_t
hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP
)));
639 ObjectStore::Transaction t
;
640 t
.create_collection(cid
, 0);
641 cerr
<< "Creating collection " << cid
<< std::endl
;
642 r
= apply_transaction(store
, &osr
, std::move(t
));
646 bufferptr
ap(0x1000);
647 memset(ap
.c_str(), 'a', 0x1000);
650 bufferptr
bp(0x1000);
651 memset(bp
.c_str(), 'b', 0x1000);
654 bufferptr
cp(0x1000);
655 memset(cp
.c_str(), 'c', 0x1000);
657 bufferptr
zp(0x1000);
662 ObjectStore::Transaction t
;
663 t
.write(cid
, hoid
, 0, 0x1000, a
);
664 r
= apply_transaction(store
, &osr
, std::move(t
));
668 r
= store
->read(cid
, hoid
, 0, 0x4000, in
);
669 ASSERT_EQ(0x1000, r
);
671 ASSERT_TRUE(bl_eq(exp
, in
));
674 ObjectStore::Transaction t
;
675 t
.write(cid
, hoid
, 0x1000, 0x1000, b
);
676 r
= apply_transaction(store
, &osr
, std::move(t
));
680 r
= store
->read(cid
, hoid
, 0, 0x4000, in
);
681 ASSERT_EQ(0x2000, r
);
684 ASSERT_TRUE(bl_eq(exp
, in
));
687 ObjectStore::Transaction t
;
688 t
.write(cid
, hoid
, 0x3000, 0x1000, c
);
689 r
= apply_transaction(store
, &osr
, std::move(t
));
693 r
= store
->read(cid
, hoid
, 0, 0x4000, in
);
694 ASSERT_EQ(0x4000, r
);
699 ASSERT_TRUE(bl_eq(exp
, in
));
702 ObjectStore::Transaction t
;
703 t
.write(cid
, hoid
, 0x2000, 0x1000, a
);
704 r
= apply_transaction(store
, &osr
, std::move(t
));
708 r
= store
->read(cid
, hoid
, 0, 0x4000, in
);
709 ASSERT_EQ(0x4000, r
);
714 ASSERT_TRUE(bl_eq(exp
, in
));
717 ObjectStore::Transaction t
;
718 t
.write(cid
, hoid
, 0, 0x1000, c
);
719 r
= apply_transaction(store
, &osr
, std::move(t
));
724 r
= store
->read(cid
, hoid
, 0, 0x4000, in
);
725 ASSERT_EQ(0x4000, r
);
730 ASSERT_TRUE(bl_eq(exp
, in
));
733 ObjectStore::Transaction t
;
735 t
.remove_collection(cid
);
736 cerr
<< "Cleaning" << std::endl
;
737 r
= apply_transaction(store
, &osr
, std::move(t
));
742 TEST_P(StoreTest
, BufferCacheReadTest
) {
743 ObjectStore::Sequencer
osr("test");
746 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
749 r
= store
->read(cid
, hoid
, 0, 5, in
);
750 ASSERT_EQ(-ENOENT
, r
);
753 ObjectStore::Transaction t
;
754 t
.create_collection(cid
, 0);
755 cerr
<< "Creating collection " << cid
<< std::endl
;
756 r
= apply_transaction(store
, &osr
, std::move(t
));
760 bool exists
= store
->exists(cid
, hoid
);
761 ASSERT_TRUE(!exists
);
763 ObjectStore::Transaction t
;
765 cerr
<< "Creating object " << hoid
<< std::endl
;
766 r
= apply_transaction(store
, &osr
, std::move(t
));
769 exists
= store
->exists(cid
, hoid
);
770 ASSERT_EQ(true, exists
);
773 ObjectStore::Transaction t
;
774 bufferlist bl
, newdata
;
776 t
.write(cid
, hoid
, 0, 5, bl
);
777 t
.write(cid
, hoid
, 10, 5, bl
);
778 cerr
<< "TwinWrite" << std::endl
;
779 r
= apply_transaction(store
, &osr
, std::move(t
));
782 r
= store
->read(cid
, hoid
, 0, 15, newdata
);
787 expected
.append_zero(5);
789 ASSERT_TRUE(bl_eq(expected
, newdata
));
792 //overwrite over the same extents
794 ObjectStore::Transaction t
;
795 bufferlist bl
, newdata
;
797 t
.write(cid
, hoid
, 0, 5, bl
);
798 t
.write(cid
, hoid
, 10, 5, bl
);
799 cerr
<< "TwinWrite" << std::endl
;
800 r
= apply_transaction(store
, &osr
, std::move(t
));
803 r
= store
->read(cid
, hoid
, 0, 15, newdata
);
808 expected
.append_zero(5);
810 ASSERT_TRUE(bl_eq(expected
, newdata
));
813 //additional write to an unused region of some blob
815 ObjectStore::Transaction t
;
816 bufferlist bl2
, newdata
;
817 bl2
.append("1234567890");
819 t
.write(cid
, hoid
, 20, bl2
.length(), bl2
);
820 cerr
<< "Append" << std::endl
;
821 r
= apply_transaction(store
, &osr
, std::move(t
));
824 r
= store
->read(cid
, hoid
, 0, 30, newdata
);
828 expected
.append("edcba");
829 expected
.append_zero(5);
830 expected
.append("edcba");
831 expected
.append_zero(5);
832 expected
.append(bl2
);
834 ASSERT_TRUE(bl_eq(expected
, newdata
));
837 //additional write to an unused region of some blob and partial owerite over existing extents
839 ObjectStore::Transaction t
;
840 bufferlist bl
, bl2
, bl3
, newdata
;
842 bl2
.append("1234567890");
845 t
.write(cid
, hoid
, 30, bl2
.length(), bl2
);
846 t
.write(cid
, hoid
, 1, bl
.length(), bl
);
847 t
.write(cid
, hoid
, 13, bl3
.length(), bl3
);
848 cerr
<< "TripleWrite" << std::endl
;
849 r
= apply_transaction(store
, &osr
, std::move(t
));
852 r
= store
->read(cid
, hoid
, 0, 40, newdata
);
856 expected
.append("eDCBa");
857 expected
.append_zero(5);
858 expected
.append("edcBA");
859 expected
.append_zero(5);
860 expected
.append(bl2
);
861 expected
.append(bl2
);
863 ASSERT_TRUE(bl_eq(expected
, newdata
));
868 void doCompressionTest( boost::scoped_ptr
<ObjectStore
>& store
)
870 ObjectStore::Sequencer
osr("test");
873 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
876 r
= store
->read(cid
, hoid
, 0, 5, in
);
877 ASSERT_EQ(-ENOENT
, r
);
880 ObjectStore::Transaction t
;
881 t
.create_collection(cid
, 0);
882 cerr
<< "Creating collection " << cid
<< std::endl
;
883 r
= apply_transaction(store
, &osr
, std::move(t
));
887 bool exists
= store
->exists(cid
, hoid
);
888 ASSERT_TRUE(!exists
);
890 ObjectStore::Transaction t
;
892 cerr
<< "Creating object " << hoid
<< std::endl
;
893 r
= apply_transaction(store
, &osr
, std::move(t
));
896 exists
= store
->exists(cid
, hoid
);
897 ASSERT_EQ(true, exists
);
900 data
.resize(0x10000 * 4);
901 for(size_t i
= 0;i
< data
.size(); i
++)
904 ObjectStore::Transaction t
;
905 bufferlist bl
, newdata
;
907 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
908 cerr
<< "CompressibleData (4xAU) Write" << std::endl
;
909 r
= apply_transaction(store
, &osr
, std::move(t
));
912 r
= store
->read(cid
, hoid
, 0, data
.size() , newdata
);
914 ASSERT_EQ(r
, (int)data
.size());
917 expected
.append(data
);
918 ASSERT_TRUE(bl_eq(expected
, newdata
));
921 r
= store
->read(cid
, hoid
, 0, 711 , newdata
);
925 expected
.append(data
.substr(0,711));
926 ASSERT_TRUE(bl_eq(expected
, newdata
));
929 r
= store
->read(cid
, hoid
, 0xf00f, data
.size(), newdata
);
930 ASSERT_EQ(r
, int(data
.size() - 0xf00f) );
933 expected
.append(data
.substr(0xf00f));
934 ASSERT_TRUE(bl_eq(expected
, newdata
));
937 struct store_statfs_t statfs
;
938 int r
= store
->statfs(&statfs
);
940 ASSERT_EQ(statfs
.stored
, (unsigned)data
.size());
941 ASSERT_LE(statfs
.compressed
, (unsigned)data
.size());
942 ASSERT_EQ(statfs
.compressed_original
, (unsigned)data
.size());
943 ASSERT_LE(statfs
.compressed_allocated
, (unsigned)data
.size());
947 data2
.resize(0x10000 * 4 - 0x9000);
948 for(size_t i
= 0;i
< data2
.size(); i
++)
949 data2
[i
] = (i
+1) / 256;
951 ObjectStore::Transaction t
;
952 bufferlist bl
, newdata
;
954 t
.write(cid
, hoid
, 0x8000, bl
.length(), bl
);
955 cerr
<< "CompressibleData partial overwrite" << std::endl
;
956 r
= apply_transaction(store
, &osr
, std::move(t
));
959 r
= store
->read(cid
, hoid
, 0, 0x10000, newdata
);
960 ASSERT_EQ(r
, (int)0x10000);
963 expected
.append(data
.substr(0, 0x8000));
964 expected
.append(data2
.substr(0, 0x8000));
965 ASSERT_TRUE(bl_eq(expected
, newdata
));
968 r
= store
->read(cid
, hoid
, 0x9000, 711 , newdata
);
972 expected
.append(data2
.substr(0x1000,711));
973 ASSERT_TRUE(bl_eq(expected
, newdata
));
976 r
= store
->read(cid
, hoid
, 0x0, 0x40000, newdata
);
977 ASSERT_EQ(r
, int(0x40000) );
980 expected
.append(data
.substr(0, 0x8000));
981 expected
.append(data2
.substr(0, 0x37000));
982 expected
.append(data
.substr(0x3f000, 0x1000));
983 ASSERT_TRUE(bl_eq(expected
, newdata
));
986 data2
.resize(0x3f000);
987 for(size_t i
= 0;i
< data2
.size(); i
++)
988 data2
[i
] = (i
+2) / 256;
990 ObjectStore::Transaction t
;
991 bufferlist bl
, newdata
;
993 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
994 cerr
<< "CompressibleData partial overwrite, two extents overlapped, single one to be removed" << std::endl
;
995 r
= apply_transaction(store
, &osr
, std::move(t
));
998 r
= store
->read(cid
, hoid
, 0, 0x3e000 - 1, newdata
);
999 ASSERT_EQ(r
, (int)0x3e000 - 1);
1001 bufferlist expected
;
1002 expected
.append(data2
.substr(0, 0x3e000 - 1));
1003 ASSERT_TRUE(bl_eq(expected
, newdata
));
1006 r
= store
->read(cid
, hoid
, 0x3e000-1, 0x2001, newdata
);
1007 ASSERT_EQ(r
, 0x2001);
1009 bufferlist expected
;
1010 expected
.append(data2
.substr(0x3e000-1, 0x1001));
1011 expected
.append(data
.substr(0x3f000, 0x1000));
1012 ASSERT_TRUE(bl_eq(expected
, newdata
));
1015 r
= store
->read(cid
, hoid
, 0x0, 0x40000, newdata
);
1016 ASSERT_EQ(r
, int(0x40000) );
1018 bufferlist expected
;
1019 expected
.append(data2
.substr(0, 0x3f000));
1020 expected
.append(data
.substr(0x3f000, 0x1000));
1021 ASSERT_TRUE(bl_eq(expected
, newdata
));
1024 data
.resize(0x1001);
1025 for(size_t i
= 0;i
< data
.size(); i
++)
1026 data
[i
] = (i
+3) / 256;
1028 ObjectStore::Transaction t
;
1029 bufferlist bl
, newdata
;
1031 t
.write(cid
, hoid
, 0x3f000-1, bl
.length(), bl
);
1032 cerr
<< "Small chunk partial overwrite, two extents overlapped, single one to be removed" << std::endl
;
1033 r
= apply_transaction(store
, &osr
, std::move(t
));
1036 r
= store
->read(cid
, hoid
, 0x3e000, 0x2000, newdata
);
1037 ASSERT_EQ(r
, (int)0x2000);
1039 bufferlist expected
;
1040 expected
.append(data2
.substr(0x3e000, 0x1000 - 1));
1041 expected
.append(data
.substr(0, 0x1001));
1042 ASSERT_TRUE(bl_eq(expected
, newdata
));
1046 ObjectStore::Transaction t
;
1047 t
.remove(cid
, hoid
);
1048 cerr
<< "Cleaning object" << std::endl
;
1049 r
= apply_transaction(store
, &osr
, std::move(t
));
1053 EXPECT_EQ(store
->umount(), 0);
1054 EXPECT_EQ(store
->mount(), 0);
1055 auto orig_min_blob_size
= g_conf
->bluestore_compression_min_blob_size
;
1057 g_conf
->set_val("bluestore_compression_min_blob_size", "262144");
1058 g_ceph_context
->_conf
->apply_changes(NULL
);
1059 data
.resize(0x10000*6);
1061 for(size_t i
= 0;i
< data
.size(); i
++)
1063 ObjectStore::Transaction t
;
1064 bufferlist bl
, newdata
;
1066 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
1067 cerr
<< "CompressibleData large blob" << std::endl
;
1068 r
= apply_transaction(store
, &osr
, std::move(t
));
1072 EXPECT_EQ(store
->umount(), 0);
1073 EXPECT_EQ(store
->mount(), 0);
1076 ObjectStore::Transaction t
;
1077 t
.remove(cid
, hoid
);
1078 t
.remove_collection(cid
);
1079 cerr
<< "Cleaning" << std::endl
;
1080 r
= apply_transaction(store
, &osr
, std::move(t
));
1083 g_conf
->set_val("bluestore_compression_min_blob_size", stringify(orig_min_blob_size
));
1084 g_ceph_context
->_conf
->apply_changes(NULL
);
1087 TEST_P(StoreTest
, CompressionTest
) {
1088 if (string(GetParam()) != "bluestore")
1091 g_conf
->set_val("bluestore_compression_algorithm", "snappy");
1092 g_conf
->set_val("bluestore_compression_mode", "force");
1094 g_ceph_context
->_conf
->apply_changes(NULL
);
1096 doCompressionTest(store
);
1098 g_conf
->set_val("bluestore_compression_algorithm", "zlib");
1099 g_conf
->set_val("bluestore_compression_mode", "force");
1100 g_ceph_context
->_conf
->apply_changes(NULL
);
1102 doCompressionTest(store
);
1104 g_conf
->set_val("bluestore_compression_algorithm", "snappy");
1105 g_conf
->set_val("bluestore_compression_mode", "none");
1106 g_ceph_context
->_conf
->apply_changes(NULL
);
1109 TEST_P(StoreTest
, SimpleObjectTest
) {
1110 ObjectStore::Sequencer
osr("test");
1113 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
1116 r
= store
->read(cid
, hoid
, 0, 5, in
);
1117 ASSERT_EQ(-ENOENT
, r
);
1120 ObjectStore::Transaction t
;
1121 t
.create_collection(cid
, 0);
1122 cerr
<< "Creating collection " << cid
<< std::endl
;
1123 r
= apply_transaction(store
, &osr
, std::move(t
));
1127 bool exists
= store
->exists(cid
, hoid
);
1128 ASSERT_TRUE(!exists
);
1130 ObjectStore::Transaction t
;
1132 cerr
<< "Creating object " << hoid
<< std::endl
;
1133 r
= apply_transaction(store
, &osr
, std::move(t
));
1136 exists
= store
->exists(cid
, hoid
);
1137 ASSERT_EQ(true, exists
);
1140 ObjectStore::Transaction t
;
1141 t
.remove(cid
, hoid
);
1143 cerr
<< "Remove then create" << std::endl
;
1144 r
= apply_transaction(store
, &osr
, std::move(t
));
1148 ObjectStore::Transaction t
;
1149 bufferlist bl
, orig
;
1152 t
.remove(cid
, hoid
);
1153 t
.write(cid
, hoid
, 0, 5, bl
);
1154 cerr
<< "Remove then create" << std::endl
;
1155 r
= apply_transaction(store
, &osr
, std::move(t
));
1159 r
= store
->read(cid
, hoid
, 0, 5, in
);
1161 ASSERT_TRUE(bl_eq(orig
, in
));
1164 ObjectStore::Transaction t
;
1169 t
.write(cid
, hoid
, 5, 5, bl
);
1170 cerr
<< "Append" << std::endl
;
1171 r
= apply_transaction(store
, &osr
, std::move(t
));
1175 r
= store
->read(cid
, hoid
, 0, 10, in
);
1177 ASSERT_TRUE(bl_eq(exp
, in
));
1180 ObjectStore::Transaction t
;
1182 bl
.append("abcdeabcde");
1184 t
.write(cid
, hoid
, 0, 10, bl
);
1185 cerr
<< "Full overwrite" << std::endl
;
1186 r
= apply_transaction(store
, &osr
, std::move(t
));
1190 r
= store
->read(cid
, hoid
, 0, 10, in
);
1192 ASSERT_TRUE(bl_eq(exp
, in
));
1195 ObjectStore::Transaction t
;
1198 t
.write(cid
, hoid
, 3, 5, bl
);
1199 cerr
<< "Partial overwrite" << std::endl
;
1200 r
= apply_transaction(store
, &osr
, std::move(t
));
1204 exp
.append("abcabcdede");
1205 r
= store
->read(cid
, hoid
, 0, 10, in
);
1208 ASSERT_TRUE(bl_eq(exp
, in
));
1212 ObjectStore::Transaction t
;
1215 t
.truncate(cid
, hoid
, 0);
1216 t
.write(cid
, hoid
, 5, 5, bl
);
1217 cerr
<< "Truncate + hole" << std::endl
;
1218 r
= apply_transaction(store
, &osr
, std::move(t
));
1222 ObjectStore::Transaction t
;
1225 t
.write(cid
, hoid
, 0, 5, bl
);
1226 cerr
<< "Reverse fill-in" << std::endl
;
1227 r
= apply_transaction(store
, &osr
, std::move(t
));
1232 exp
.append("abcdefghij");
1233 r
= store
->read(cid
, hoid
, 0, 10, in
);
1236 ASSERT_TRUE(bl_eq(exp
, in
));
1239 ObjectStore::Transaction t
;
1241 bl
.append("abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234");
1242 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
1243 cerr
<< "larger overwrite" << std::endl
;
1244 r
= apply_transaction(store
, &osr
, std::move(t
));
1248 r
= store
->read(cid
, hoid
, 0, bl
.length(), in
);
1249 ASSERT_EQ((int)bl
.length(), r
);
1251 ASSERT_TRUE(bl_eq(bl
, in
));
1255 bl
.append("abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234");
1257 //test: offset=len=0 mean read all data
1259 r
= store
->read(cid
, hoid
, 0, 0, in
);
1260 ASSERT_EQ((int)bl
.length(), r
);
1262 ASSERT_TRUE(bl_eq(bl
, in
));
1265 //verifying unaligned csums
1266 std::string
s1("1"), s2(0x1000, '2'), s3("00");
1268 ObjectStore::Transaction t
;
1272 t
.truncate(cid
, hoid
, 0);
1273 t
.write(cid
, hoid
, 0x1000-1, bl
.length(), bl
);
1274 cerr
<< "Write unaligned csum, stage 1" << std::endl
;
1275 r
= apply_transaction(store
, &osr
, std::move(t
));
1279 bufferlist in
, exp1
, exp2
, exp3
;
1283 r
= store
->read(cid
, hoid
, 0x1000-1, 1, in
);
1285 ASSERT_TRUE(bl_eq(exp1
, in
));
1287 r
= store
->read(cid
, hoid
, 0x1000, 0x1000, in
);
1288 ASSERT_EQ(0x1000, r
);
1289 ASSERT_TRUE(bl_eq(exp2
, in
));
1292 ObjectStore::Transaction t
;
1295 t
.write(cid
, hoid
, 1, bl
.length(), bl
);
1296 cerr
<< "Write unaligned csum, stage 2" << std::endl
;
1297 r
= apply_transaction(store
, &osr
, std::move(t
));
1301 r
= store
->read(cid
, hoid
, 1, 2, in
);
1303 ASSERT_TRUE(bl_eq(exp3
, in
));
1305 r
= store
->read(cid
, hoid
, 0x1000-1, 1, in
);
1307 ASSERT_TRUE(bl_eq(exp1
, in
));
1309 r
= store
->read(cid
, hoid
, 0x1000, 0x1000, in
);
1310 ASSERT_EQ(0x1000, r
);
1311 ASSERT_TRUE(bl_eq(exp2
, in
));
1316 ObjectStore::Transaction t
;
1317 t
.remove(cid
, hoid
);
1318 t
.remove_collection(cid
);
1319 cerr
<< "Cleaning" << std::endl
;
1320 r
= apply_transaction(store
, &osr
, std::move(t
));
1325 #if defined(HAVE_LIBAIO)
1326 TEST_P(StoreTestSpecificAUSize
, BluestoreStatFSTest
) {
1327 if(string(GetParam()) != "bluestore")
1329 StartDeferred(65536);
1330 g_conf
->set_val("bluestore_compression_mode", "force");
1332 // just a big number to disble gc
1333 g_conf
->set_val("bluestore_gc_enable_total_threshold", "100000");
1334 g_conf
->apply_changes(NULL
);
1337 ObjectStore::Sequencer
osr("test");
1339 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
1340 ghobject_t hoid2
= hoid
;
1341 hoid2
.hobj
.snap
= 1;
1344 r
= store
->read(cid
, hoid
, 0, 5, in
);
1345 ASSERT_EQ(-ENOENT
, r
);
1348 ObjectStore::Transaction t
;
1349 t
.create_collection(cid
, 0);
1350 cerr
<< "Creating collection " << cid
<< std::endl
;
1351 r
= apply_transaction(store
, &osr
, std::move(t
));
1355 bool exists
= store
->exists(cid
, hoid
);
1356 ASSERT_TRUE(!exists
);
1358 ObjectStore::Transaction t
;
1360 cerr
<< "Creating object " << hoid
<< std::endl
;
1361 r
= apply_transaction(store
, &osr
, std::move(t
));
1364 exists
= store
->exists(cid
, hoid
);
1365 ASSERT_EQ(true, exists
);
1368 struct store_statfs_t statfs
;
1369 int r
= store
->statfs(&statfs
);
1371 ASSERT_EQ( 0u, statfs
.allocated
);
1372 ASSERT_EQ( 0u, statfs
.stored
);
1373 ASSERT_EQ(g_conf
->bluestore_block_size
, statfs
.total
);
1374 ASSERT_TRUE(statfs
.available
> 0u && statfs
.available
< g_conf
->bluestore_block_size
);
1376 EXPECT_EQ(store
->umount(), 0);
1377 EXPECT_EQ(store
->mount(), 0);
1380 ObjectStore::Transaction t
;
1383 t
.write(cid
, hoid
, 0, 5, bl
);
1384 cerr
<< "Append 5 bytes" << std::endl
;
1385 r
= apply_transaction(store
, &osr
, std::move(t
));
1388 struct store_statfs_t statfs
;
1389 int r
= store
->statfs(&statfs
);
1391 ASSERT_EQ(5, statfs
.stored
);
1392 ASSERT_EQ(0x10000, statfs
.allocated
);
1393 ASSERT_EQ(0, statfs
.compressed
);
1394 ASSERT_EQ(0, statfs
.compressed_original
);
1395 ASSERT_EQ(0, statfs
.compressed_allocated
);
1397 EXPECT_EQ(store
->umount(), 0);
1398 EXPECT_EQ(store
->mount(), 0);
1401 ObjectStore::Transaction t
;
1402 std::string
s(0x30000, 'a');
1405 t
.write(cid
, hoid
, 0x10000, bl
.length(), bl
);
1406 cerr
<< "Append 0x30000 compressible bytes" << std::endl
;
1407 r
= apply_transaction(store
, &osr
, std::move(t
));
1410 struct store_statfs_t statfs
;
1411 int r
= store
->statfs(&statfs
);
1413 ASSERT_EQ(0x30005, statfs
.stored
);
1414 ASSERT_EQ(0x30000, statfs
.allocated
);
1415 ASSERT_LE(statfs
.compressed
, 0x10000);
1416 ASSERT_EQ(0x20000, statfs
.compressed_original
);
1417 ASSERT_EQ(statfs
.compressed_allocated
, 0x10000);
1419 EXPECT_EQ(store
->umount(), 0);
1420 EXPECT_EQ(store
->mount(), 0);
1423 ObjectStore::Transaction t
;
1424 t
.zero(cid
, hoid
, 1, 3);
1425 t
.zero(cid
, hoid
, 0x20000, 9);
1426 cerr
<< "Punch hole at 1~3, 0x20000~9" << std::endl
;
1427 r
= apply_transaction(store
, &osr
, std::move(t
));
1430 struct store_statfs_t statfs
;
1431 int r
= store
->statfs(&statfs
);
1433 ASSERT_EQ(0x30005 - 3 - 9, statfs
.stored
);
1434 ASSERT_EQ(0x30000, statfs
.allocated
);
1435 ASSERT_LE(statfs
.compressed
, 0x10000);
1436 ASSERT_EQ(0x20000 - 9, statfs
.compressed_original
);
1437 ASSERT_EQ(statfs
.compressed_allocated
, 0x10000);
1439 EXPECT_EQ(store
->umount(), 0);
1440 EXPECT_EQ(store
->mount(), 0);
1443 ObjectStore::Transaction t
;
1444 std::string
s(0x1000, 'b');
1447 t
.write(cid
, hoid
, 1, bl
.length(), bl
);
1448 t
.write(cid
, hoid
, 0x10001, bl
.length(), bl
);
1449 cerr
<< "Overwrite first and second(compressible) extents" << std::endl
;
1450 r
= apply_transaction(store
, &osr
, std::move(t
));
1453 struct store_statfs_t statfs
;
1454 int r
= store
->statfs(&statfs
);
1456 ASSERT_EQ(0x30001 - 9 + 0x1000, statfs
.stored
);
1457 ASSERT_EQ(0x40000, statfs
.allocated
);
1458 ASSERT_LE(statfs
.compressed
, 0x10000);
1459 ASSERT_EQ(0x20000 - 9 - 0x1000, statfs
.compressed_original
);
1460 ASSERT_EQ(statfs
.compressed_allocated
, 0x10000);
1462 EXPECT_EQ(store
->umount(), 0);
1463 EXPECT_EQ(store
->mount(), 0);
1466 ObjectStore::Transaction t
;
1467 std::string
s(0x10000, 'c');
1470 t
.write(cid
, hoid
, 0x10000, bl
.length(), bl
);
1471 t
.write(cid
, hoid
, 0x20000, bl
.length(), bl
);
1472 t
.write(cid
, hoid
, 0x30000, bl
.length(), bl
);
1473 cerr
<< "Overwrite compressed extent with 3 uncompressible ones" << std::endl
;
1474 r
= apply_transaction(store
, &osr
, std::move(t
));
1477 struct store_statfs_t statfs
;
1478 int r
= store
->statfs(&statfs
);
1480 ASSERT_EQ(0x30000 + 0x1001, statfs
.stored
);
1481 ASSERT_EQ(0x40000, statfs
.allocated
);
1482 ASSERT_LE(statfs
.compressed
, 0);
1483 ASSERT_EQ(0, statfs
.compressed_original
);
1484 ASSERT_EQ(0, statfs
.compressed_allocated
);
1486 EXPECT_EQ(store
->umount(), 0);
1487 EXPECT_EQ(store
->mount(), 0);
1490 ObjectStore::Transaction t
;
1491 t
.zero(cid
, hoid
, 0, 0x40000);
1492 cerr
<< "Zero object" << std::endl
;
1493 r
= apply_transaction(store
, &osr
, std::move(t
));
1495 struct store_statfs_t statfs
;
1496 int r
= store
->statfs(&statfs
);
1498 ASSERT_EQ(0u, statfs
.allocated
);
1499 ASSERT_EQ(0u, statfs
.stored
);
1500 ASSERT_EQ(0u, statfs
.compressed_original
);
1501 ASSERT_EQ(0u, statfs
.compressed
);
1502 ASSERT_EQ(0u, statfs
.compressed_allocated
);
1504 EXPECT_EQ(store
->umount(), 0);
1505 EXPECT_EQ(store
->mount(), 0);
1508 ObjectStore::Transaction t
;
1509 std::string
s(0x10000, 'c');
1514 bl
.append(s
.substr(0, 0x10000-2));
1515 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
1516 cerr
<< "Yet another compressible write" << std::endl
;
1517 r
= apply_transaction(store
, &osr
, std::move(t
));
1519 struct store_statfs_t statfs
;
1520 r
= store
->statfs(&statfs
);
1522 ASSERT_EQ(0x40000 - 2, statfs
.stored
);
1523 ASSERT_EQ(0x30000, statfs
.allocated
);
1524 ASSERT_LE(statfs
.compressed
, 0x10000);
1525 ASSERT_EQ(0x20000, statfs
.compressed_original
);
1526 ASSERT_EQ(0x10000, statfs
.compressed_allocated
);
1528 EXPECT_EQ(store
->umount(), 0);
1529 EXPECT_EQ(store
->mount(), 0);
1532 struct store_statfs_t statfs
;
1533 r
= store
->statfs(&statfs
);
1536 ObjectStore::Transaction t
;
1537 t
.clone(cid
, hoid
, hoid2
);
1538 cerr
<< "Clone compressed objecte" << std::endl
;
1539 r
= apply_transaction(store
, &osr
, std::move(t
));
1541 struct store_statfs_t statfs2
;
1542 r
= store
->statfs(&statfs2
);
1544 ASSERT_GT(statfs2
.stored
, statfs
.stored
);
1545 ASSERT_EQ(statfs2
.allocated
, statfs
.allocated
);
1546 ASSERT_GT(statfs2
.compressed
, statfs
.compressed
);
1547 ASSERT_GT(statfs2
.compressed_original
, statfs
.compressed_original
);
1548 ASSERT_EQ(statfs2
.compressed_allocated
, statfs
.compressed_allocated
);
1552 ObjectStore::Transaction t
;
1553 t
.remove(cid
, hoid
);
1554 t
.remove(cid
, hoid2
);
1555 t
.remove_collection(cid
);
1556 cerr
<< "Cleaning" << std::endl
;
1557 r
= apply_transaction(store
, &osr
, std::move(t
));
1560 struct store_statfs_t statfs
;
1561 r
= store
->statfs(&statfs
);
1563 ASSERT_EQ( 0u, statfs
.allocated
);
1564 ASSERT_EQ( 0u, statfs
.stored
);
1565 ASSERT_EQ( 0u, statfs
.compressed_original
);
1566 ASSERT_EQ( 0u, statfs
.compressed
);
1567 ASSERT_EQ( 0u, statfs
.compressed_allocated
);
1569 g_conf
->set_val("bluestore_gc_enable_total_threshold", "0");
1570 g_conf
->set_val("bluestore_compression_mode", "none");
1571 g_ceph_context
->_conf
->apply_changes(NULL
);
1574 TEST_P(StoreTestSpecificAUSize
, BluestoreFragmentedBlobTest
) {
1575 if(string(GetParam()) != "bluestore")
1577 StartDeferred(0x10000);
1579 ObjectStore::Sequencer
osr("test");
1582 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
1584 ObjectStore::Transaction t
;
1585 t
.create_collection(cid
, 0);
1586 cerr
<< "Creating collection " << cid
<< std::endl
;
1587 r
= apply_transaction(store
, &osr
, std::move(t
));
1591 bool exists
= store
->exists(cid
, hoid
);
1592 ASSERT_TRUE(!exists
);
1594 ObjectStore::Transaction t
;
1596 cerr
<< "Creating object " << hoid
<< std::endl
;
1597 r
= apply_transaction(store
, &osr
, std::move(t
));
1600 exists
= store
->exists(cid
, hoid
);
1601 ASSERT_EQ(true, exists
);
1604 struct store_statfs_t statfs
;
1605 int r
= store
->statfs(&statfs
);
1607 ASSERT_EQ(g_conf
->bluestore_block_size
, statfs
.total
);
1608 ASSERT_EQ(0u, statfs
.allocated
);
1609 ASSERT_EQ(0u, statfs
.stored
);
1610 ASSERT_TRUE(statfs
.available
> 0u && statfs
.available
< g_conf
->bluestore_block_size
);
1613 data
.resize(0x10000 * 3);
1615 ObjectStore::Transaction t
;
1616 for(size_t i
= 0;i
< data
.size(); i
++)
1617 data
[i
] = i
/ 256 + 1;
1618 bufferlist bl
, newdata
;
1620 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
1621 t
.zero(cid
, hoid
, 0x10000, 0x10000);
1622 cerr
<< "Append 3*0x10000 bytes and punch a hole 0x10000~10000" << std::endl
;
1623 r
= apply_transaction(store
, &osr
, std::move(t
));
1626 struct store_statfs_t statfs
;
1627 int r
= store
->statfs(&statfs
);
1629 ASSERT_EQ(0x20000, statfs
.stored
);
1630 ASSERT_EQ(0x20000, statfs
.allocated
);
1632 r
= store
->read(cid
, hoid
, 0, data
.size(), newdata
);
1633 ASSERT_EQ(r
, (int)data
.size());
1635 bufferlist expected
;
1636 expected
.append(data
.substr(0, 0x10000));
1637 expected
.append(string(0x10000, 0));
1638 expected
.append(data
.substr(0x20000, 0x10000));
1639 ASSERT_TRUE(bl_eq(expected
, newdata
));
1643 r
= store
->read(cid
, hoid
, 1, data
.size()-2, newdata
);
1644 ASSERT_EQ(r
, (int)data
.size()-2);
1646 bufferlist expected
;
1647 expected
.append(data
.substr(1, 0x10000-1));
1648 expected
.append(string(0x10000, 0));
1649 expected
.append(data
.substr(0x20000, 0x10000 - 1));
1650 ASSERT_TRUE(bl_eq(expected
, newdata
));
1655 EXPECT_EQ(store
->umount(), 0);
1656 EXPECT_EQ(store
->mount(), 0);
1659 ObjectStore::Transaction t
;
1660 std::string
data2(3, 'b');
1661 bufferlist bl
, newdata
;
1663 t
.write(cid
, hoid
, 0x20000, bl
.length(), bl
);
1664 cerr
<< "Write 3 bytes after the hole" << std::endl
;
1665 r
= apply_transaction(store
, &osr
, std::move(t
));
1668 struct store_statfs_t statfs
;
1669 int r
= store
->statfs(&statfs
);
1671 ASSERT_EQ(0x20000, statfs
.allocated
);
1672 ASSERT_EQ(0x20000, statfs
.stored
);
1674 r
= store
->read(cid
, hoid
, 0x20000-1, 21, newdata
);
1675 ASSERT_EQ(r
, (int)21);
1677 bufferlist expected
;
1678 expected
.append(string(0x1, 0));
1679 expected
.append(string(data2
));
1680 expected
.append(data
.substr(0x20003, 21-4));
1681 ASSERT_TRUE(bl_eq(expected
, newdata
));
1686 EXPECT_EQ(store
->umount(), 0);
1687 EXPECT_EQ(store
->mount(), 0);
1690 ObjectStore::Transaction t
;
1691 std::string
data2(3, 'a');
1692 bufferlist bl
, newdata
;
1694 t
.write(cid
, hoid
, 0x10000+1, bl
.length(), bl
);
1695 cerr
<< "Write 3 bytes to the hole" << std::endl
;
1696 r
= apply_transaction(store
, &osr
, std::move(t
));
1699 struct store_statfs_t statfs
;
1700 int r
= store
->statfs(&statfs
);
1702 ASSERT_EQ(0x30000, statfs
.allocated
);
1703 ASSERT_EQ(0x20003, statfs
.stored
);
1705 r
= store
->read(cid
, hoid
, 0x10000-1, 0x10000+22, newdata
);
1706 ASSERT_EQ(r
, (int)0x10000+22);
1708 bufferlist expected
;
1709 expected
.append(data
.substr(0x10000-1, 1));
1710 expected
.append(string(0x1, 0));
1711 expected
.append(data2
);
1712 expected
.append(string(0x10000-4, 0));
1713 expected
.append(string(0x3, 'b'));
1714 expected
.append(data
.substr(0x20004, 21-3));
1715 ASSERT_TRUE(bl_eq(expected
, newdata
));
1720 ObjectStore::Transaction t
;
1721 bufferlist bl
, newdata
;
1722 bl
.append(string(0x30000, 'c'));
1723 t
.write(cid
, hoid
, 0, 0x30000, bl
);
1724 t
.zero(cid
, hoid
, 0, 0x10000);
1725 t
.zero(cid
, hoid
, 0x20000, 0x10000);
1726 cerr
<< "Rewrite an object and create two holes at the begining and the end" << std::endl
;
1727 r
= apply_transaction(store
, &osr
, std::move(t
));
1730 struct store_statfs_t statfs
;
1731 int r
= store
->statfs(&statfs
);
1733 ASSERT_EQ(0x10000, statfs
.allocated
);
1734 ASSERT_EQ(0x10000, statfs
.stored
);
1736 r
= store
->read(cid
, hoid
, 0, 0x30000, newdata
);
1737 ASSERT_EQ(r
, (int)0x30000);
1739 bufferlist expected
;
1740 expected
.append(string(0x10000, 0));
1741 expected
.append(string(0x10000, 'c'));
1742 expected
.append(string(0x10000, 0));
1743 ASSERT_TRUE(bl_eq(expected
, newdata
));
1749 EXPECT_EQ(store
->umount(), 0);
1750 EXPECT_EQ(store
->mount(), 0);
1753 ObjectStore::Transaction t
;
1754 t
.remove(cid
, hoid
);
1755 t
.remove_collection(cid
);
1756 cerr
<< "Cleaning" << std::endl
;
1757 r
= apply_transaction(store
, &osr
, std::move(t
));
1760 struct store_statfs_t statfs
;
1761 r
= store
->statfs(&statfs
);
1763 ASSERT_EQ( 0u, statfs
.allocated
);
1764 ASSERT_EQ( 0u, statfs
.stored
);
1765 ASSERT_EQ( 0u, statfs
.compressed_original
);
1766 ASSERT_EQ( 0u, statfs
.compressed
);
1767 ASSERT_EQ( 0u, statfs
.compressed_allocated
);
1772 TEST_P(StoreTest
, ManySmallWrite
) {
1773 ObjectStore::Sequencer
osr("test");
1776 ghobject_t
a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
1777 ghobject_t
b(hobject_t(sobject_t("Object 2", CEPH_NOSNAP
)));
1779 ObjectStore::Transaction t
;
1780 t
.create_collection(cid
, 0);
1781 cerr
<< "Creating collection " << cid
<< std::endl
;
1782 r
= apply_transaction(store
, &osr
, std::move(t
));
1789 for (int i
=0; i
<100; ++i
) {
1790 ObjectStore::Transaction t
;
1791 t
.write(cid
, a
, i
*4096, 4096, bl
, 0);
1792 r
= apply_transaction(store
, &osr
, std::move(t
));
1795 for (int i
=0; i
<100; ++i
) {
1796 ObjectStore::Transaction t
;
1797 t
.write(cid
, b
, (rand() % 1024)*4096, 4096, bl
, 0);
1798 r
= apply_transaction(store
, &osr
, std::move(t
));
1802 ObjectStore::Transaction t
;
1805 t
.remove_collection(cid
);
1806 cerr
<< "Cleaning" << std::endl
;
1807 r
= apply_transaction(store
, &osr
, std::move(t
));
1812 TEST_P(StoreTest
, MultiSmallWriteSameBlock
) {
1813 ObjectStore::Sequencer
osr("test");
1816 ghobject_t
a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
1818 ObjectStore::Transaction t
;
1819 t
.create_collection(cid
, 0);
1820 cerr
<< "Creating collection " << cid
<< std::endl
;
1821 r
= apply_transaction(store
, &osr
, std::move(t
));
1827 // touch same block in both same transaction, tls, and pipelined txns
1829 ObjectStore::Transaction t
, u
;
1830 t
.write(cid
, a
, 0, 5, bl
, 0);
1831 t
.write(cid
, a
, 5, 5, bl
, 0);
1832 t
.write(cid
, a
, 4094, 5, bl
, 0);
1833 t
.write(cid
, a
, 9000, 5, bl
, 0);
1834 u
.write(cid
, a
, 10, 5, bl
, 0);
1835 u
.write(cid
, a
, 7000, 5, bl
, 0);
1836 vector
<ObjectStore::Transaction
> v
= {t
, u
};
1837 store
->queue_transactions(&osr
, v
, nullptr, &c
);
1840 ObjectStore::Transaction t
, u
;
1841 t
.write(cid
, a
, 40, 5, bl
, 0);
1842 t
.write(cid
, a
, 45, 5, bl
, 0);
1843 t
.write(cid
, a
, 4094, 5, bl
, 0);
1844 t
.write(cid
, a
, 6000, 5, bl
, 0);
1845 u
.write(cid
, a
, 610, 5, bl
, 0);
1846 u
.write(cid
, a
, 11000, 5, bl
, 0);
1847 vector
<ObjectStore::Transaction
> v
= {t
, u
};
1848 store
->queue_transactions(&osr
, v
, nullptr, &d
);
1854 r
= store
->read(cid
, a
, 0, 16000, bl2
);
1858 ObjectStore::Transaction t
;
1860 t
.remove_collection(cid
);
1861 cerr
<< "Cleaning" << std::endl
;
1862 r
= apply_transaction(store
, &osr
, std::move(t
));
1867 TEST_P(StoreTest
, SmallSkipFront
) {
1868 ObjectStore::Sequencer
osr("test");
1871 ghobject_t
a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
1873 ObjectStore::Transaction t
;
1874 t
.create_collection(cid
, 0);
1875 cerr
<< "Creating collection " << cid
<< std::endl
;
1876 r
= apply_transaction(store
, &osr
, std::move(t
));
1880 ObjectStore::Transaction t
;
1882 t
.truncate(cid
, a
, 3000);
1883 r
= apply_transaction(store
, &osr
, std::move(t
));
1889 memset(bp
.c_str(), 1, 4096);
1891 ObjectStore::Transaction t
;
1892 t
.write(cid
, a
, 4096, 4096, bl
);
1893 r
= apply_transaction(store
, &osr
, std::move(t
));
1898 ASSERT_EQ(8192, store
->read(cid
, a
, 0, 8192, bl
));
1899 for (unsigned i
=0; i
<4096; ++i
)
1900 ASSERT_EQ(0, bl
[i
]);
1901 for (unsigned i
=4096; i
<8192; ++i
)
1902 ASSERT_EQ(1, bl
[i
]);
1905 ObjectStore::Transaction t
;
1907 t
.remove_collection(cid
);
1908 cerr
<< "Cleaning" << std::endl
;
1909 r
= apply_transaction(store
, &osr
, std::move(t
));
1914 TEST_P(StoreTest
, AppendDeferredVsTailCache
) {
1915 ObjectStore::Sequencer
osr("test");
1918 ghobject_t
a(hobject_t(sobject_t("fooo", CEPH_NOSNAP
)));
1920 ObjectStore::Transaction t
;
1921 t
.create_collection(cid
, 0);
1922 cerr
<< "Creating collection " << cid
<< std::endl
;
1923 r
= store
->apply_transaction(&osr
, std::move(t
));
1926 unsigned min_alloc
= g_conf
->bluestore_min_alloc_size
;
1927 g_conf
->set_val("bluestore_inject_deferred_apply_delay", "1.0");
1928 g_ceph_context
->_conf
->apply_changes(NULL
);
1929 unsigned size
= min_alloc
/ 3;
1930 bufferptr
bpa(size
);
1931 memset(bpa
.c_str(), 1, bpa
.length());
1935 ObjectStore::Transaction t
;
1936 t
.write(cid
, a
, 0, bla
.length(), bla
, 0);
1937 r
= store
->apply_transaction(&osr
, std::move(t
));
1941 // force cached tail to clear ...
1943 int r
= store
->umount();
1949 bufferptr
bpb(size
);
1950 memset(bpb
.c_str(), 2, bpb
.length());
1954 ObjectStore::Transaction t
;
1955 t
.write(cid
, a
, bla
.length(), blb
.length(), blb
, 0);
1956 r
= store
->apply_transaction(&osr
, std::move(t
));
1959 bufferptr
bpc(size
);
1960 memset(bpc
.c_str(), 3, bpc
.length());
1964 ObjectStore::Transaction t
;
1965 t
.write(cid
, a
, bla
.length() + blb
.length(), blc
.length(), blc
, 0);
1966 r
= store
->apply_transaction(&osr
, std::move(t
));
1975 ASSERT_EQ((int)final
.length(),
1976 store
->read(cid
, a
, 0, final
.length(), actual
));
1977 ASSERT_TRUE(bl_eq(final
, actual
));
1980 ObjectStore::Transaction t
;
1982 t
.remove_collection(cid
);
1983 cerr
<< "Cleaning" << std::endl
;
1984 r
= store
->apply_transaction(&osr
, std::move(t
));
1987 g_conf
->set_val("bluestore_inject_deferred_apply_delay", "0");
1988 g_ceph_context
->_conf
->apply_changes(NULL
);
1991 TEST_P(StoreTest
, AppendZeroTrailingSharedBlock
) {
1992 ObjectStore::Sequencer
osr("test");
1995 ghobject_t
a(hobject_t(sobject_t("fooo", CEPH_NOSNAP
)));
1999 ObjectStore::Transaction t
;
2000 t
.create_collection(cid
, 0);
2001 cerr
<< "Creating collection " << cid
<< std::endl
;
2002 r
= store
->apply_transaction(&osr
, std::move(t
));
2005 unsigned min_alloc
= g_conf
->bluestore_min_alloc_size
;
2006 unsigned size
= min_alloc
/ 3;
2007 bufferptr
bpa(size
);
2008 memset(bpa
.c_str(), 1, bpa
.length());
2011 // make sure there is some trailing gunk in the last block
2015 bt
.append("BADBADBADBAD");
2016 ObjectStore::Transaction t
;
2017 t
.write(cid
, a
, 0, bt
.length(), bt
, 0);
2018 r
= store
->apply_transaction(&osr
, std::move(t
));
2022 ObjectStore::Transaction t
;
2023 t
.truncate(cid
, a
, size
);
2024 r
= store
->apply_transaction(&osr
, std::move(t
));
2030 ObjectStore::Transaction t
;
2032 r
= store
->apply_transaction(&osr
, std::move(t
));
2036 // append with implicit zeroing
2037 bufferptr
bpb(size
);
2038 memset(bpb
.c_str(), 2, bpb
.length());
2042 ObjectStore::Transaction t
;
2043 t
.write(cid
, a
, min_alloc
* 3, blb
.length(), blb
, 0);
2044 r
= store
->apply_transaction(&osr
, std::move(t
));
2050 zeros
.append_zero(min_alloc
* 3 - size
);
2051 final
.append(zeros
);
2055 ASSERT_EQ((int)final
.length(),
2056 store
->read(cid
, a
, 0, final
.length(), actual
));
2057 final
.hexdump(cout
);
2058 actual
.hexdump(cout
);
2059 ASSERT_TRUE(bl_eq(final
, actual
));
2062 ObjectStore::Transaction t
;
2065 t
.remove_collection(cid
);
2066 cerr
<< "Cleaning" << std::endl
;
2067 r
= store
->apply_transaction(&osr
, std::move(t
));
2072 TEST_P(StoreTest
, SmallSequentialUnaligned
) {
2073 ObjectStore::Sequencer
osr("test");
2076 ghobject_t
a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
2078 ObjectStore::Transaction t
;
2079 t
.create_collection(cid
, 0);
2080 cerr
<< "Creating collection " << cid
<< std::endl
;
2081 r
= apply_transaction(store
, &osr
, std::move(t
));
2089 for (int i
=0; i
<1000; ++i
) {
2090 ObjectStore::Transaction t
;
2091 t
.write(cid
, a
, i
*len
, len
, bl
, 0);
2092 r
= apply_transaction(store
, &osr
, std::move(t
));
2096 ObjectStore::Transaction t
;
2098 t
.remove_collection(cid
);
2099 cerr
<< "Cleaning" << std::endl
;
2100 r
= apply_transaction(store
, &osr
, std::move(t
));
2105 TEST_P(StoreTest
, ManyBigWrite
) {
2106 ObjectStore::Sequencer
osr("test");
2109 ghobject_t
a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
2110 ghobject_t
b(hobject_t(sobject_t("Object 2", CEPH_NOSNAP
)));
2112 ObjectStore::Transaction t
;
2113 t
.create_collection(cid
, 0);
2114 cerr
<< "Creating collection " << cid
<< std::endl
;
2115 r
= apply_transaction(store
, &osr
, std::move(t
));
2119 bufferptr
bp(4 * 1048576);
2122 for (int i
=0; i
<10; ++i
) {
2123 ObjectStore::Transaction t
;
2124 t
.write(cid
, a
, i
*4*1048586, 4*1048576, bl
, 0);
2125 r
= apply_transaction(store
, &osr
, std::move(t
));
2129 for (int i
=0; i
<10; ++i
) {
2130 ObjectStore::Transaction t
;
2131 t
.write(cid
, b
, (rand() % 256)*4*1048576, 4*1048576, bl
, 0);
2132 r
= apply_transaction(store
, &osr
, std::move(t
));
2136 for (int i
=0; i
<10; ++i
) {
2137 ObjectStore::Transaction t
;
2138 t
.write(cid
, b
, (rand() % (256*4096))*1024, 4*1048576, bl
, 0);
2139 r
= apply_transaction(store
, &osr
, std::move(t
));
2143 for (int i
=0; i
<10; ++i
) {
2144 ObjectStore::Transaction t
;
2145 t
.zero(cid
, b
, (rand() % (256*4096))*1024, 16*1048576);
2146 r
= apply_transaction(store
, &osr
, std::move(t
));
2150 ObjectStore::Transaction t
;
2153 t
.remove_collection(cid
);
2154 cerr
<< "Cleaning" << std::endl
;
2155 r
= apply_transaction(store
, &osr
, std::move(t
));
2160 TEST_P(StoreTest
, BigWriteBigZero
) {
2161 ObjectStore::Sequencer
osr("test");
2164 ghobject_t
a(hobject_t(sobject_t("foo", CEPH_NOSNAP
)));
2166 ObjectStore::Transaction t
;
2167 t
.create_collection(cid
, 0);
2168 r
= apply_transaction(store
, &osr
, std::move(t
));
2172 bufferptr
bp(1048576);
2173 memset(bp
.c_str(), 'b', bp
.length());
2177 memset(sp
.c_str(), 's', sp
.length());
2180 ObjectStore::Transaction t
;
2181 t
.write(cid
, a
, 0, bl
.length(), bl
);
2182 r
= apply_transaction(store
, &osr
, std::move(t
));
2186 ObjectStore::Transaction t
;
2187 t
.zero(cid
, a
, bl
.length() / 4, bl
.length() / 2);
2188 r
= apply_transaction(store
, &osr
, std::move(t
));
2192 ObjectStore::Transaction t
;
2193 t
.write(cid
, a
, bl
.length() / 2, s
.length(), s
);
2194 r
= apply_transaction(store
, &osr
, std::move(t
));
2198 ObjectStore::Transaction t
;
2200 t
.remove_collection(cid
);
2201 r
= apply_transaction(store
, &osr
, std::move(t
));
2206 TEST_P(StoreTest
, MiscFragmentTests
) {
2207 ObjectStore::Sequencer
osr("test");
2210 ghobject_t
a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
2212 ObjectStore::Transaction t
;
2213 t
.create_collection(cid
, 0);
2214 cerr
<< "Creating collection " << cid
<< std::endl
;
2215 r
= apply_transaction(store
, &osr
, std::move(t
));
2219 bufferptr
bp(524288);
2223 ObjectStore::Transaction t
;
2224 t
.write(cid
, a
, 0, 524288, bl
, 0);
2225 r
= apply_transaction(store
, &osr
, std::move(t
));
2229 ObjectStore::Transaction t
;
2230 t
.write(cid
, a
, 1048576, 524288, bl
, 0);
2231 r
= apply_transaction(store
, &osr
, std::move(t
));
2236 int r
= store
->read(cid
, a
, 524288 + 131072, 1024, inbl
);
2238 ASSERT_EQ(inbl
.length(), 1024u);
2239 ASSERT_TRUE(inbl
.is_zero());
2242 ObjectStore::Transaction t
;
2243 t
.write(cid
, a
, 1048576 - 4096, 524288, bl
, 0);
2244 r
= apply_transaction(store
, &osr
, std::move(t
));
2248 ObjectStore::Transaction t
;
2250 t
.remove_collection(cid
);
2251 cerr
<< "Cleaning" << std::endl
;
2252 r
= apply_transaction(store
, &osr
, std::move(t
));
2258 TEST_P(StoreTest
, ZeroLengthWrite
) {
2259 ObjectStore::Sequencer
osr("test");
2262 ghobject_t
hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP
)));
2264 ObjectStore::Transaction t
;
2265 t
.create_collection(cid
, 0);
2267 r
= apply_transaction(store
, &osr
, std::move(t
));
2271 ObjectStore::Transaction t
;
2273 t
.write(cid
, hoid
, 1048576, 0, empty
);
2274 r
= apply_transaction(store
, &osr
, std::move(t
));
2278 r
= store
->stat(cid
, hoid
, &stat
);
2280 ASSERT_EQ(0, stat
.st_size
);
2283 r
= store
->read(cid
, hoid
, 0, 1048576, newdata
);
2287 TEST_P(StoreTest
, SimpleAttrTest
) {
2288 ObjectStore::Sequencer
osr("test");
2291 ghobject_t
hoid(hobject_t(sobject_t("attr object 1", CEPH_NOSNAP
)));
2292 bufferlist val
, val2
;
2293 val
.append("value");
2294 val
.append("value2");
2297 map
<string
,bufferptr
> aset
;
2298 r
= store
->getattr(cid
, hoid
, "nofoo", bp
);
2299 ASSERT_EQ(-ENOENT
, r
);
2300 r
= store
->getattrs(cid
, hoid
, aset
);
2301 ASSERT_EQ(-ENOENT
, r
);
2304 ObjectStore::Transaction t
;
2305 t
.create_collection(cid
, 0);
2306 r
= apply_transaction(store
, &osr
, std::move(t
));
2311 int r
= store
->collection_empty(cid
, &empty
);
2317 r
= store
->getattr(cid
, hoid
, "nofoo", bp
);
2318 ASSERT_EQ(-ENOENT
, r
);
2321 ObjectStore::Transaction t
;
2323 t
.setattr(cid
, hoid
, "foo", val
);
2324 t
.setattr(cid
, hoid
, "bar", val2
);
2325 r
= apply_transaction(store
, &osr
, std::move(t
));
2330 int r
= store
->collection_empty(cid
, &empty
);
2332 ASSERT_TRUE(!empty
);
2336 r
= store
->getattr(cid
, hoid
, "nofoo", bp
);
2337 ASSERT_EQ(-ENODATA
, r
);
2339 r
= store
->getattr(cid
, hoid
, "foo", bp
);
2343 ASSERT_TRUE(bl_eq(val
, bl
));
2345 map
<string
,bufferptr
> bm
;
2346 r
= store
->getattrs(cid
, hoid
, bm
);
2351 ObjectStore::Transaction t
;
2352 t
.remove(cid
, hoid
);
2353 t
.remove_collection(cid
);
2354 r
= apply_transaction(store
, &osr
, std::move(t
));
2359 TEST_P(StoreTest
, SimpleListTest
) {
2360 ObjectStore::Sequencer
osr("test");
2362 coll_t
cid(spg_t(pg_t(0, 1), shard_id_t(1)));
2364 ObjectStore::Transaction t
;
2365 t
.create_collection(cid
, 0);
2366 cerr
<< "Creating collection " << cid
<< std::endl
;
2367 r
= apply_transaction(store
, &osr
, std::move(t
));
2370 set
<ghobject_t
> all
;
2372 ObjectStore::Transaction t
;
2373 for (int i
=0; i
<200; ++i
) {
2374 string
name("object_");
2375 name
+= stringify(i
);
2376 ghobject_t
hoid(hobject_t(sobject_t(name
, CEPH_NOSNAP
)),
2377 ghobject_t::NO_GEN
, shard_id_t(1));
2381 cerr
<< "Creating object " << hoid
<< std::endl
;
2383 r
= apply_transaction(store
, &osr
, std::move(t
));
2387 set
<ghobject_t
> saw
;
2388 vector
<ghobject_t
> objects
;
2389 ghobject_t next
, current
;
2390 while (!next
.is_max()) {
2391 int r
= store
->collection_list(cid
, current
, ghobject_t::get_max(),
2395 ASSERT_TRUE(sorted(objects
));
2396 cout
<< " got " << objects
.size() << " next " << next
<< std::endl
;
2397 for (vector
<ghobject_t
>::iterator p
= objects
.begin(); p
!= objects
.end();
2399 if (saw
.count(*p
)) {
2400 cout
<< "got DUP " << *p
<< std::endl
;
2402 //cout << "got new " << *p << std::endl;
2409 ASSERT_EQ(saw
.size(), all
.size());
2410 ASSERT_EQ(saw
, all
);
2413 ObjectStore::Transaction t
;
2414 for (set
<ghobject_t
>::iterator p
= all
.begin(); p
!= all
.end(); ++p
)
2416 t
.remove_collection(cid
);
2417 cerr
<< "Cleaning" << std::endl
;
2418 r
= apply_transaction(store
, &osr
, std::move(t
));
2423 TEST_P(StoreTest
, ListEndTest
) {
2424 ObjectStore::Sequencer
osr("test");
2426 coll_t
cid(spg_t(pg_t(0, 1), shard_id_t(1)));
2428 ObjectStore::Transaction t
;
2429 t
.create_collection(cid
, 0);
2430 cerr
<< "Creating collection " << cid
<< std::endl
;
2431 r
= apply_transaction(store
, &osr
, std::move(t
));
2434 set
<ghobject_t
> all
;
2436 ObjectStore::Transaction t
;
2437 for (int i
=0; i
<200; ++i
) {
2438 string
name("object_");
2439 name
+= stringify(i
);
2440 ghobject_t
hoid(hobject_t(sobject_t(name
, CEPH_NOSNAP
)),
2441 ghobject_t::NO_GEN
, shard_id_t(1));
2445 cerr
<< "Creating object " << hoid
<< std::endl
;
2447 r
= apply_transaction(store
, &osr
, std::move(t
));
2451 ghobject_t
end(hobject_t(sobject_t("object_100", CEPH_NOSNAP
)),
2452 ghobject_t::NO_GEN
, shard_id_t(1));
2454 vector
<ghobject_t
> objects
;
2456 int r
= store
->collection_list(cid
, ghobject_t(), end
, 500,
2459 for (auto &p
: objects
) {
2464 ObjectStore::Transaction t
;
2465 for (set
<ghobject_t
>::iterator p
= all
.begin(); p
!= all
.end(); ++p
)
2467 t
.remove_collection(cid
);
2468 cerr
<< "Cleaning" << std::endl
;
2469 r
= apply_transaction(store
, &osr
, std::move(t
));
2474 TEST_P(StoreTest
, Sort
) {
2476 hobject_t
a(sobject_t("a", CEPH_NOSNAP
));
2489 ghobject_t
a(hobject_t(sobject_t("a", CEPH_NOSNAP
)));
2490 ghobject_t
b(hobject_t(sobject_t("b", CEPH_NOSNAP
)));
2502 TEST_P(StoreTest
, MultipoolListTest
) {
2503 ObjectStore::Sequencer
osr("test");
2506 coll_t cid
= coll_t(spg_t(pg_t(0, poolid
), shard_id_t::NO_SHARD
));
2508 ObjectStore::Transaction t
;
2509 t
.create_collection(cid
, 0);
2510 cerr
<< "Creating collection " << cid
<< std::endl
;
2511 r
= apply_transaction(store
, &osr
, std::move(t
));
2514 set
<ghobject_t
> all
, saw
;
2516 ObjectStore::Transaction t
;
2517 for (int i
=0; i
<200; ++i
) {
2518 string
name("object_");
2519 name
+= stringify(i
);
2520 ghobject_t
hoid(hobject_t(sobject_t(name
, CEPH_NOSNAP
)));
2522 hoid
.hobj
.pool
= -2 - poolid
;
2524 hoid
.hobj
.pool
= poolid
;
2527 cerr
<< "Creating object " << hoid
<< std::endl
;
2529 r
= apply_transaction(store
, &osr
, std::move(t
));
2533 vector
<ghobject_t
> objects
;
2534 ghobject_t next
, current
;
2535 while (!next
.is_max()) {
2536 int r
= store
->collection_list(cid
, current
, ghobject_t::get_max(), 50,
2539 cout
<< " got " << objects
.size() << " next " << next
<< std::endl
;
2540 for (vector
<ghobject_t
>::iterator p
= objects
.begin(); p
!= objects
.end();
2547 ASSERT_EQ(saw
, all
);
2550 ObjectStore::Transaction t
;
2551 for (set
<ghobject_t
>::iterator p
= all
.begin(); p
!= all
.end(); ++p
)
2553 t
.remove_collection(cid
);
2554 cerr
<< "Cleaning" << std::endl
;
2555 r
= apply_transaction(store
, &osr
, std::move(t
));
2560 TEST_P(StoreTest
, SimpleCloneTest
) {
2561 ObjectStore::Sequencer
osr("test");
2565 ObjectStore::Transaction t
;
2566 t
.create_collection(cid
, 0);
2567 cerr
<< "Creating collection " << cid
<< std::endl
;
2568 r
= apply_transaction(store
, &osr
, std::move(t
));
2571 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
),
2572 "key", 123, -1, ""));
2573 bufferlist small
, large
, xlarge
, newdata
, attr
;
2574 small
.append("small");
2575 large
.append("large");
2576 xlarge
.append("xlarge");
2578 ObjectStore::Transaction t
;
2580 t
.setattr(cid
, hoid
, "attr1", small
);
2581 t
.setattr(cid
, hoid
, "attr2", large
);
2582 t
.setattr(cid
, hoid
, "attr3", xlarge
);
2583 t
.write(cid
, hoid
, 0, small
.length(), small
);
2584 t
.write(cid
, hoid
, 10, small
.length(), small
);
2585 cerr
<< "Creating object and set attr " << hoid
<< std::endl
;
2586 r
= apply_transaction(store
, &osr
, std::move(t
));
2590 ghobject_t
hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP
),
2591 "key", 123, -1, ""));
2592 ghobject_t
hoid3(hobject_t(sobject_t("Object 3", CEPH_NOSNAP
)));
2594 ObjectStore::Transaction t
;
2595 t
.clone(cid
, hoid
, hoid2
);
2596 t
.setattr(cid
, hoid2
, "attr2", small
);
2597 t
.rmattr(cid
, hoid2
, "attr1");
2598 t
.write(cid
, hoid
, 10, large
.length(), large
);
2599 t
.setattr(cid
, hoid
, "attr1", large
);
2600 t
.setattr(cid
, hoid
, "attr2", small
);
2601 cerr
<< "Clone object and rm attr" << std::endl
;
2602 r
= apply_transaction(store
, &osr
, std::move(t
));
2605 r
= store
->read(cid
, hoid
, 10, 5, newdata
);
2607 ASSERT_TRUE(bl_eq(large
, newdata
));
2610 r
= store
->read(cid
, hoid
, 0, 5, newdata
);
2612 ASSERT_TRUE(bl_eq(small
, newdata
));
2615 r
= store
->read(cid
, hoid2
, 10, 5, newdata
);
2617 ASSERT_TRUE(bl_eq(small
, newdata
));
2619 r
= store
->getattr(cid
, hoid2
, "attr2", attr
);
2621 ASSERT_TRUE(bl_eq(small
, attr
));
2624 r
= store
->getattr(cid
, hoid2
, "attr3", attr
);
2626 ASSERT_TRUE(bl_eq(xlarge
, attr
));
2629 r
= store
->getattr(cid
, hoid
, "attr1", attr
);
2631 ASSERT_TRUE(bl_eq(large
, attr
));
2634 ObjectStore::Transaction t
;
2635 t
.remove(cid
, hoid
);
2636 t
.remove(cid
, hoid2
);
2637 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2642 memset(p
.c_str(), 1, p
.length());
2646 ObjectStore::Transaction t
;
2647 t
.write(cid
, hoid
, 0, pl
.length(), pl
);
2648 t
.clone(cid
, hoid
, hoid2
);
2650 memset(a
.c_str(), 2, a
.length());
2654 t
.write(cid
, hoid
, pl
.length(), a
.length(), al
);
2655 r
= apply_transaction(store
, &osr
, std::move(t
));
2658 ASSERT_EQ((int)final
.length(),
2659 store
->read(cid
, hoid
, 0, final
.length(), rl
));
2660 ASSERT_TRUE(bl_eq(rl
, final
));
2663 ObjectStore::Transaction t
;
2664 t
.remove(cid
, hoid
);
2665 t
.remove(cid
, hoid2
);
2666 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2671 memset(p
.c_str(), 111, p
.length());
2675 ObjectStore::Transaction t
;
2676 t
.write(cid
, hoid
, 0, pl
.length(), pl
);
2677 t
.clone(cid
, hoid
, hoid2
);
2682 memset(a
.c_str(), 112, a
.length());
2686 t
.write(cid
, hoid
, pl
.length() + z
.length(), a
.length(), al
);
2687 r
= apply_transaction(store
, &osr
, std::move(t
));
2690 ASSERT_EQ((int)final
.length(),
2691 store
->read(cid
, hoid
, 0, final
.length(), rl
));
2692 ASSERT_TRUE(bl_eq(rl
, final
));
2695 ObjectStore::Transaction t
;
2696 t
.remove(cid
, hoid
);
2697 t
.remove(cid
, hoid2
);
2698 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2703 memset(p
.c_str(), 5, p
.length());
2707 ObjectStore::Transaction t
;
2708 t
.write(cid
, hoid
, 0, pl
.length(), pl
);
2709 t
.clone(cid
, hoid
, hoid2
);
2714 memset(a
.c_str(), 6, a
.length());
2718 t
.write(cid
, hoid
, 17000, a
.length(), al
);
2719 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2721 ASSERT_EQ((int)final
.length(),
2722 store
->read(cid
, hoid
, 0, final
.length(), rl
));
2723 /*cout << "expected:\n";
2724 final.hexdump(cout);
2727 ASSERT_TRUE(bl_eq(rl
, final
));
2730 ObjectStore::Transaction t
;
2731 t
.remove(cid
, hoid
);
2732 t
.remove(cid
, hoid2
);
2733 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2736 bufferptr
p(1048576);
2737 memset(p
.c_str(), 3, p
.length());
2740 ObjectStore::Transaction t
;
2741 t
.write(cid
, hoid
, 0, pl
.length(), pl
);
2742 t
.clone(cid
, hoid
, hoid2
);
2744 memset(a
.c_str(), 4, a
.length());
2747 t
.write(cid
, hoid
, a
.length(), a
.length(), al
);
2748 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2751 final
.substr_of(pl
, 0, al
.length());
2754 end
.substr_of(pl
, al
.length()*2, pl
.length() - al
.length()*2);
2756 ASSERT_EQ((int)final
.length(),
2757 store
->read(cid
, hoid
, 0, final
.length(), rl
));
2758 /*cout << "expected:\n";
2759 final.hexdump(cout);
2762 ASSERT_TRUE(bl_eq(rl
, final
));
2765 ObjectStore::Transaction t
;
2766 t
.remove(cid
, hoid
);
2767 t
.remove(cid
, hoid2
);
2768 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2772 memset(p
.c_str(), 7, p
.length());
2775 ObjectStore::Transaction t
;
2776 t
.write(cid
, hoid
, 0, pl
.length(), pl
);
2777 t
.clone(cid
, hoid
, hoid2
);
2779 memset(a
.c_str(), 8, a
.length());
2782 t
.write(cid
, hoid
, 32768, a
.length(), al
);
2783 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2786 final
.substr_of(pl
, 0, 32768);
2789 end
.substr_of(pl
, final
.length(), pl
.length() - final
.length());
2791 ASSERT_EQ((int)final
.length(),
2792 store
->read(cid
, hoid
, 0, final
.length(), rl
));
2793 /*cout << "expected:\n";
2794 final.hexdump(cout);
2797 ASSERT_TRUE(bl_eq(rl
, final
));
2800 ObjectStore::Transaction t
;
2801 t
.remove(cid
, hoid
);
2802 t
.remove(cid
, hoid2
);
2803 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2807 memset(p
.c_str(), 9, p
.length());
2810 ObjectStore::Transaction t
;
2811 t
.write(cid
, hoid
, 0, pl
.length(), pl
);
2812 t
.clone(cid
, hoid
, hoid2
);
2814 memset(a
.c_str(), 10, a
.length());
2817 t
.write(cid
, hoid
, 33768, a
.length(), al
);
2818 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2821 final
.substr_of(pl
, 0, 33768);
2824 end
.substr_of(pl
, final
.length(), pl
.length() - final
.length());
2826 ASSERT_EQ((int)final
.length(),
2827 store
->read(cid
, hoid
, 0, final
.length(), rl
));
2828 /*cout << "expected:\n";
2829 final.hexdump(cout);
2832 ASSERT_TRUE(bl_eq(rl
, final
));
2835 //Unfortunately we need a workaround for filestore since EXPECT_DEATH
2836 // macro has potential issues when using /in multithread environments.
2837 //It works well for all stores but filestore for now.
2838 //A fix setting gtest_death_test_style = "threadsafe" doesn't help as well -
2839 // test app clone asserts on store folder presence.
2841 if (string(GetParam()) != "filestore") {
2842 //verify if non-empty collection is properly handled after store reload
2843 r
= store
->umount();
2848 ObjectStore::Transaction t
;
2849 t
.remove_collection(cid
);
2850 cerr
<< "Invalid rm coll" << std::endl
;
2851 PrCtl unset_dumpable
;
2852 EXPECT_DEATH(apply_transaction(store
, &osr
, std::move(t
)), ".*Directory not empty.*");
2855 ObjectStore::Transaction t
;
2856 t
.touch(cid
, hoid3
); //new record in db
2857 r
= apply_transaction(store
, &osr
, std::move(t
));
2860 //See comment above for "filestore" check explanation.
2861 if (string(GetParam()) != "filestore") {
2862 ObjectStore::Transaction t
;
2863 //verify if non-empty collection is properly handled when there are some pending removes and live records in db
2864 cerr
<< "Invalid rm coll again" << std::endl
;
2865 r
= store
->umount();
2870 t
.remove(cid
, hoid
);
2871 t
.remove(cid
, hoid2
);
2872 t
.remove_collection(cid
);
2873 PrCtl unset_dumpable
;
2874 EXPECT_DEATH(apply_transaction(store
, &osr
, std::move(t
)), ".*Directory not empty.*");
2877 ObjectStore::Transaction t
;
2878 t
.remove(cid
, hoid
);
2879 t
.remove(cid
, hoid2
);
2880 t
.remove(cid
, hoid3
);
2881 t
.remove_collection(cid
);
2882 cerr
<< "Cleaning" << std::endl
;
2883 r
= apply_transaction(store
, &osr
, std::move(t
));
2888 TEST_P(StoreTest
, OmapSimple
) {
2889 ObjectStore::Sequencer
osr("test");
2893 ObjectStore::Transaction t
;
2894 t
.create_collection(cid
, 0);
2895 cerr
<< "Creating collection " << cid
<< std::endl
;
2896 r
= apply_transaction(store
, &osr
, std::move(t
));
2899 ghobject_t
hoid(hobject_t(sobject_t("omap_obj", CEPH_NOSNAP
),
2900 "key", 123, -1, ""));
2902 small
.append("small");
2903 map
<string
,bufferlist
> km
;
2905 km
["bar"].append("asdfjkasdkjdfsjkafskjsfdj");
2907 header
.append("this is a header");
2909 ObjectStore::Transaction t
;
2911 t
.omap_setkeys(cid
, hoid
, km
);
2912 t
.omap_setheader(cid
, hoid
, header
);
2913 cerr
<< "Creating object and set omap " << hoid
<< std::endl
;
2914 r
= apply_transaction(store
, &osr
, std::move(t
));
2920 map
<string
,bufferlist
> r
;
2921 store
->omap_get(cid
, hoid
, &h
, &r
);
2922 ASSERT_TRUE(bl_eq(header
, h
));
2923 ASSERT_EQ(r
.size(), km
.size());
2924 cout
<< "r: " << r
<< std::endl
;
2926 // test iterator with seek_to_first
2928 map
<string
,bufferlist
> r
;
2929 ObjectMap::ObjectMapIterator iter
= store
->get_omap_iterator(cid
, hoid
);
2930 for (iter
->seek_to_first(); iter
->valid(); iter
->next(false)) {
2931 r
[iter
->key()] = iter
->value();
2933 cout
<< "r: " << r
<< std::endl
;
2934 ASSERT_EQ(r
.size(), km
.size());
2936 // test iterator with initial lower_bound
2938 map
<string
,bufferlist
> r
;
2939 ObjectMap::ObjectMapIterator iter
= store
->get_omap_iterator(cid
, hoid
);
2940 for (iter
->lower_bound(string()); iter
->valid(); iter
->next(false)) {
2941 r
[iter
->key()] = iter
->value();
2943 cout
<< "r: " << r
<< std::endl
;
2944 ASSERT_EQ(r
.size(), km
.size());
2947 ObjectStore::Transaction t
;
2948 t
.remove(cid
, hoid
);
2949 t
.remove_collection(cid
);
2950 cerr
<< "Cleaning" << std::endl
;
2951 r
= apply_transaction(store
, &osr
, std::move(t
));
2956 TEST_P(StoreTest
, OmapCloneTest
) {
2957 ObjectStore::Sequencer
osr("test");
2961 ObjectStore::Transaction t
;
2962 t
.create_collection(cid
, 0);
2963 cerr
<< "Creating collection " << cid
<< std::endl
;
2964 r
= apply_transaction(store
, &osr
, std::move(t
));
2967 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
),
2968 "key", 123, -1, ""));
2970 small
.append("small");
2971 map
<string
,bufferlist
> km
;
2973 km
["bar"].append("asdfjkasdkjdfsjkafskjsfdj");
2975 header
.append("this is a header");
2977 ObjectStore::Transaction t
;
2979 t
.omap_setkeys(cid
, hoid
, km
);
2980 t
.omap_setheader(cid
, hoid
, header
);
2981 cerr
<< "Creating object and set omap " << hoid
<< std::endl
;
2982 r
= apply_transaction(store
, &osr
, std::move(t
));
2985 ghobject_t
hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP
),
2986 "key", 123, -1, ""));
2988 ObjectStore::Transaction t
;
2989 t
.clone(cid
, hoid
, hoid2
);
2990 cerr
<< "Clone object" << std::endl
;
2991 r
= apply_transaction(store
, &osr
, std::move(t
));
2995 map
<string
,bufferlist
> r
;
2997 store
->omap_get(cid
, hoid2
, &h
, &r
);
2998 ASSERT_TRUE(bl_eq(header
, h
));
2999 ASSERT_EQ(r
.size(), km
.size());
3002 ObjectStore::Transaction t
;
3003 t
.remove(cid
, hoid
);
3004 t
.remove(cid
, hoid2
);
3005 t
.remove_collection(cid
);
3006 cerr
<< "Cleaning" << std::endl
;
3007 r
= apply_transaction(store
, &osr
, std::move(t
));
3012 TEST_P(StoreTest
, SimpleCloneRangeTest
) {
3013 ObjectStore::Sequencer
osr("test");
3017 ObjectStore::Transaction t
;
3018 t
.create_collection(cid
, 0);
3019 cerr
<< "Creating collection " << cid
<< std::endl
;
3020 r
= apply_transaction(store
, &osr
, std::move(t
));
3023 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
3024 hoid
.hobj
.pool
= -1;
3025 bufferlist small
, newdata
;
3026 small
.append("small");
3028 ObjectStore::Transaction t
;
3029 t
.write(cid
, hoid
, 10, 5, small
);
3030 cerr
<< "Creating object and write bl " << hoid
<< std::endl
;
3031 r
= apply_transaction(store
, &osr
, std::move(t
));
3034 ghobject_t
hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP
)));
3035 hoid2
.hobj
.pool
= -1;
3037 ObjectStore::Transaction t
;
3038 t
.clone_range(cid
, hoid
, hoid2
, 10, 5, 10);
3039 cerr
<< "Clone range object" << std::endl
;
3040 r
= apply_transaction(store
, &osr
, std::move(t
));
3042 r
= store
->read(cid
, hoid2
, 10, 5, newdata
);
3044 ASSERT_TRUE(bl_eq(small
, newdata
));
3047 ObjectStore::Transaction t
;
3048 t
.truncate(cid
, hoid
, 1024*1024);
3049 t
.clone_range(cid
, hoid
, hoid2
, 0, 1024*1024, 0);
3050 cerr
<< "Clone range object" << std::endl
;
3051 r
= apply_transaction(store
, &osr
, std::move(t
));
3053 struct stat stat
, stat2
;
3054 r
= store
->stat(cid
, hoid
, &stat
);
3055 r
= store
->stat(cid
, hoid2
, &stat2
);
3056 ASSERT_EQ(stat
.st_size
, stat2
.st_size
);
3057 ASSERT_EQ(1024*1024, stat2
.st_size
);
3060 ObjectStore::Transaction t
;
3061 t
.remove(cid
, hoid
);
3062 t
.remove(cid
, hoid2
);
3063 t
.remove_collection(cid
);
3064 cerr
<< "Cleaning" << std::endl
;
3065 r
= apply_transaction(store
, &osr
, std::move(t
));
3071 TEST_P(StoreTest
, SimpleObjectLongnameTest
) {
3072 ObjectStore::Sequencer
osr("test");
3076 ObjectStore::Transaction t
;
3077 t
.create_collection(cid
, 0);
3078 cerr
<< "Creating collection " << cid
<< std::endl
;
3079 r
= apply_transaction(store
, &osr
, std::move(t
));
3082 ghobject_t
hoid(hobject_t(sobject_t("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaObjectaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 1", CEPH_NOSNAP
)));
3084 ObjectStore::Transaction t
;
3086 cerr
<< "Creating object " << hoid
<< std::endl
;
3087 r
= apply_transaction(store
, &osr
, std::move(t
));
3091 ObjectStore::Transaction t
;
3092 t
.remove(cid
, hoid
);
3093 t
.remove_collection(cid
);
3094 cerr
<< "Cleaning" << std::endl
;
3095 r
= apply_transaction(store
, &osr
, std::move(t
));
3100 ghobject_t
generate_long_name(unsigned i
)
3103 name
<< "object id " << i
<< " ";
3104 for (unsigned j
= 0; j
< 500; ++j
) name
<< 'a';
3105 ghobject_t
hoid(hobject_t(sobject_t(name
.str(), CEPH_NOSNAP
)));
3106 hoid
.hobj
.set_hash(i
% 2);
3110 TEST_P(StoreTest
, LongnameSplitTest
) {
3111 ObjectStore::Sequencer
osr("test");
3115 ObjectStore::Transaction t
;
3116 t
.create_collection(cid
, 0);
3117 cerr
<< "Creating collection " << cid
<< std::endl
;
3118 r
= apply_transaction(store
, &osr
, std::move(t
));
3121 for (unsigned i
= 0; i
< 320; ++i
) {
3122 ObjectStore::Transaction t
;
3123 ghobject_t hoid
= generate_long_name(i
);
3125 cerr
<< "Creating object " << hoid
<< std::endl
;
3126 r
= apply_transaction(store
, &osr
, std::move(t
));
3130 ghobject_t test_obj
= generate_long_name(319);
3131 ghobject_t test_obj_2
= test_obj
;
3132 test_obj_2
.generation
= 0;
3134 ObjectStore::Transaction t
;
3135 // should cause a split
3136 t
.collection_move_rename(
3139 r
= apply_transaction(store
, &osr
, std::move(t
));
3143 for (unsigned i
= 0; i
< 319; ++i
) {
3144 ObjectStore::Transaction t
;
3145 ghobject_t hoid
= generate_long_name(i
);
3146 t
.remove(cid
, hoid
);
3147 cerr
<< "Removing object " << hoid
<< std::endl
;
3148 r
= apply_transaction(store
, &osr
, std::move(t
));
3152 ObjectStore::Transaction t
;
3153 t
.remove(cid
, test_obj_2
);
3154 t
.remove_collection(cid
);
3155 cerr
<< "Cleaning" << std::endl
;
3156 r
= apply_transaction(store
, &osr
, std::move(t
));
3162 TEST_P(StoreTest
, ManyObjectTest
) {
3163 ObjectStore::Sequencer
osr("test");
3164 int NUM_OBJS
= 2000;
3168 for (int i
= 0; i
< 100; ++i
) base
.append("aaaaa");
3169 set
<ghobject_t
> created
;
3171 ObjectStore::Transaction t
;
3172 t
.create_collection(cid
, 0);
3173 r
= apply_transaction(store
, &osr
, std::move(t
));
3176 for (int i
= 0; i
< NUM_OBJS
; ++i
) {
3178 cerr
<< "Object " << i
<< std::endl
;
3180 ObjectStore::Transaction t
;
3182 snprintf(buf
, sizeof(buf
), "%d", i
);
3183 ghobject_t
hoid(hobject_t(sobject_t(string(buf
) + base
, CEPH_NOSNAP
)));
3185 created
.insert(hoid
);
3186 r
= apply_transaction(store
, &osr
, std::move(t
));
3190 for (set
<ghobject_t
>::iterator i
= created
.begin();
3194 ASSERT_TRUE(!store
->stat(cid
, *i
, &buf
));
3197 set
<ghobject_t
> listed
, listed2
;
3198 vector
<ghobject_t
> objects
;
3199 r
= store
->collection_list(cid
, ghobject_t(), ghobject_t::get_max(), INT_MAX
, &objects
, 0);
3202 cerr
<< "objects.size() is " << objects
.size() << std::endl
;
3203 for (vector
<ghobject_t
> ::iterator i
= objects
.begin();
3207 ASSERT_TRUE(created
.count(*i
));
3209 ASSERT_TRUE(listed
.size() == created
.size());
3211 ghobject_t start
, next
;
3213 r
= store
->collection_list(
3215 ghobject_t::get_max(),
3216 ghobject_t::get_max(),
3222 ASSERT_TRUE(objects
.empty());
3226 ghobject_t start2
, next2
;
3228 r
= store
->collection_list(cid
, start
, ghobject_t::get_max(),
3232 ASSERT_TRUE(sorted(objects
));
3234 listed
.insert(objects
.begin(), objects
.end());
3235 if (objects
.size() < 50) {
3236 ASSERT_TRUE(next
.is_max());
3243 cerr
<< "listed.size() is " << listed
.size() << std::endl
;
3244 ASSERT_TRUE(listed
.size() == created
.size());
3245 if (listed2
.size()) {
3246 ASSERT_EQ(listed
.size(), listed2
.size());
3248 for (set
<ghobject_t
>::iterator i
= listed
.begin();
3251 ASSERT_TRUE(created
.count(*i
));
3254 for (set
<ghobject_t
>::iterator i
= created
.begin();
3257 ObjectStore::Transaction t
;
3259 r
= apply_transaction(store
, &osr
, std::move(t
));
3262 cerr
<< "cleaning up" << std::endl
;
3264 ObjectStore::Transaction t
;
3265 t
.remove_collection(cid
);
3266 r
= apply_transaction(store
, &osr
, std::move(t
));
3272 class ObjectGenerator
{
3274 virtual ghobject_t
create_object(gen_type
*gen
) = 0;
3275 virtual ~ObjectGenerator() {}
3278 class MixedGenerator
: public ObjectGenerator
{
3282 explicit MixedGenerator(int64_t p
) : seq(0), poolid(p
) {}
3283 ghobject_t
create_object(gen_type
*gen
) override
{
3285 snprintf(buf
, sizeof(buf
), "OBJ_%u", seq
);
3288 for (unsigned i
= 0; i
< 300; ++i
) {
3289 name
.push_back('a');
3295 name
, string(), rand() & 2 ? CEPH_NOSNAP
: rand(),
3296 (((seq
/ 1024) % 2) * 0xF00 ) +
3302 class SyntheticWorkloadState
{
3305 map
<string
, bufferlist
> attrs
;
3308 static const unsigned max_in_flight
= 16;
3309 static const unsigned max_objects
= 3000;
3310 static const unsigned max_attr_size
= 5;
3311 static const unsigned max_attr_name_len
= 100;
3312 static const unsigned max_attr_value_len
= 1024 * 64;
3314 unsigned write_alignment
;
3315 unsigned max_object_len
, max_write_len
;
3317 map
<ghobject_t
, Object
> contents
;
3318 set
<ghobject_t
> available_objects
;
3319 set
<ghobject_t
> in_flight_objects
;
3320 ObjectGenerator
*object_gen
;
3323 ObjectStore::Sequencer
*osr
;
3330 explicit EnterExit(const char *m
) : msg(m
) {
3331 //cout << pthread_self() << " enter " << msg << std::endl;
3334 //cout << pthread_self() << " exit " << msg << std::endl;
3338 class C_SyntheticOnReadable
: public Context
{
3340 SyntheticWorkloadState
*state
;
3342 C_SyntheticOnReadable(SyntheticWorkloadState
*state
, ghobject_t hoid
)
3343 : state(state
), hoid(hoid
) {}
3345 void finish(int r
) override
{
3346 Mutex::Locker
locker(state
->lock
);
3347 EnterExit
ee("onreadable finish");
3348 ASSERT_TRUE(state
->in_flight_objects
.count(hoid
));
3350 state
->in_flight_objects
.erase(hoid
);
3351 if (state
->contents
.count(hoid
))
3352 state
->available_objects
.insert(hoid
);
3353 --(state
->in_flight
);
3354 state
->cond
.Signal();
3357 r
= state
->store
->read(state
->cid
, hoid
, 0, state
->contents
[hoid
].data
.length(), r2
);
3358 assert(bl_eq(state
->contents
[hoid
].data
, r2
));
3359 state
->cond
.Signal();
3363 class C_SyntheticOnStash
: public Context
{
3365 SyntheticWorkloadState
*state
;
3366 ghobject_t oid
, noid
;
3368 C_SyntheticOnStash(SyntheticWorkloadState
*state
,
3369 ghobject_t oid
, ghobject_t noid
)
3370 : state(state
), oid(oid
), noid(noid
) {}
3372 void finish(int r
) override
{
3373 Mutex::Locker
locker(state
->lock
);
3374 EnterExit
ee("stash finish");
3375 ASSERT_TRUE(state
->in_flight_objects
.count(oid
));
3377 state
->in_flight_objects
.erase(oid
);
3378 if (state
->contents
.count(noid
))
3379 state
->available_objects
.insert(noid
);
3380 --(state
->in_flight
);
3382 r
= state
->store
->read(
3383 state
->cid
, noid
, 0,
3384 state
->contents
[noid
].data
.length(), r2
);
3385 assert(bl_eq(state
->contents
[noid
].data
, r2
));
3386 state
->cond
.Signal();
3390 class C_SyntheticOnClone
: public Context
{
3392 SyntheticWorkloadState
*state
;
3393 ghobject_t oid
, noid
;
3395 C_SyntheticOnClone(SyntheticWorkloadState
*state
,
3396 ghobject_t oid
, ghobject_t noid
)
3397 : state(state
), oid(oid
), noid(noid
) {}
3399 void finish(int r
) override
{
3400 Mutex::Locker
locker(state
->lock
);
3401 EnterExit
ee("clone finish");
3402 ASSERT_TRUE(state
->in_flight_objects
.count(oid
));
3404 state
->in_flight_objects
.erase(oid
);
3405 if (state
->contents
.count(oid
))
3406 state
->available_objects
.insert(oid
);
3407 if (state
->contents
.count(noid
))
3408 state
->available_objects
.insert(noid
);
3409 --(state
->in_flight
);
3411 r
= state
->store
->read(state
->cid
, noid
, 0, state
->contents
[noid
].data
.length(), r2
);
3412 assert(bl_eq(state
->contents
[noid
].data
, r2
));
3413 state
->cond
.Signal();
3417 static void filled_byte_array(bufferlist
& bl
, size_t size
)
3419 static const char alphanum
[] = "0123456789"
3420 "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
3421 "abcdefghijklmnopqrstuvwxyz";
3426 for (unsigned int i
= 0; i
< size
- 1; i
++) {
3427 // severely limit entropy so we can compress...
3428 bp
[i
] = alphanum
[rand() % 10]; //(sizeof(alphanum) - 1)];
3430 bp
[size
- 1] = '\0';
3435 SyntheticWorkloadState(ObjectStore
*store
,
3436 ObjectGenerator
*gen
,
3438 ObjectStore::Sequencer
*osr
,
3443 : cid(cid
), write_alignment(alignment
), max_object_len(max_size
),
3444 max_write_len(max_write
), in_flight(0), object_gen(gen
),
3445 rng(rng
), store(store
), osr(osr
), lock("State lock") {}
3448 ObjectStore::Transaction t
;
3449 t
.create_collection(cid
, 0);
3450 return apply_transaction(store
, osr
, std::move(t
));
3454 vector
<ghobject_t
> objects
;
3455 int r
= store
->collection_list(cid
, ghobject_t(), ghobject_t::get_max(),
3458 if (objects
.empty())
3460 ObjectStore::Transaction t
;
3461 for (vector
<ghobject_t
>::iterator p
= objects
.begin();
3462 p
!= objects
.end(); ++p
) {
3465 apply_transaction(store
, osr
, std::move(t
));
3467 ObjectStore::Transaction t
;
3468 t
.remove_collection(cid
);
3469 apply_transaction(store
, osr
, std::move(t
));
3471 void statfs(store_statfs_t
& stat
) {
3472 store
->statfs(&stat
);
3475 ghobject_t
get_uniform_random_object() {
3476 while (in_flight
>= max_in_flight
|| available_objects
.empty())
3478 boost::uniform_int
<> choose(0, available_objects
.size() - 1);
3479 int index
= choose(*rng
);
3480 set
<ghobject_t
>::iterator i
= available_objects
.begin();
3481 for ( ; index
> 0; --index
, ++i
) ;
3482 ghobject_t ret
= *i
;
3486 void wait_for_ready() {
3487 while (in_flight
>= max_in_flight
)
3491 void wait_for_done() {
3493 Mutex::Locker
locker(lock
);
3499 return (available_objects
.size() + in_flight_objects
.size()) < max_objects
;
3503 return (available_objects
.size() + in_flight_objects
.size()) > 0;
3506 unsigned get_random_alloc_hints() {
3509 boost::uniform_int
<> u(0, 3);
3512 f
|= CEPH_OSD_ALLOC_HINT_FLAG_SEQUENTIAL_WRITE
;
3515 f
|= CEPH_OSD_ALLOC_HINT_FLAG_RANDOM_WRITE
;
3520 boost::uniform_int
<> u(0, 3);
3523 f
|= CEPH_OSD_ALLOC_HINT_FLAG_SEQUENTIAL_READ
;
3526 f
|= CEPH_OSD_ALLOC_HINT_FLAG_RANDOM_READ
;
3531 // append_only, immutable
3532 boost::uniform_int
<> u(0, 4);
3536 boost::uniform_int
<> u(0, 3);
3539 f
|= CEPH_OSD_ALLOC_HINT_FLAG_SHORTLIVED
;
3542 f
|= CEPH_OSD_ALLOC_HINT_FLAG_LONGLIVED
;
3547 boost::uniform_int
<> u(0, 3);
3550 f
|= CEPH_OSD_ALLOC_HINT_FLAG_COMPRESSIBLE
;
3553 f
|= CEPH_OSD_ALLOC_HINT_FLAG_INCOMPRESSIBLE
;
3561 Mutex::Locker
locker(lock
);
3562 EnterExit
ee("touch");
3566 ghobject_t new_obj
= object_gen
->create_object(rng
);
3567 available_objects
.erase(new_obj
);
3568 ObjectStore::Transaction t
;
3569 t
.touch(cid
, new_obj
);
3570 boost::uniform_int
<> u(17, 22);
3571 boost::uniform_int
<> v(12, 17);
3572 t
.set_alloc_hint(cid
, new_obj
,
3575 get_random_alloc_hints());
3577 in_flight_objects
.insert(new_obj
);
3578 if (!contents
.count(new_obj
))
3579 contents
[new_obj
] = Object();
3580 int status
= store
->queue_transaction(osr
, std::move(t
), new C_SyntheticOnReadable(this, new_obj
));
3585 Mutex::Locker
locker(lock
);
3586 EnterExit
ee("stash");
3596 old_obj
= get_uniform_random_object();
3597 } while (--max
&& !contents
[old_obj
].data
.length());
3598 available_objects
.erase(old_obj
);
3599 ghobject_t new_obj
= old_obj
;
3600 new_obj
.generation
++;
3601 available_objects
.erase(new_obj
);
3603 ObjectStore::Transaction t
;
3604 t
.collection_move_rename(cid
, old_obj
, cid
, new_obj
);
3606 in_flight_objects
.insert(old_obj
);
3608 contents
[new_obj
].attrs
= contents
[old_obj
].attrs
;
3609 contents
[new_obj
].data
= contents
[old_obj
].data
;
3610 contents
.erase(old_obj
);
3611 int status
= store
->queue_transaction(
3613 new C_SyntheticOnStash(this, old_obj
, new_obj
));
3618 Mutex::Locker
locker(lock
);
3619 EnterExit
ee("clone");
3629 old_obj
= get_uniform_random_object();
3630 } while (--max
&& !contents
[old_obj
].data
.length());
3631 available_objects
.erase(old_obj
);
3632 ghobject_t new_obj
= object_gen
->create_object(rng
);
3633 // make the hash match
3634 new_obj
.hobj
.set_hash(old_obj
.hobj
.get_hash());
3635 available_objects
.erase(new_obj
);
3637 ObjectStore::Transaction t
;
3638 t
.clone(cid
, old_obj
, new_obj
);
3640 in_flight_objects
.insert(old_obj
);
3642 contents
[new_obj
].attrs
= contents
[old_obj
].attrs
;
3643 contents
[new_obj
].data
= contents
[old_obj
].data
;
3645 int status
= store
->queue_transaction(
3647 new C_SyntheticOnClone(this, old_obj
, new_obj
));
3652 Mutex::Locker
locker(lock
);
3653 EnterExit
ee("clone_range");
3663 old_obj
= get_uniform_random_object();
3664 } while (--max
&& !contents
[old_obj
].data
.length());
3665 bufferlist
&srcdata
= contents
[old_obj
].data
;
3666 if (srcdata
.length() == 0) {
3669 available_objects
.erase(old_obj
);
3670 ghobject_t new_obj
= get_uniform_random_object();
3671 available_objects
.erase(new_obj
);
3673 boost::uniform_int
<> u1(0, max_object_len
- max_write_len
);
3674 boost::uniform_int
<> u2(0, max_write_len
);
3675 uint64_t srcoff
= u1(*rng
);
3676 // make src and dst offsets match, since that's what the osd does
3677 uint64_t dstoff
= srcoff
; //u1(*rng);
3678 uint64_t len
= u2(*rng
);
3679 if (write_alignment
) {
3680 srcoff
= ROUND_UP_TO(srcoff
, write_alignment
);
3681 dstoff
= ROUND_UP_TO(dstoff
, write_alignment
);
3682 len
= ROUND_UP_TO(len
, write_alignment
);
3685 if (srcoff
> srcdata
.length() - 1) {
3686 srcoff
= srcdata
.length() - 1;
3688 if (srcoff
+ len
> srcdata
.length()) {
3689 len
= srcdata
.length() - srcoff
;
3692 cout
<< __func__
<< " from " << srcoff
<< "~" << len
3693 << " (size " << srcdata
.length() << ") to "
3694 << dstoff
<< "~" << len
<< std::endl
;
3696 ObjectStore::Transaction t
;
3697 t
.clone_range(cid
, old_obj
, new_obj
, srcoff
, len
, dstoff
);
3699 in_flight_objects
.insert(old_obj
);
3702 if (srcoff
< srcdata
.length()) {
3703 if (srcoff
+ len
> srcdata
.length()) {
3704 bl
.substr_of(srcdata
, srcoff
, srcdata
.length() - srcoff
);
3706 bl
.substr_of(srcdata
, srcoff
, len
);
3710 bufferlist
& dstdata
= contents
[new_obj
].data
;
3711 if (dstdata
.length() <= dstoff
) {
3712 if (bl
.length() > 0) {
3713 dstdata
.append_zero(dstoff
- dstdata
.length());
3718 assert(dstdata
.length() > dstoff
);
3719 dstdata
.copy(0, dstoff
, value
);
3721 if (value
.length() < dstdata
.length())
3722 dstdata
.copy(value
.length(),
3723 dstdata
.length() - value
.length(), value
);
3724 value
.swap(dstdata
);
3727 int status
= store
->queue_transaction(
3728 osr
, std::move(t
), new C_SyntheticOnClone(this, old_obj
, new_obj
));
3734 Mutex::Locker
locker(lock
);
3735 EnterExit
ee("write");
3740 ghobject_t new_obj
= get_uniform_random_object();
3741 available_objects
.erase(new_obj
);
3742 ObjectStore::Transaction t
;
3744 boost::uniform_int
<> u1(0, max_object_len
- max_write_len
);
3745 boost::uniform_int
<> u2(0, max_write_len
);
3746 uint64_t offset
= u1(*rng
);
3747 uint64_t len
= u2(*rng
);
3749 if (write_alignment
) {
3750 offset
= ROUND_UP_TO(offset
, write_alignment
);
3751 len
= ROUND_UP_TO(len
, write_alignment
);
3754 filled_byte_array(bl
, len
);
3756 bufferlist
& data
= contents
[new_obj
].data
;
3757 if (data
.length() <= offset
) {
3759 data
.append_zero(offset
-data
.length());
3764 assert(data
.length() > offset
);
3765 data
.copy(0, offset
, value
);
3767 if (value
.length() < data
.length())
3768 data
.copy(value
.length(),
3769 data
.length()-value
.length(), value
);
3773 t
.write(cid
, new_obj
, offset
, len
, bl
);
3775 in_flight_objects
.insert(new_obj
);
3776 int status
= store
->queue_transaction(
3777 osr
, std::move(t
), new C_SyntheticOnReadable(this, new_obj
));
3782 Mutex::Locker
locker(lock
);
3783 EnterExit
ee("truncate");
3788 ghobject_t obj
= get_uniform_random_object();
3789 available_objects
.erase(obj
);
3790 ObjectStore::Transaction t
;
3792 boost::uniform_int
<> choose(0, max_object_len
);
3793 size_t len
= choose(*rng
);
3794 if (write_alignment
) {
3795 len
= ROUND_UP_TO(len
, write_alignment
);
3798 t
.truncate(cid
, obj
, len
);
3800 in_flight_objects
.insert(obj
);
3801 bufferlist
& data
= contents
[obj
].data
;
3802 if (data
.length() <= len
) {
3803 data
.append_zero(len
- data
.length());
3806 data
.copy(0, len
, bl
);
3810 int status
= store
->queue_transaction(
3811 osr
, std::move(t
), new C_SyntheticOnReadable(this, obj
));
3816 Mutex::Locker
locker(lock
);
3817 EnterExit
ee("zero");
3822 ghobject_t new_obj
= get_uniform_random_object();
3823 available_objects
.erase(new_obj
);
3824 ObjectStore::Transaction t
;
3826 boost::uniform_int
<> u1(0, max_object_len
- max_write_len
);
3827 boost::uniform_int
<> u2(0, max_write_len
);
3828 uint64_t offset
= u1(*rng
);
3829 uint64_t len
= u2(*rng
);
3830 if (write_alignment
) {
3831 offset
= ROUND_UP_TO(offset
, write_alignment
);
3832 len
= ROUND_UP_TO(len
, write_alignment
);
3835 auto& data
= contents
[new_obj
].data
;
3836 if (data
.length() < offset
+ len
) {
3837 data
.append_zero(offset
+len
-data
.length());
3840 n
.substr_of(data
, 0, offset
);
3842 if (data
.length() > offset
+ len
)
3843 data
.copy(offset
+ len
, data
.length() - offset
- len
, n
);
3846 t
.zero(cid
, new_obj
, offset
, len
);
3848 in_flight_objects
.insert(new_obj
);
3849 int status
= store
->queue_transaction(
3850 osr
, std::move(t
), new C_SyntheticOnReadable(this, new_obj
));
3855 EnterExit
ee("read");
3856 boost::uniform_int
<> u1(0, max_object_len
/2);
3857 boost::uniform_int
<> u2(0, max_object_len
);
3858 uint64_t offset
= u1(*rng
);
3859 uint64_t len
= u2(*rng
);
3864 bufferlist expected
;
3867 Mutex::Locker
locker(lock
);
3868 EnterExit
ee("read locked");
3873 obj
= get_uniform_random_object();
3874 expected
= contents
[obj
].data
;
3876 bufferlist bl
, result
;
3877 if (0) cout
<< " obj " << obj
3878 << " size " << expected
.length()
3879 << " offset " << offset
3880 << " len " << len
<< std::endl
;
3881 r
= store
->read(cid
, obj
, offset
, len
, result
);
3882 if (offset
>= expected
.length()) {
3885 size_t max_len
= expected
.length() - offset
;
3888 assert(len
== result
.length());
3889 ASSERT_EQ(len
, result
.length());
3890 expected
.copy(offset
, len
, bl
);
3891 ASSERT_EQ(r
, (int)len
);
3892 ASSERT_TRUE(bl_eq(bl
, result
));
3897 Mutex::Locker
locker(lock
);
3898 EnterExit
ee("setattrs");
3903 ghobject_t obj
= get_uniform_random_object();
3904 available_objects
.erase(obj
);
3905 ObjectStore::Transaction t
;
3907 boost::uniform_int
<> u0(1, max_attr_size
);
3908 boost::uniform_int
<> u1(4, max_attr_name_len
);
3909 boost::uniform_int
<> u2(4, max_attr_value_len
);
3910 boost::uniform_int
<> u3(0, 100);
3911 uint64_t size
= u0(*rng
);
3913 map
<string
, bufferlist
> attrs
;
3915 for (map
<string
, bufferlist
>::iterator it
= contents
[obj
].attrs
.begin();
3916 it
!= contents
[obj
].attrs
.end(); ++it
)
3917 keys
.insert(it
->first
);
3920 bufferlist name
, value
;
3921 uint64_t get_exist
= u3(*rng
);
3922 uint64_t value_len
= u2(*rng
);
3923 filled_byte_array(value
, value_len
);
3924 if (get_exist
< 50 && keys
.size()) {
3925 set
<string
>::iterator k
= keys
.begin();
3927 contents
[obj
].attrs
[*k
] = value
;
3930 name_len
= u1(*rng
);
3931 filled_byte_array(name
, name_len
);
3932 attrs
[name
.c_str()] = value
;
3933 contents
[obj
].attrs
[name
.c_str()] = value
;
3936 t
.setattrs(cid
, obj
, attrs
);
3938 in_flight_objects
.insert(obj
);
3939 int status
= store
->queue_transaction(
3940 osr
, std::move(t
), new C_SyntheticOnReadable(this, obj
));
3945 EnterExit
ee("getattrs");
3947 map
<string
, bufferlist
> expected
;
3949 Mutex::Locker
locker(lock
);
3950 EnterExit
ee("getattrs locked");
3957 obj
= get_uniform_random_object();
3960 } while (contents
[obj
].attrs
.empty());
3961 expected
= contents
[obj
].attrs
;
3963 map
<string
, bufferlist
> attrs
;
3964 int r
= store
->getattrs(cid
, obj
, attrs
);
3965 ASSERT_TRUE(r
== 0);
3966 ASSERT_TRUE(attrs
.size() == expected
.size());
3967 for (map
<string
, bufferlist
>::iterator it
= expected
.begin();
3968 it
!= expected
.end(); ++it
) {
3969 ASSERT_TRUE(bl_eq(attrs
[it
->first
], it
->second
));
3974 EnterExit
ee("getattr");
3978 map
<string
, bufferlist
> expected
;
3980 Mutex::Locker
locker(lock
);
3981 EnterExit
ee("getattr locked");
3988 obj
= get_uniform_random_object();
3991 } while (contents
[obj
].attrs
.empty());
3992 expected
= contents
[obj
].attrs
;
3994 boost::uniform_int
<> u(0, expected
.size()-1);
3996 map
<string
, bufferlist
>::iterator it
= expected
.begin();
4003 r
= store
->getattr(cid
, obj
, it
->first
, bl
);
4005 ASSERT_TRUE(bl_eq(it
->second
, bl
));
4009 Mutex::Locker
locker(lock
);
4010 EnterExit
ee("rmattr");
4018 obj
= get_uniform_random_object();
4021 } while (contents
[obj
].attrs
.empty());
4023 boost::uniform_int
<> u(0, contents
[obj
].attrs
.size()-1);
4025 map
<string
, bufferlist
>::iterator it
= contents
[obj
].attrs
.begin();
4031 available_objects
.erase(obj
);
4032 ObjectStore::Transaction t
;
4033 t
.rmattr(cid
, obj
, it
->first
);
4035 contents
[obj
].attrs
.erase(it
->first
);
4037 in_flight_objects
.insert(obj
);
4038 int status
= store
->queue_transaction(
4039 osr
, std::move(t
), new C_SyntheticOnReadable(this, obj
));
4043 void fsck(bool deep
) {
4044 Mutex::Locker
locker(lock
);
4045 EnterExit
ee("fsck");
4049 int r
= store
->fsck(deep
);
4050 assert(r
== 0 || r
== -EOPNOTSUPP
);
4055 Mutex::Locker
locker(lock
);
4056 EnterExit
ee("scan");
4059 vector
<ghobject_t
> objects
;
4060 set
<ghobject_t
> objects_set
, objects_set2
;
4061 ghobject_t next
, current
;
4063 //cerr << "scanning..." << std::endl;
4064 int r
= store
->collection_list(cid
, current
, ghobject_t::get_max(), 100,
4067 ASSERT_TRUE(sorted(objects
));
4068 objects_set
.insert(objects
.begin(), objects
.end());
4070 if (next
.is_max()) break;
4073 if (objects_set
.size() != available_objects
.size()) {
4074 for (set
<ghobject_t
>::iterator p
= objects_set
.begin();
4075 p
!= objects_set
.end();
4077 if (available_objects
.count(*p
) == 0) {
4078 cerr
<< "+ " << *p
<< std::endl
;
4081 for (set
<ghobject_t
>::iterator p
= available_objects
.begin();
4082 p
!= available_objects
.end();
4084 if (objects_set
.count(*p
) == 0)
4085 cerr
<< "- " << *p
<< std::endl
;
4086 //cerr << " objects_set: " << objects_set << std::endl;
4087 //cerr << " available_set: " << available_objects << std::endl;
4088 assert(0 == "badness");
4091 ASSERT_EQ(objects_set
.size(), available_objects
.size());
4092 for (set
<ghobject_t
>::iterator i
= objects_set
.begin();
4093 i
!= objects_set
.end();
4095 ASSERT_GT(available_objects
.count(*i
), (unsigned)0);
4098 int r
= store
->collection_list(cid
, ghobject_t(), ghobject_t::get_max(),
4099 INT_MAX
, &objects
, 0);
4101 objects_set2
.insert(objects
.begin(), objects
.end());
4102 ASSERT_EQ(objects_set2
.size(), available_objects
.size());
4103 for (set
<ghobject_t
>::iterator i
= objects_set2
.begin();
4104 i
!= objects_set2
.end();
4106 ASSERT_GT(available_objects
.count(*i
), (unsigned)0);
4107 if (available_objects
.count(*i
) == 0) {
4108 cerr
<< "+ " << *i
<< std::endl
;
4114 EnterExit
ee("stat");
4118 Mutex::Locker
locker(lock
);
4119 EnterExit
ee("stat lock1");
4122 hoid
= get_uniform_random_object();
4123 in_flight_objects
.insert(hoid
);
4124 available_objects
.erase(hoid
);
4126 expected
= contents
[hoid
].data
.length();
4129 int r
= store
->stat(cid
, hoid
, &buf
);
4131 assert((uint64_t)buf
.st_size
== expected
);
4132 ASSERT_TRUE((uint64_t)buf
.st_size
== expected
);
4134 Mutex::Locker
locker(lock
);
4135 EnterExit
ee("stat lock2");
4138 in_flight_objects
.erase(hoid
);
4139 available_objects
.insert(hoid
);
4144 Mutex::Locker
locker(lock
);
4145 EnterExit
ee("unlink");
4148 ghobject_t to_remove
= get_uniform_random_object();
4149 ObjectStore::Transaction t
;
4150 t
.remove(cid
, to_remove
);
4152 available_objects
.erase(to_remove
);
4153 in_flight_objects
.insert(to_remove
);
4154 contents
.erase(to_remove
);
4155 int status
= store
->queue_transaction(osr
, std::move(t
), new C_SyntheticOnReadable(this, to_remove
));
4159 void print_internal_state() {
4160 Mutex::Locker
locker(lock
);
4161 cerr
<< "available_objects: " << available_objects
.size()
4162 << " in_flight_objects: " << in_flight_objects
.size()
4163 << " total objects: " << in_flight_objects
.size() + available_objects
.size()
4164 << " in_flight " << in_flight
<< std::endl
;
4169 void doSyntheticTest(boost::scoped_ptr
<ObjectStore
>& store
,
4171 uint64_t max_obj
, uint64_t max_wr
, uint64_t align
)
4173 ObjectStore::Sequencer
osr("test");
4174 MixedGenerator
gen(555);
4175 gen_type
rng(time(NULL
));
4176 coll_t
cid(spg_t(pg_t(0,555), shard_id_t::NO_SHARD
));
4178 g_ceph_context
->_conf
->set_val("bluestore_fsck_on_mount", "false");
4179 g_ceph_context
->_conf
->set_val("bluestore_fsck_on_umount", "false");
4180 g_ceph_context
->_conf
->apply_changes(NULL
);
4182 SyntheticWorkloadState
test_obj(store
.get(), &gen
, &rng
, &osr
, cid
,
4183 max_obj
, max_wr
, align
);
4185 for (int i
= 0; i
< num_ops
/10; ++i
) {
4186 if (!(i
% 500)) cerr
<< "seeding object " << i
<< std::endl
;
4189 for (int i
= 0; i
< num_ops
; ++i
) {
4191 cerr
<< "Op " << i
<< std::endl
;
4192 test_obj
.print_internal_state();
4194 boost::uniform_int
<> true_false(0, 999);
4195 int val
= true_false(rng
);
4197 test_obj
.fsck(true);
4198 } else if (val
> 997) {
4199 test_obj
.fsck(false);
4200 } else if (val
> 970) {
4202 } else if (val
> 950) {
4204 } else if (val
> 850) {
4206 } else if (val
> 800) {
4208 } else if (val
> 550) {
4210 } else if (val
> 500) {
4212 } else if (val
> 450) {
4213 test_obj
.clone_range();
4214 } else if (val
> 300) {
4216 } else if (val
> 100) {
4219 test_obj
.truncate();
4222 test_obj
.wait_for_done();
4223 test_obj
.shutdown();
4225 g_ceph_context
->_conf
->set_val("bluestore_fsck_on_mount", "true");
4226 g_ceph_context
->_conf
->set_val("bluestore_fsck_on_umount", "true");
4227 g_ceph_context
->_conf
->apply_changes(NULL
);
4230 TEST_P(StoreTest
, Synthetic
) {
4231 doSyntheticTest(store
, 10000, 400*1024, 40*1024, 0);
4235 TEST_P(StoreTestSpecificAUSize
, SyntheticMatrixSharding
) {
4236 if (string(GetParam()) != "bluestore")
4239 const char *m
[][10] = {
4240 { "bluestore_min_alloc_size", "4096", 0 }, // must be the first!
4241 { "num_ops", "50000", 0 },
4242 { "max_write", "65536", 0 },
4243 { "max_size", "262144", 0 },
4244 { "alignment", "4096", 0 },
4245 { "bluestore_max_blob_size", "65536", 0 },
4246 { "bluestore_extent_map_shard_min_size", "60", 0 },
4247 { "bluestore_extent_map_shard_max_size", "300", 0 },
4248 { "bluestore_extent_map_shard_target_size", "150", 0 },
4249 { "bluestore_default_buffered_read", "true", 0 },
4250 { "bluestore_default_buffered_write", "true", 0 },
4253 do_matrix(m
, store
, doSyntheticTest
);
4256 TEST_P(StoreTestSpecificAUSize
, ZipperPatternSharded
) {
4257 if(string(GetParam()) != "bluestore")
4259 StartDeferred(4096);
4262 ObjectStore::Sequencer
osr("test");
4264 ghobject_t
a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
4266 ObjectStore::Transaction t
;
4267 t
.create_collection(cid
, 0);
4268 cerr
<< "Creating collection " << cid
<< std::endl
;
4269 r
= apply_transaction(store
, &osr
, std::move(t
));
4277 for (int i
=0; i
<1000; ++i
) {
4278 ObjectStore::Transaction t
;
4279 t
.write(cid
, a
, i
*2*len
, len
, bl
, 0);
4280 r
= apply_transaction(store
, &osr
, std::move(t
));
4283 for (int i
=0; i
<1000; ++i
) {
4284 ObjectStore::Transaction t
;
4285 t
.write(cid
, a
, i
*2*len
+ 1, len
, bl
, 0);
4286 r
= apply_transaction(store
, &osr
, std::move(t
));
4290 ObjectStore::Transaction t
;
4292 t
.remove_collection(cid
);
4293 cerr
<< "Cleaning" << std::endl
;
4294 r
= apply_transaction(store
, &osr
, std::move(t
));
4299 TEST_P(StoreTestSpecificAUSize
, SyntheticMatrixCsumAlgorithm
) {
4300 if (string(GetParam()) != "bluestore")
4303 const char *m
[][10] = {
4304 { "bluestore_min_alloc_size", "65536", 0 }, // must be the first!
4305 { "max_write", "65536", 0 },
4306 { "max_size", "1048576", 0 },
4307 { "alignment", "16", 0 },
4308 { "bluestore_csum_type", "crc32c", "crc32c_16", "crc32c_8", "xxhash32",
4309 "xxhash64", "none", 0 },
4310 { "bluestore_default_buffered_write", "false", 0 },
4313 do_matrix(m
, store
, doSyntheticTest
);
4316 TEST_P(StoreTestSpecificAUSize
, SyntheticMatrixCsumVsCompression
) {
4317 if (string(GetParam()) != "bluestore")
4320 const char *m
[][10] = {
4321 { "bluestore_min_alloc_size", "4096", "16384", 0 }, //to be the first!
4322 { "max_write", "131072", 0 },
4323 { "max_size", "262144", 0 },
4324 { "alignment", "512", 0 },
4325 { "bluestore_compression_mode", "force", 0},
4326 { "bluestore_compression_algorithm", "snappy", "zlib", 0 },
4327 { "bluestore_csum_type", "crc32c", 0 },
4328 { "bluestore_default_buffered_read", "true", "false", 0 },
4329 { "bluestore_default_buffered_write", "true", "false", 0 },
4330 { "bluestore_sync_submit_transaction", "false", 0 },
4333 do_matrix(m
, store
, doSyntheticTest
);
4336 TEST_P(StoreTestSpecificAUSize
, SyntheticMatrixCompression
) {
4337 if (string(GetParam()) != "bluestore")
4340 const char *m
[][10] = {
4341 { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first!
4342 { "max_write", "1048576", 0 },
4343 { "max_size", "4194304", 0 },
4344 { "alignment", "65536", 0 },
4345 { "bluestore_compression_mode", "force", "aggressive", "passive", "none", 0},
4346 { "bluestore_default_buffered_write", "false", 0 },
4347 { "bluestore_sync_submit_transaction", "true", 0 },
4350 do_matrix(m
, store
, doSyntheticTest
);
4353 TEST_P(StoreTestSpecificAUSize
, SyntheticMatrixCompressionAlgorithm
) {
4354 if (string(GetParam()) != "bluestore")
4357 const char *m
[][10] = {
4358 { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first!
4359 { "max_write", "1048576", 0 },
4360 { "max_size", "4194304", 0 },
4361 { "alignment", "65536", 0 },
4362 { "bluestore_compression_algorithm", "zlib", "snappy", 0 },
4363 { "bluestore_compression_mode", "force", 0 },
4364 { "bluestore_default_buffered_write", "false", 0 },
4367 do_matrix(m
, store
, doSyntheticTest
);
4370 TEST_P(StoreTestSpecificAUSize
, SyntheticMatrixNoCsum
) {
4371 if (string(GetParam()) != "bluestore")
4374 const char *m
[][10] = {
4375 { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first!
4376 { "max_write", "65536", 0 },
4377 { "max_size", "1048576", 0 },
4378 { "alignment", "512", 0 },
4379 { "bluestore_max_blob_size", "262144", 0 },
4380 { "bluestore_compression_mode", "force", "none", 0},
4381 { "bluestore_csum_type", "none", 0},
4382 { "bluestore_default_buffered_read", "true", "false", 0 },
4383 { "bluestore_default_buffered_write", "true", 0 },
4384 { "bluestore_sync_submit_transaction", "true", "false", 0 },
4387 do_matrix(m
, store
, doSyntheticTest
);
4390 TEST_P(StoreTestSpecificAUSize
, SyntheticMatrixPreferDeferred
) {
4391 if (string(GetParam()) != "bluestore")
4394 const char *m
[][10] = {
4395 { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first!
4396 { "max_write", "65536", 0 },
4397 { "max_size", "1048576", 0 },
4398 { "alignment", "512", 0 },
4399 { "bluestore_max_blob_size", "262144", 0 },
4400 { "bluestore_compression_mode", "force", "none", 0},
4401 { "bluestore_prefer_deferred_size", "32768", "0", 0},
4404 do_matrix(m
, store
, doSyntheticTest
);
4407 TEST_P(StoreTest
, AttrSynthetic
) {
4408 ObjectStore::Sequencer
osr("test");
4409 MixedGenerator
gen(447);
4410 gen_type
rng(time(NULL
));
4411 coll_t
cid(spg_t(pg_t(0,447),shard_id_t::NO_SHARD
));
4413 SyntheticWorkloadState
test_obj(store
.get(), &gen
, &rng
, &osr
, cid
, 40*1024, 4*1024, 0);
4415 for (int i
= 0; i
< 500; ++i
) {
4416 if (!(i
% 10)) cerr
<< "seeding object " << i
<< std::endl
;
4419 for (int i
= 0; i
< 1000; ++i
) {
4421 cerr
<< "Op " << i
<< std::endl
;
4422 test_obj
.print_internal_state();
4424 boost::uniform_int
<> true_false(0, 99);
4425 int val
= true_false(rng
);
4428 } else if (val
> 93) {
4430 } else if (val
> 75) {
4432 } else if (val
> 47) {
4433 test_obj
.setattrs();
4434 } else if (val
> 45) {
4436 } else if (val
> 37) {
4438 } else if (val
> 30) {
4439 test_obj
.getattrs();
4444 test_obj
.wait_for_done();
4445 test_obj
.shutdown();
4448 TEST_P(StoreTest
, HashCollisionTest
) {
4449 ObjectStore::Sequencer
osr("test");
4450 int64_t poolid
= 11;
4451 coll_t
cid(spg_t(pg_t(0,poolid
),shard_id_t::NO_SHARD
));
4454 ObjectStore::Transaction t
;
4455 t
.create_collection(cid
, 0);
4456 r
= apply_transaction(store
, &osr
, std::move(t
));
4460 for (int i
= 0; i
< 100; ++i
) base
.append("aaaaa");
4461 set
<ghobject_t
> created
;
4462 for (int n
= 0; n
< 10; ++n
) {
4464 sprintf(nbuf
, "n%d", n
);
4465 for (int i
= 0; i
< 1000; ++i
) {
4467 sprintf(buf
, "%d", i
);
4469 cerr
<< "Object n" << n
<< " "<< i
<< std::endl
;
4471 ghobject_t
hoid(hobject_t(string(buf
) + base
, string(), CEPH_NOSNAP
, 0, poolid
, string(nbuf
)));
4473 ObjectStore::Transaction t
;
4475 r
= apply_transaction(store
, &osr
, std::move(t
));
4478 created
.insert(hoid
);
4481 vector
<ghobject_t
> objects
;
4482 r
= store
->collection_list(cid
, ghobject_t(), ghobject_t::get_max(), INT_MAX
, &objects
, 0);
4484 set
<ghobject_t
> listed(objects
.begin(), objects
.end());
4485 cerr
<< "listed.size() is " << listed
.size() << " and created.size() is " << created
.size() << std::endl
;
4486 ASSERT_TRUE(listed
.size() == created
.size());
4489 ghobject_t current
, next
;
4491 r
= store
->collection_list(cid
, current
, ghobject_t::get_max(), 60,
4494 ASSERT_TRUE(sorted(objects
));
4495 for (vector
<ghobject_t
>::iterator i
= objects
.begin();
4498 if (listed
.count(*i
))
4499 cerr
<< *i
<< " repeated" << std::endl
;
4502 if (objects
.size() < 50) {
4503 ASSERT_TRUE(next
.is_max());
4509 cerr
<< "listed.size() is " << listed
.size() << std::endl
;
4510 ASSERT_TRUE(listed
.size() == created
.size());
4511 for (set
<ghobject_t
>::iterator i
= listed
.begin();
4514 ASSERT_TRUE(created
.count(*i
));
4517 for (set
<ghobject_t
>::iterator i
= created
.begin();
4520 ObjectStore::Transaction t
;
4522 r
= apply_transaction(store
, &osr
, std::move(t
));
4525 ObjectStore::Transaction t
;
4526 t
.remove_collection(cid
);
4527 r
= apply_transaction(store
, &osr
, std::move(t
));
4531 TEST_P(StoreTest
, ScrubTest
) {
4532 ObjectStore::Sequencer
osr("test");
4533 int64_t poolid
= 111;
4534 coll_t
cid(spg_t(pg_t(0, poolid
),shard_id_t(1)));
4537 ObjectStore::Transaction t
;
4538 t
.create_collection(cid
, 0);
4539 r
= apply_transaction(store
, &osr
, std::move(t
));
4542 string base
= "aaaaa";
4543 set
<ghobject_t
> created
;
4544 for (int i
= 0; i
< 1000; ++i
) {
4546 sprintf(buf
, "%d", i
);
4548 cerr
<< "Object " << i
<< std::endl
;
4550 ghobject_t
hoid(hobject_t(string(buf
) + base
, string(), CEPH_NOSNAP
, i
,
4552 ghobject_t::NO_GEN
, shard_id_t(1));
4554 ObjectStore::Transaction t
;
4556 r
= apply_transaction(store
, &osr
, std::move(t
));
4559 created
.insert(hoid
);
4562 // Add same hobject_t but different generation
4564 ghobject_t
hoid1(hobject_t("same-object", string(), CEPH_NOSNAP
, 0, poolid
, ""),
4565 ghobject_t::NO_GEN
, shard_id_t(1));
4566 ghobject_t
hoid2(hobject_t("same-object", string(), CEPH_NOSNAP
, 0, poolid
, ""), (gen_t
)1, shard_id_t(1));
4567 ghobject_t
hoid3(hobject_t("same-object", string(), CEPH_NOSNAP
, 0, poolid
, ""), (gen_t
)2, shard_id_t(1));
4568 ObjectStore::Transaction t
;
4569 t
.touch(cid
, hoid1
);
4570 t
.touch(cid
, hoid2
);
4571 t
.touch(cid
, hoid3
);
4572 r
= apply_transaction(store
, &osr
, std::move(t
));
4573 created
.insert(hoid1
);
4574 created
.insert(hoid2
);
4575 created
.insert(hoid3
);
4579 vector
<ghobject_t
> objects
;
4580 r
= store
->collection_list(cid
, ghobject_t(), ghobject_t::get_max(),
4581 INT_MAX
, &objects
, 0);
4583 set
<ghobject_t
> listed(objects
.begin(), objects
.end());
4584 cerr
<< "listed.size() is " << listed
.size() << " and created.size() is " << created
.size() << std::endl
;
4585 ASSERT_TRUE(listed
.size() == created
.size());
4588 ghobject_t current
, next
;
4590 r
= store
->collection_list(cid
, current
, ghobject_t::get_max(), 60,
4593 ASSERT_TRUE(sorted(objects
));
4594 for (vector
<ghobject_t
>::iterator i
= objects
.begin();
4595 i
!= objects
.end(); ++i
) {
4596 if (listed
.count(*i
))
4597 cerr
<< *i
<< " repeated" << std::endl
;
4600 if (objects
.size() < 50) {
4601 ASSERT_TRUE(next
.is_max());
4605 current
= next
.get_boundary();
4607 cerr
<< "listed.size() is " << listed
.size() << std::endl
;
4608 ASSERT_TRUE(listed
.size() == created
.size());
4609 for (set
<ghobject_t
>::iterator i
= listed
.begin();
4612 ASSERT_TRUE(created
.count(*i
));
4615 for (set
<ghobject_t
>::iterator i
= created
.begin();
4618 ObjectStore::Transaction t
;
4620 r
= apply_transaction(store
, &osr
, std::move(t
));
4623 ObjectStore::Transaction t
;
4624 t
.remove_collection(cid
);
4625 r
= apply_transaction(store
, &osr
, std::move(t
));
4630 TEST_P(StoreTest
, OMapTest
) {
4631 ObjectStore::Sequencer
osr("test");
4633 ghobject_t
hoid(hobject_t("tesomap", "", CEPH_NOSNAP
, 0, 0, ""));
4636 ObjectStore::Transaction t
;
4637 t
.create_collection(cid
, 0);
4638 r
= apply_transaction(store
, &osr
, std::move(t
));
4642 map
<string
, bufferlist
> attrs
;
4644 ObjectStore::Transaction t
;
4646 t
.omap_clear(cid
, hoid
);
4647 map
<string
, bufferlist
> start_set
;
4648 t
.omap_setkeys(cid
, hoid
, start_set
);
4649 r
= apply_transaction(store
, &osr
, std::move(t
));
4653 for (int i
= 0; i
< 100; i
++) {
4655 std::cout
<< "On iteration " << i
<< std::endl
;
4657 ObjectStore::Transaction t
;
4659 map
<string
, bufferlist
> cur_attrs
;
4660 r
= store
->omap_get(cid
, hoid
, &bl
, &cur_attrs
);
4662 for (map
<string
, bufferlist
>::iterator j
= attrs
.begin();
4665 bool correct
= cur_attrs
.count(j
->first
) && string(cur_attrs
[j
->first
].c_str()) == string(j
->second
.c_str());
4667 std::cout
<< j
->first
<< " is present in cur_attrs " << cur_attrs
.count(j
->first
) << " times " << std::endl
;
4668 if (cur_attrs
.count(j
->first
) > 0) {
4669 std::cout
<< j
->second
.c_str() << " : " << cur_attrs
[j
->first
].c_str() << std::endl
;
4672 ASSERT_EQ(correct
, true);
4674 ASSERT_EQ(attrs
.size(), cur_attrs
.size());
4677 snprintf(buf
, sizeof(buf
), "%d", i
);
4679 bufferptr
bp(buf
, strlen(buf
) + 1);
4681 map
<string
, bufferlist
> to_add
;
4682 to_add
.insert(pair
<string
, bufferlist
>("key-" + string(buf
), bl
));
4683 attrs
.insert(pair
<string
, bufferlist
>("key-" + string(buf
), bl
));
4684 t
.omap_setkeys(cid
, hoid
, to_add
);
4685 r
= apply_transaction(store
, &osr
, std::move(t
));
4690 while (attrs
.size()) {
4692 std::cout
<< "removal: On iteration " << i
<< std::endl
;
4694 ObjectStore::Transaction t
;
4696 map
<string
, bufferlist
> cur_attrs
;
4697 r
= store
->omap_get(cid
, hoid
, &bl
, &cur_attrs
);
4699 for (map
<string
, bufferlist
>::iterator j
= attrs
.begin();
4702 bool correct
= cur_attrs
.count(j
->first
) && string(cur_attrs
[j
->first
].c_str()) == string(j
->second
.c_str());
4704 std::cout
<< j
->first
<< " is present in cur_attrs " << cur_attrs
.count(j
->first
) << " times " << std::endl
;
4705 if (cur_attrs
.count(j
->first
) > 0) {
4706 std::cout
<< j
->second
.c_str() << " : " << cur_attrs
[j
->first
].c_str() << std::endl
;
4709 ASSERT_EQ(correct
, true);
4712 string to_remove
= attrs
.begin()->first
;
4713 set
<string
> keys_to_remove
;
4714 keys_to_remove
.insert(to_remove
);
4715 t
.omap_rmkeys(cid
, hoid
, keys_to_remove
);
4716 r
= apply_transaction(store
, &osr
, std::move(t
));
4719 attrs
.erase(to_remove
);
4726 bl1
.append("omap_header");
4727 ObjectStore::Transaction t
;
4728 t
.omap_setheader(cid
, hoid
, bl1
);
4729 r
= apply_transaction(store
, &osr
, std::move(t
));
4731 t
= ObjectStore::Transaction();
4734 bl2
.append("value");
4735 map
<string
, bufferlist
> to_add
;
4736 to_add
.insert(pair
<string
, bufferlist
>("key", bl2
));
4737 t
.omap_setkeys(cid
, hoid
, to_add
);
4738 r
= apply_transaction(store
, &osr
, std::move(t
));
4742 map
<string
, bufferlist
> cur_attrs
;
4743 r
= store
->omap_get(cid
, hoid
, &bl3
, &cur_attrs
);
4745 ASSERT_EQ(cur_attrs
.size(), size_t(1));
4746 ASSERT_TRUE(bl_eq(bl1
, bl3
));
4749 r
= store
->omap_get_keys(cid
, hoid
, &keys
);
4751 ASSERT_EQ(keys
.size(), size_t(1));
4754 // test omap_clear, omap_rmkey_range
4757 map
<string
,bufferlist
> to_set
;
4758 for (int n
=0; n
<10; ++n
) {
4759 to_set
[stringify(n
)].append("foo");
4763 ObjectStore::Transaction t
;
4764 t
.remove(cid
, hoid
);
4766 t
.omap_setheader(cid
, hoid
, h
);
4767 t
.omap_setkeys(cid
, hoid
, to_set
);
4768 r
= apply_transaction(store
, &osr
, std::move(t
));
4772 ObjectStore::Transaction t
;
4773 t
.omap_rmkeyrange(cid
, hoid
, "3", "7");
4774 r
= apply_transaction(store
, &osr
, std::move(t
));
4779 map
<string
,bufferlist
> m
;
4780 store
->omap_get(cid
, hoid
, &hdr
, &m
);
4781 ASSERT_EQ(6u, hdr
.length());
4782 ASSERT_TRUE(m
.count("2"));
4783 ASSERT_TRUE(!m
.count("3"));
4784 ASSERT_TRUE(!m
.count("6"));
4785 ASSERT_TRUE(m
.count("7"));
4786 ASSERT_TRUE(m
.count("8"));
4787 //cout << m << std::endl;
4788 ASSERT_EQ(6u, m
.size());
4791 ObjectStore::Transaction t
;
4792 t
.omap_clear(cid
, hoid
);
4793 r
= apply_transaction(store
, &osr
, std::move(t
));
4798 map
<string
,bufferlist
> m
;
4799 store
->omap_get(cid
, hoid
, &hdr
, &m
);
4800 ASSERT_EQ(0u, hdr
.length());
4801 ASSERT_EQ(0u, m
.size());
4805 ObjectStore::Transaction t
;
4806 t
.remove(cid
, hoid
);
4807 t
.remove_collection(cid
);
4808 r
= apply_transaction(store
, &osr
, std::move(t
));
4812 TEST_P(StoreTest
, OMapIterator
) {
4813 ObjectStore::Sequencer
osr("test");
4815 ghobject_t
hoid(hobject_t("tesomap", "", CEPH_NOSNAP
, 0, 0, ""));
4819 ObjectStore::Transaction t
;
4820 t
.create_collection(cid
, 0);
4821 r
= apply_transaction(store
, &osr
, std::move(t
));
4825 map
<string
, bufferlist
> attrs
;
4827 ObjectStore::Transaction t
;
4829 t
.omap_clear(cid
, hoid
);
4830 map
<string
, bufferlist
> start_set
;
4831 t
.omap_setkeys(cid
, hoid
, start_set
);
4832 r
= apply_transaction(store
, &osr
, std::move(t
));
4835 ObjectMap::ObjectMapIterator iter
;
4838 for (int i
= 0; i
< 100; i
++) {
4840 std::cout
<< "On iteration " << i
<< std::endl
;
4844 // FileStore may deadlock two active iterators over the same data
4845 iter
= ObjectMap::ObjectMapIterator();
4847 iter
= store
->get_omap_iterator(cid
, hoid
);
4848 for (iter
->seek_to_first(), count
=0; iter
->valid(); iter
->next(), count
++) {
4849 string key
= iter
->key();
4850 bufferlist value
= iter
->value();
4851 correct
= attrs
.count(key
) && (string(value
.c_str()) == string(attrs
[key
].c_str()));
4853 if (attrs
.count(key
) > 0) {
4854 std::cout
<< "key " << key
<< "in omap , " << value
.c_str() << " : " << attrs
[key
].c_str() << std::endl
;
4857 std::cout
<< "key " << key
<< "should not exists in omap" << std::endl
;
4859 ASSERT_EQ(correct
, true);
4861 ASSERT_EQ((int)attrs
.size(), count
);
4863 // FileStore may deadlock an active iterator vs apply_transaction
4864 iter
= ObjectMap::ObjectMapIterator();
4867 snprintf(buf
, sizeof(buf
), "%d", i
);
4869 bufferptr
bp(buf
, strlen(buf
) + 1);
4871 map
<string
, bufferlist
> to_add
;
4872 to_add
.insert(pair
<string
, bufferlist
>("key-" + string(buf
), bl
));
4873 attrs
.insert(pair
<string
, bufferlist
>("key-" + string(buf
), bl
));
4874 ObjectStore::Transaction t
;
4875 t
.omap_setkeys(cid
, hoid
, to_add
);
4876 r
= apply_transaction(store
, &osr
, std::move(t
));
4880 iter
= store
->get_omap_iterator(cid
, hoid
);
4882 string bound_key
= "key-5";
4883 iter
->lower_bound(bound_key
);
4884 correct
= bound_key
<= iter
->key();
4886 std::cout
<< "lower bound, bound key is " << bound_key
<< " < iter key is " << iter
->key() << std::endl
;
4888 ASSERT_EQ(correct
, true);
4890 iter
->upper_bound(bound_key
);
4891 correct
= iter
->key() > bound_key
;
4893 std::cout
<< "upper bound, bound key is " << bound_key
<< " >= iter key is " << iter
->key() << std::endl
;
4895 ASSERT_EQ(correct
, true);
4897 // FileStore may deadlock an active iterator vs apply_transaction
4898 iter
= ObjectMap::ObjectMapIterator();
4900 ObjectStore::Transaction t
;
4901 t
.remove(cid
, hoid
);
4902 t
.remove_collection(cid
);
4903 r
= apply_transaction(store
, &osr
, std::move(t
));
4908 TEST_P(StoreTest
, XattrTest
) {
4909 ObjectStore::Sequencer
osr("test");
4911 ghobject_t
hoid(hobject_t("tesomap", "", CEPH_NOSNAP
, 0, 0, ""));
4913 for (unsigned i
= 0; i
< 10000; ++i
) {
4917 for (unsigned i
= 0; i
< 10; ++i
) {
4922 ObjectStore::Transaction t
;
4923 t
.create_collection(cid
, 0);
4925 r
= apply_transaction(store
, &osr
, std::move(t
));
4929 map
<string
, bufferlist
> attrs
;
4931 ObjectStore::Transaction t
;
4932 t
.setattr(cid
, hoid
, "attr1", small
);
4933 attrs
["attr1"] = small
;
4934 t
.setattr(cid
, hoid
, "attr2", big
);
4935 attrs
["attr2"] = big
;
4936 t
.setattr(cid
, hoid
, "attr3", small
);
4937 attrs
["attr3"] = small
;
4938 t
.setattr(cid
, hoid
, "attr1", small
);
4939 attrs
["attr1"] = small
;
4940 t
.setattr(cid
, hoid
, "attr4", big
);
4941 attrs
["attr4"] = big
;
4942 t
.setattr(cid
, hoid
, "attr3", big
);
4943 attrs
["attr3"] = big
;
4944 r
= apply_transaction(store
, &osr
, std::move(t
));
4948 map
<string
, bufferptr
> aset
;
4949 store
->getattrs(cid
, hoid
, aset
);
4950 ASSERT_EQ(aset
.size(), attrs
.size());
4951 for (map
<string
, bufferptr
>::iterator i
= aset
.begin();
4955 bl
.push_back(i
->second
);
4956 ASSERT_TRUE(attrs
[i
->first
] == bl
);
4960 ObjectStore::Transaction t
;
4961 t
.rmattr(cid
, hoid
, "attr2");
4962 attrs
.erase("attr2");
4963 r
= apply_transaction(store
, &osr
, std::move(t
));
4968 store
->getattrs(cid
, hoid
, aset
);
4969 ASSERT_EQ(aset
.size(), attrs
.size());
4970 for (map
<string
, bufferptr
>::iterator i
= aset
.begin();
4974 bl
.push_back(i
->second
);
4975 ASSERT_TRUE(attrs
[i
->first
] == bl
);
4979 r
= store
->getattr(cid
, hoid
, "attr2", bp
);
4980 ASSERT_EQ(r
, -ENODATA
);
4982 r
= store
->getattr(cid
, hoid
, "attr3", bp
);
4986 ASSERT_TRUE(bl2
== attrs
["attr3"]);
4988 ObjectStore::Transaction t
;
4989 t
.remove(cid
, hoid
);
4990 t
.remove_collection(cid
);
4991 r
= apply_transaction(store
, &osr
, std::move(t
));
4997 unsigned num_objects
,
4998 unsigned common_suffix_size
,
5001 ObjectStore::Sequencer
osr("test");
5002 coll_t
cid(spg_t(pg_t(0,52),shard_id_t::NO_SHARD
));
5003 coll_t
tid(spg_t(pg_t(1<<common_suffix_size
,52),shard_id_t::NO_SHARD
));
5006 ObjectStore::Transaction t
;
5007 t
.create_collection(cid
, common_suffix_size
);
5008 r
= apply_transaction(store
, &osr
, std::move(t
));
5012 small
.append("small");
5014 ObjectStore::Transaction t
;
5015 for (uint32_t i
= 0; i
< (2 - (int)clones
)*num_objects
; ++i
) {
5016 stringstream objname
;
5017 objname
<< "obj" << i
;
5018 ghobject_t
a(hobject_t(
5022 i
<<common_suffix_size
,
5024 t
.write(cid
, a
, 0, small
.length(), small
,
5025 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
5027 objname
<< "-clone";
5028 ghobject_t
b(hobject_t(
5032 i
<<common_suffix_size
,
5037 r
= apply_transaction(store
, &osr
, std::move(t
));
5039 t
= ObjectStore::Transaction();
5042 r
= apply_transaction(store
, &osr
, std::move(t
));
5046 ObjectStore::Transaction t
;
5047 t
.create_collection(tid
, common_suffix_size
+ 1);
5048 t
.split_collection(cid
, common_suffix_size
+1, 1<<common_suffix_size
, tid
);
5049 r
= apply_transaction(store
, &osr
, std::move(t
));
5053 ObjectStore::Transaction t
;
5054 vector
<ghobject_t
> objects
;
5055 r
= store
->collection_list(cid
, ghobject_t(), ghobject_t::get_max(),
5056 INT_MAX
, &objects
, 0);
5058 ASSERT_EQ(objects
.size(), num_objects
);
5060 for (vector
<ghobject_t
>::iterator i
= objects
.begin();
5063 ASSERT_EQ(!!(i
->hobj
.get_hash() & (1<<common_suffix_size
)), 0u);
5067 r
= apply_transaction(store
, &osr
, std::move(t
));
5069 t
= ObjectStore::Transaction();
5074 r
= store
->collection_list(tid
, ghobject_t(), ghobject_t::get_max(),
5075 INT_MAX
, &objects
, 0);
5077 ASSERT_EQ(objects
.size(), num_objects
);
5078 for (vector
<ghobject_t
>::iterator i
= objects
.begin();
5081 ASSERT_EQ(!(i
->hobj
.get_hash() & (1<<common_suffix_size
)), 0u);
5085 r
= apply_transaction(store
, &osr
, std::move(t
));
5087 t
= ObjectStore::Transaction();
5091 t
.remove_collection(cid
);
5092 t
.remove_collection(tid
);
5093 r
= apply_transaction(store
, &osr
, std::move(t
));
5097 TEST_P(StoreTest
, ColSplitTest1
) {
5098 colsplittest(store
.get(), 10000, 11, false);
5100 TEST_P(StoreTest
, ColSplitTest1Clones
) {
5101 colsplittest(store
.get(), 10000, 11, true);
5103 TEST_P(StoreTest
, ColSplitTest2
) {
5104 colsplittest(store
.get(), 100, 7, false);
5106 TEST_P(StoreTest
, ColSplitTest2Clones
) {
5107 colsplittest(store
.get(), 100, 7, true);
5111 TEST_P(StoreTest
, ColSplitTest3
) {
5112 colsplittest(store
.get(), 100000, 25);
5117 * This test tests adding two different groups
5118 * of objects, each with 1 common prefix and 1
5119 * different prefix. We then remove half
5120 * in order to verify that the merging correctly
5121 * stops at the common prefix subdir. See bug
5123 TEST_P(StoreTest
, TwoHash
) {
5124 ObjectStore::Sequencer
osr("test");
5128 ObjectStore::Transaction t
;
5129 t
.create_collection(cid
, 0);
5130 r
= apply_transaction(store
, &osr
, std::move(t
));
5133 std::cout
<< "Making objects" << std::endl
;
5134 for (int i
= 0; i
< 360; ++i
) {
5135 ObjectStore::Transaction t
;
5139 o
.hobj
.set_hash((i
<< 16) | 0xA1);
5142 o
.hobj
.set_hash((i
<< 16) | 0xB1);
5144 r
= apply_transaction(store
, &osr
, std::move(t
));
5147 std::cout
<< "Removing half" << std::endl
;
5148 for (int i
= 1; i
< 8; ++i
) {
5149 ObjectStore::Transaction t
;
5152 o
.hobj
.set_hash((i
<< 16) | 0xA1);
5154 r
= apply_transaction(store
, &osr
, std::move(t
));
5157 std::cout
<< "Checking" << std::endl
;
5158 for (int i
= 1; i
< 8; ++i
) {
5159 ObjectStore::Transaction t
;
5161 o
.hobj
.set_hash((i
<< 16) | 0xA1);
5163 bool exists
= store
->exists(cid
, o
);
5164 ASSERT_EQ(exists
, false);
5168 o
.hobj
.set_hash(0xA1);
5170 bool exists
= store
->exists(cid
, o
);
5171 ASSERT_EQ(exists
, true);
5173 std::cout
<< "Cleanup" << std::endl
;
5174 for (int i
= 0; i
< 360; ++i
) {
5175 ObjectStore::Transaction t
;
5177 o
.hobj
.set_hash((i
<< 16) | 0xA1);
5180 o
.hobj
.set_hash((i
<< 16) | 0xB1);
5182 r
= apply_transaction(store
, &osr
, std::move(t
));
5185 ObjectStore::Transaction t
;
5186 t
.remove_collection(cid
);
5187 r
= apply_transaction(store
, &osr
, std::move(t
));
5191 TEST_P(StoreTest
, Rename
) {
5192 ObjectStore::Sequencer
osr("test");
5193 coll_t
cid(spg_t(pg_t(0, 2122),shard_id_t::NO_SHARD
));
5194 ghobject_t
srcoid(hobject_t("src_oid", "", CEPH_NOSNAP
, 0, 0, ""));
5195 ghobject_t
dstoid(hobject_t("dest_oid", "", CEPH_NOSNAP
, 0, 0, ""));
5201 ObjectStore::Transaction t
;
5202 t
.create_collection(cid
, 0);
5203 t
.write(cid
, srcoid
, 0, a
.length(), a
);
5204 r
= apply_transaction(store
, &osr
, std::move(t
));
5207 ASSERT_TRUE(store
->exists(cid
, srcoid
));
5209 ObjectStore::Transaction t
;
5210 t
.collection_move_rename(cid
, srcoid
, cid
, dstoid
);
5211 t
.write(cid
, srcoid
, 0, b
.length(), b
);
5212 t
.setattr(cid
, srcoid
, "attr", b
);
5213 r
= apply_transaction(store
, &osr
, std::move(t
));
5216 ASSERT_TRUE(store
->exists(cid
, srcoid
));
5217 ASSERT_TRUE(store
->exists(cid
, dstoid
));
5220 store
->read(cid
, srcoid
, 0, 3, bl
);
5221 ASSERT_TRUE(bl_eq(b
, bl
));
5222 store
->read(cid
, dstoid
, 0, 3, bl
);
5223 ASSERT_TRUE(bl_eq(a
, bl
));
5226 ObjectStore::Transaction t
;
5227 t
.remove(cid
, dstoid
);
5228 t
.collection_move_rename(cid
, srcoid
, cid
, dstoid
);
5229 r
= apply_transaction(store
, &osr
, std::move(t
));
5232 ASSERT_TRUE(store
->exists(cid
, dstoid
));
5233 ASSERT_FALSE(store
->exists(cid
, srcoid
));
5236 store
->read(cid
, dstoid
, 0, 3, bl
);
5237 ASSERT_TRUE(bl_eq(b
, bl
));
5240 ObjectStore::Transaction t
;
5241 t
.remove(cid
, dstoid
);
5242 t
.remove_collection(cid
);
5243 r
= apply_transaction(store
, &osr
, std::move(t
));
5248 TEST_P(StoreTest
, MoveRename
) {
5249 ObjectStore::Sequencer
osr("test");
5250 coll_t
cid(spg_t(pg_t(0, 212),shard_id_t::NO_SHARD
));
5251 ghobject_t
temp_oid(hobject_t("tmp_oid", "", CEPH_NOSNAP
, 0, 0, ""));
5252 ghobject_t
oid(hobject_t("dest_oid", "", CEPH_NOSNAP
, 0, 0, ""));
5255 ObjectStore::Transaction t
;
5256 t
.create_collection(cid
, 0);
5258 r
= apply_transaction(store
, &osr
, std::move(t
));
5261 ASSERT_TRUE(store
->exists(cid
, oid
));
5262 bufferlist data
, attr
;
5263 map
<string
, bufferlist
> omap
;
5264 data
.append("data payload");
5265 attr
.append("attr value");
5266 omap
["omap_key"].append("omap value");
5268 ObjectStore::Transaction t
;
5269 t
.touch(cid
, temp_oid
);
5270 t
.write(cid
, temp_oid
, 0, data
.length(), data
);
5271 t
.setattr(cid
, temp_oid
, "attr", attr
);
5272 t
.omap_setkeys(cid
, temp_oid
, omap
);
5273 r
= apply_transaction(store
, &osr
, std::move(t
));
5276 ASSERT_TRUE(store
->exists(cid
, temp_oid
));
5278 ObjectStore::Transaction t
;
5280 t
.collection_move_rename(cid
, temp_oid
, cid
, oid
);
5281 r
= apply_transaction(store
, &osr
, std::move(t
));
5284 ASSERT_TRUE(store
->exists(cid
, oid
));
5285 ASSERT_FALSE(store
->exists(cid
, temp_oid
));
5288 r
= store
->read(cid
, oid
, 0, 1000, newdata
);
5290 ASSERT_TRUE(bl_eq(data
, newdata
));
5292 r
= store
->getattr(cid
, oid
, "attr", newattr
);
5294 ASSERT_TRUE(bl_eq(attr
, newattr
));
5296 keys
.insert("omap_key");
5297 map
<string
, bufferlist
> newomap
;
5298 r
= store
->omap_get_values(cid
, oid
, keys
, &newomap
);
5300 ASSERT_EQ(1u, newomap
.size());
5301 ASSERT_TRUE(newomap
.count("omap_key"));
5302 ASSERT_TRUE(bl_eq(omap
["omap_key"], newomap
["omap_key"]));
5305 ObjectStore::Transaction t
;
5307 t
.remove_collection(cid
);
5308 r
= apply_transaction(store
, &osr
, std::move(t
));
5313 TEST_P(StoreTest
, BigRGWObjectName
) {
5314 ObjectStore::Sequencer
osr("test");
5315 coll_t
cid(spg_t(pg_t(0,12),shard_id_t::NO_SHARD
));
5318 "default.4106.50_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
5325 shard_id_t::NO_SHARD
);
5326 ghobject_t
oid2(oid
);
5327 oid2
.generation
= 17;
5328 ghobject_t
oidhead(oid
);
5329 oidhead
.generation
= ghobject_t::NO_GEN
;
5333 ObjectStore::Transaction t
;
5334 t
.create_collection(cid
, 0);
5335 t
.touch(cid
, oidhead
);
5336 t
.collection_move_rename(cid
, oidhead
, cid
, oid
);
5337 t
.touch(cid
, oidhead
);
5338 t
.collection_move_rename(cid
, oidhead
, cid
, oid2
);
5339 r
= apply_transaction(store
, &osr
, std::move(t
));
5344 ObjectStore::Transaction t
;
5346 r
= apply_transaction(store
, &osr
, std::move(t
));
5351 vector
<ghobject_t
> objects
;
5352 r
= store
->collection_list(cid
, ghobject_t(), ghobject_t::get_max(),
5353 INT_MAX
, &objects
, 0);
5355 ASSERT_EQ(objects
.size(), 1u);
5356 ASSERT_EQ(objects
[0], oid2
);
5359 ASSERT_FALSE(store
->exists(cid
, oid
));
5362 ObjectStore::Transaction t
;
5363 t
.remove(cid
, oid2
);
5364 t
.remove_collection(cid
);
5365 r
= apply_transaction(store
, &osr
, std::move(t
));
5371 TEST_P(StoreTest
, SetAllocHint
) {
5372 ObjectStore::Sequencer
osr("test");
5374 ghobject_t
hoid(hobject_t("test_hint", "", CEPH_NOSNAP
, 0, 0, ""));
5377 ObjectStore::Transaction t
;
5378 t
.create_collection(cid
, 0);
5380 r
= apply_transaction(store
, &osr
, std::move(t
));
5384 ObjectStore::Transaction t
;
5385 t
.set_alloc_hint(cid
, hoid
, 4*1024*1024, 1024*4, 0);
5386 r
= apply_transaction(store
, &osr
, std::move(t
));
5390 ObjectStore::Transaction t
;
5391 t
.remove(cid
, hoid
);
5392 r
= apply_transaction(store
, &osr
, std::move(t
));
5396 ObjectStore::Transaction t
;
5397 t
.set_alloc_hint(cid
, hoid
, 4*1024*1024, 1024*4, 0);
5398 r
= apply_transaction(store
, &osr
, std::move(t
));
5402 ObjectStore::Transaction t
;
5403 t
.remove_collection(cid
);
5404 r
= apply_transaction(store
, &osr
, std::move(t
));
5409 TEST_P(StoreTest
, TryMoveRename
) {
5410 ObjectStore::Sequencer
osr("test");
5412 ghobject_t
hoid(hobject_t("test_hint", "", CEPH_NOSNAP
, 0, -1, ""));
5413 ghobject_t
hoid2(hobject_t("test_hint2", "", CEPH_NOSNAP
, 0, -1, ""));
5416 ObjectStore::Transaction t
;
5417 t
.create_collection(cid
, 0);
5418 r
= apply_transaction(store
, &osr
, std::move(t
));
5422 ObjectStore::Transaction t
;
5423 t
.try_rename(cid
, hoid
, hoid2
);
5424 r
= apply_transaction(store
, &osr
, std::move(t
));
5428 ObjectStore::Transaction t
;
5430 r
= apply_transaction(store
, &osr
, std::move(t
));
5434 ObjectStore::Transaction t
;
5435 t
.try_rename(cid
, hoid
, hoid2
);
5436 r
= apply_transaction(store
, &osr
, std::move(t
));
5440 ASSERT_EQ(store
->stat(cid
, hoid
, &st
), -ENOENT
);
5441 ASSERT_EQ(store
->stat(cid
, hoid2
, &st
), 0);
5444 #if defined(HAVE_LIBAIO)
5445 TEST_P(StoreTest
, BluestoreOnOffCSumTest
) {
5446 if (string(GetParam()) != "bluestore")
5448 g_conf
->set_val("bluestore_csum_type", "crc32c");
5449 g_conf
->apply_changes(NULL
);
5451 ObjectStore::Sequencer
osr("test");
5454 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
5457 r
= store
->read(cid
, hoid
, 0, 5, in
);
5458 ASSERT_EQ(-ENOENT
, r
);
5461 ObjectStore::Transaction t
;
5462 t
.create_collection(cid
, 0);
5463 cerr
<< "Creating collection " << cid
<< std::endl
;
5464 r
= apply_transaction(store
, &osr
, std::move(t
));
5468 //write with csum enabled followed by read with csum disabled
5469 size_t block_size
= 64*1024;
5470 ObjectStore::Transaction t
;
5471 bufferlist bl
, orig
;
5472 bl
.append(std::string(block_size
, 'a'));
5474 t
.remove(cid
, hoid
);
5475 t
.set_alloc_hint(cid
, hoid
, 4*1024*1024, 1024*8, 0);
5476 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
5477 cerr
<< "Remove then create" << std::endl
;
5478 r
= apply_transaction(store
, &osr
, std::move(t
));
5481 g_conf
->set_val("bluestore_csum_type", "none");
5482 g_conf
->apply_changes(NULL
);
5485 r
= store
->read(cid
, hoid
, 0, block_size
, in
);
5486 ASSERT_EQ((int)block_size
, r
);
5487 ASSERT_TRUE(bl_eq(orig
, in
));
5491 //write with csum disabled followed by read with csum enabled
5493 size_t block_size
= 64*1024;
5494 ObjectStore::Transaction t
;
5495 bufferlist bl
, orig
;
5496 bl
.append(std::string(block_size
, 'a'));
5498 t
.remove(cid
, hoid
);
5499 t
.set_alloc_hint(cid
, hoid
, 4*1024*1024, 1024*8, 0);
5500 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
5501 cerr
<< "Remove then create" << std::endl
;
5502 r
= apply_transaction(store
, &osr
, std::move(t
));
5505 g_conf
->set_val("bluestore_csum_type", "crc32c");
5506 g_conf
->apply_changes(NULL
);
5509 r
= store
->read(cid
, hoid
, 0, block_size
, in
);
5510 ASSERT_EQ((int)block_size
, r
);
5511 ASSERT_TRUE(bl_eq(orig
, in
));
5514 //'mixed' non-overlapping writes to the same blob
5516 ObjectStore::Transaction t
;
5517 bufferlist bl
, orig
;
5518 size_t block_size
= 8000;
5519 bl
.append(std::string(block_size
, 'a'));
5521 t
.remove(cid
, hoid
);
5522 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
5523 cerr
<< "Remove then create" << std::endl
;
5524 r
= apply_transaction(store
, &osr
, std::move(t
));
5527 g_conf
->set_val("bluestore_csum_type", "none");
5528 g_conf
->apply_changes(NULL
);
5530 ObjectStore::Transaction t2
;
5531 t2
.write(cid
, hoid
, block_size
*2, bl
.length(), bl
);
5532 cerr
<< "Append 'unprotected'" << std::endl
;
5533 r
= apply_transaction(store
, &osr
, std::move(t2
));
5537 r
= store
->read(cid
, hoid
, 0, block_size
, in
);
5538 ASSERT_EQ((int)block_size
, r
);
5539 ASSERT_TRUE(bl_eq(orig
, in
));
5541 r
= store
->read(cid
, hoid
, block_size
*2, block_size
, in
);
5542 ASSERT_EQ((int)block_size
, r
);
5543 ASSERT_TRUE(bl_eq(orig
, in
));
5545 g_conf
->set_val("bluestore_csum_type", "crc32c");
5546 g_conf
->apply_changes(NULL
);
5548 r
= store
->read(cid
, hoid
, 0, block_size
, in
);
5549 ASSERT_EQ((int)block_size
, r
);
5550 ASSERT_TRUE(bl_eq(orig
, in
));
5552 r
= store
->read(cid
, hoid
, block_size
*2, block_size
, in
);
5553 ASSERT_EQ((int)block_size
, r
);
5554 ASSERT_TRUE(bl_eq(orig
, in
));
5557 //partially blob overwrite under a different csum enablement mode
5559 ObjectStore::Transaction t
;
5560 bufferlist bl
, orig
, orig2
;
5561 size_t block_size0
= 0x10000;
5562 size_t block_size
= 9000;
5563 size_t block_size2
= 5000;
5564 bl
.append(std::string(block_size0
, 'a'));
5565 t
.remove(cid
, hoid
);
5566 t
.set_alloc_hint(cid
, hoid
, 4*1024*1024, 1024*8, 0);
5567 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
5568 cerr
<< "Remove then create" << std::endl
;
5569 r
= apply_transaction(store
, &osr
, std::move(t
));
5572 g_conf
->set_val("bluestore_csum_type", "none");
5573 g_conf
->apply_changes(NULL
);
5575 ObjectStore::Transaction t2
;
5577 bl
.append(std::string(block_size
, 'b'));
5578 t2
.write(cid
, hoid
, 0, bl
.length(), bl
);
5579 t2
.write(cid
, hoid
, block_size0
, bl
.length(), bl
);
5580 cerr
<< "Overwrite with unprotected data" << std::endl
;
5581 r
= apply_transaction(store
, &osr
, std::move(t2
));
5586 orig
.append( std::string(block_size0
- block_size
, 'a'));
5589 r
= store
->read(cid
, hoid
, 0, block_size0
, in
);
5590 ASSERT_EQ((int)block_size0
, r
);
5591 ASSERT_TRUE(bl_eq(orig
, in
));
5593 r
= store
->read(cid
, hoid
, block_size0
, block_size
, in
);
5594 ASSERT_EQ((int)block_size
, r
);
5595 ASSERT_TRUE(bl_eq(orig2
, in
));
5597 g_conf
->set_val("bluestore_csum_type", "crc32c");
5598 g_conf
->apply_changes(NULL
);
5600 ObjectStore::Transaction t3
;
5602 bl
.append(std::string(block_size2
, 'c'));
5603 t3
.write(cid
, hoid
, block_size0
, bl
.length(), bl
);
5604 cerr
<< "Overwrite with protected data" << std::endl
;
5605 r
= apply_transaction(store
, &osr
, std::move(t3
));
5610 orig
.append( std::string(block_size
- block_size2
, 'b'));
5611 r
= store
->read(cid
, hoid
, block_size0
, block_size
, in
);
5612 ASSERT_EQ((int)block_size
, r
);
5613 ASSERT_TRUE(bl_eq(orig
, in
));
5617 ObjectStore::Transaction t
;
5618 t
.remove(cid
, hoid
);
5619 t
.remove_collection(cid
);
5620 cerr
<< "Cleaning" << std::endl
;
5621 r
= apply_transaction(store
, &osr
, std::move(t
));
5627 INSTANTIATE_TEST_CASE_P(
5633 #if defined(HAVE_LIBAIO)
5638 // Note: instantiate all stores to preserve store numbering order only
5639 INSTANTIATE_TEST_CASE_P(
5641 StoreTestSpecificAUSize
,
5645 #if defined(HAVE_LIBAIO)
5652 // Google Test may not support value-parameterized tests with some
5653 // compilers. If we use conditional compilation to compile out all
5654 // code referring to the gtest_main library, MSVC linker will not link
5655 // that library at all and consequently complain about missing entry
5656 // point defined in that library (fatal error LNK1561: entry point
5657 // must be defined). This dummy test keeps gtest_main linked in.
5658 TEST(DummyTest
, ValueParameterizedTestsAreNotSupportedOnThisPlatform
) {}
5662 void doMany4KWritesTest(boost::scoped_ptr
<ObjectStore
>& store
,
5663 unsigned max_objects
,
5665 unsigned max_object_size
,
5666 unsigned max_write_size
,
5667 unsigned write_alignment
,
5668 store_statfs_t
* res_stat
)
5670 ObjectStore::Sequencer
osr("test");
5671 MixedGenerator
gen(555);
5672 gen_type
rng(time(NULL
));
5673 coll_t
cid(spg_t(pg_t(0,555), shard_id_t::NO_SHARD
));
5675 SyntheticWorkloadState
test_obj(store
.get(),
5684 for (unsigned i
= 0; i
< max_objects
; ++i
) {
5685 if (!(i
% 500)) cerr
<< "seeding object " << i
<< std::endl
;
5688 for (unsigned i
= 0; i
< max_ops
; ++i
) {
5690 cerr
<< "Op " << i
<< std::endl
;
5691 test_obj
.print_internal_state();
5695 test_obj
.wait_for_done();
5697 test_obj
.statfs(*res_stat
);
5699 test_obj
.shutdown();
5702 TEST_P(StoreTestSpecificAUSize
, Many4KWritesTest
) {
5703 if (string(GetParam()) != "bluestore")
5706 StartDeferred(0x10000);
5708 store_statfs_t res_stat
;
5709 unsigned max_object
= 4*1024*1024;
5711 doMany4KWritesTest(store
, 1, 1000, 4*1024*1024, 4*1024, 0, &res_stat
);
5713 ASSERT_LE(res_stat
.stored
, max_object
);
5714 ASSERT_EQ(res_stat
.allocated
, max_object
);
5717 TEST_P(StoreTestSpecificAUSize
, Many4KWritesNoCSumTest
) {
5718 if (string(GetParam()) != "bluestore")
5720 StartDeferred(0x10000);
5721 g_conf
->set_val("bluestore_csum_type", "none");
5722 g_ceph_context
->_conf
->apply_changes(NULL
);
5723 store_statfs_t res_stat
;
5724 unsigned max_object
= 4*1024*1024;
5726 doMany4KWritesTest(store
, 1, 1000, max_object
, 4*1024, 0, &res_stat
);
5728 ASSERT_LE(res_stat
.stored
, max_object
);
5729 ASSERT_EQ(res_stat
.allocated
, max_object
);
5730 g_conf
->set_val("bluestore_csum_type", "crc32c");
5733 TEST_P(StoreTestSpecificAUSize
, TooManyBlobsTest
) {
5734 if (string(GetParam()) != "bluestore")
5736 StartDeferred(0x10000);
5737 store_statfs_t res_stat
;
5738 unsigned max_object
= 4*1024*1024;
5739 doMany4KWritesTest(store
, 1, 1000, max_object
, 4*1024, 0, &res_stat
);
5740 ASSERT_LE(res_stat
.stored
, max_object
);
5741 ASSERT_EQ(res_stat
.allocated
, max_object
);
5744 #if defined(HAVE_LIBAIO)
5745 void get_mempool_stats(uint64_t* total_bytes
, uint64_t* total_items
)
5747 uint64_t onode_allocated
= mempool::bluestore_cache_onode::allocated_bytes();
5748 uint64_t other_allocated
= mempool::bluestore_cache_other::allocated_bytes();
5750 uint64_t onode_items
= mempool::bluestore_cache_onode::allocated_items();
5751 uint64_t other_items
= mempool::bluestore_cache_other::allocated_items();
5752 cout
<< "onode(" << onode_allocated
<< "/" << onode_items
5753 << ") other(" << other_allocated
<< "/" << other_items
5754 << ")" << std::endl
;
5755 *total_bytes
= onode_allocated
+ other_allocated
;
5756 *total_items
= onode_items
;
5759 TEST_P(StoreTestSpecificAUSize
, OnodeSizeTracking
) {
5761 if (string(GetParam()) != "bluestore")
5764 size_t block_size
= 4096;
5765 StartDeferred(block_size
);
5766 g_conf
->set_val("bluestore_compression_mode", "none");
5767 g_conf
->set_val("bluestore_csum_type", "none");
5768 g_conf
->set_val("bluestore_cache_size", "400000000");
5769 g_conf
->apply_changes(NULL
);
5771 ObjectStore::Sequencer
osr("test");
5774 ghobject_t
hoid(hobject_t("test_hint", "", CEPH_NOSNAP
, 0, -1, ""));
5775 size_t obj_size
= 4 * 1024 * 1024;
5776 uint64_t total_bytes
, total_bytes2
;
5777 uint64_t total_onodes
;
5778 get_mempool_stats(&total_bytes
, &total_onodes
);
5779 ASSERT_EQ(total_onodes
, 0u);
5782 ObjectStore::Transaction t
;
5783 t
.create_collection(cid
, 0);
5784 r
= apply_transaction(store
, &osr
, std::move(t
));
5788 ObjectStore::Transaction t
;
5789 bufferlist bl
, orig
, orig2
;
5791 bl
.append(std::string(obj_size
, 'a'));
5792 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
5793 r
= apply_transaction(store
, &osr
, std::move(t
));
5796 get_mempool_stats(&total_bytes
, &total_onodes
);
5797 ASSERT_NE(total_bytes
, 0u);
5798 ASSERT_EQ(total_onodes
, 1u);
5801 ObjectStore::Transaction t
;
5802 t
.truncate(cid
, hoid
, 0);
5803 r
= apply_transaction(store
, &osr
, std::move(t
));
5807 for(size_t i
= 0; i
< 1; ++i
) {
5809 bl
.append(std::string(block_size
* (i
+1), 'a'));
5810 for( size_t j
= 0; j
< obj_size
; j
+= bl
.length()) {
5811 ObjectStore::Transaction t
;
5812 t
.write(cid
, hoid
, j
, bl
.length(), bl
);
5813 r
= apply_transaction(store
, &osr
, std::move(t
));
5816 get_mempool_stats(&total_bytes2
, &total_onodes
);
5817 ASSERT_NE(total_bytes2
, 0u);
5818 ASSERT_EQ(total_onodes
, 1u);
5821 cout
<<" mempool dump:\n";
5822 JSONFormatter
f(true);
5823 f
.open_object_section("transaction");
5831 for (size_t i
= 0; i
< obj_size
; i
+= 0x1000) {
5832 store
->read(cid
, hoid
, i
, 0x1000, bl
);
5835 get_mempool_stats(&total_bytes
, &total_onodes
);
5836 ASSERT_NE(total_bytes
, 0u);
5837 ASSERT_EQ(total_onodes
, 1u);
5840 cout
<<" mempool dump:\n";
5841 JSONFormatter
f(true);
5842 f
.open_object_section("transaction");
5849 ObjectStore::Transaction t
;
5850 t
.remove(cid
, hoid
);
5851 t
.remove_collection(cid
);
5852 cerr
<< "Cleaning" << std::endl
;
5853 r
= apply_transaction(store
, &osr
, std::move(t
));
5856 g_ceph_context
->_conf
->set_val("bluestore_cache_size", "4000000");
5857 g_conf
->set_val("bluestore_compression_mode", "none");
5858 g_conf
->set_val("bluestore_csum_type", "crc32c");
5862 TEST_P(StoreTestSpecificAUSize
, BlobReuseOnOverwrite
) {
5864 if (string(GetParam()) != "bluestore")
5867 size_t block_size
= 4096;
5868 StartDeferred(block_size
);
5869 g_conf
->set_val("bluestore_max_blob_size", "65536");
5870 g_conf
->apply_changes(NULL
);
5872 ObjectStore::Sequencer
osr("test");
5875 ghobject_t
hoid(hobject_t("test_hint", "", CEPH_NOSNAP
, 0, -1, ""));
5877 const PerfCounters
* logger
= store
->get_perf_counters();
5880 ObjectStore::Transaction t
;
5881 t
.create_collection(cid
, 0);
5882 r
= apply_transaction(store
, &osr
, std::move(t
));
5886 ObjectStore::Transaction t
;
5889 bl
.append(std::string(block_size
* 2, 'a'));
5890 t
.write(cid
, hoid
, 0, bl
.length(), bl
, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
5891 r
= apply_transaction(store
, &osr
, std::move(t
));
5895 // overwrite at the beginning
5896 ObjectStore::Transaction t
;
5899 bl
.append(std::string(block_size
, 'b'));
5900 t
.write(cid
, hoid
, 0, bl
.length(), bl
, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
5901 r
= apply_transaction(store
, &osr
, std::move(t
));
5906 ObjectStore::Transaction t
;
5909 bl
.append(std::string(block_size
* 2, 'c'));
5910 t
.write(cid
, hoid
, block_size
* 2, bl
.length(), bl
, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
5911 r
= apply_transaction(store
, &osr
, std::move(t
));
5915 // append with a gap
5916 ObjectStore::Transaction t
;
5919 bl
.append(std::string(block_size
* 2, 'd'));
5920 t
.write(cid
, hoid
, block_size
* 5, bl
.length(), bl
, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
5921 r
= apply_transaction(store
, &osr
, std::move(t
));
5925 // We need to issue a read to trigger cache stat update that refresh
5926 // perf counters. additionally we need to wait some time for mempool
5927 // thread to update stats.
5929 bufferlist bl
, expected
;
5930 r
= store
->read(cid
, hoid
, 0, block_size
, bl
);
5931 ASSERT_EQ(r
, (int)block_size
);
5932 expected
.append(string(block_size
, 'b'));
5933 ASSERT_TRUE(bl_eq(expected
, bl
));
5934 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 1u);
5935 ASSERT_EQ(logger
->get(l_bluestore_extents
), 2u);
5939 ObjectStore::Transaction t
;
5942 bl
.append(std::string(block_size
* 2, 'e'));
5944 // Currently we are unable to reuse blob when overwriting in a single step
5945 t
.write(cid
, hoid
, block_size
* 6, bl
.length(), bl
, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
5946 r
= apply_transaction(store
, &osr
, std::move(t
));
5950 // We need to issue a read to trigger cache stat update that refresh
5951 // perf counters. additionally we need to wait some time for mempool
5952 // thread to update stats.
5954 bufferlist bl
, expected
;
5955 r
= store
->read(cid
, hoid
, 0, block_size
, bl
);
5956 ASSERT_EQ(r
, (int)block_size
);
5957 expected
.append(string(block_size
, 'b'));
5958 ASSERT_TRUE(bl_eq(expected
, bl
));
5959 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 1u);
5960 ASSERT_EQ(logger
->get(l_bluestore_extents
), 2u);
5964 ObjectStore::Transaction t
;
5967 bl
.append(std::string(block_size
, 'f'));
5969 t
.write(cid
, hoid
, block_size
* 4, bl
.length(), bl
, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
5970 r
= apply_transaction(store
, &osr
, std::move(t
));
5974 // we need to wait some time for mempool
5975 // thread to update stats to be able to check blob/extent numbers from
5979 bufferlist bl
, expected
;
5980 r
= store
->read(cid
, hoid
, 0, block_size
, bl
);
5981 ASSERT_EQ(r
, (int)block_size
);
5982 expected
.append(string(block_size
, 'b'));
5983 ASSERT_TRUE(bl_eq(expected
, bl
));
5987 r
= store
->read(cid
, hoid
, block_size
, block_size
, bl
);
5988 ASSERT_EQ(r
, (int)block_size
);
5989 expected
.append(string(block_size
, 'a'));
5990 ASSERT_TRUE(bl_eq(expected
, bl
));
5994 r
= store
->read(cid
, hoid
, block_size
* 2, block_size
* 2, bl
);
5995 ASSERT_EQ(r
, (int)block_size
* 2);
5996 expected
.append(string(block_size
* 2, 'c'));
5997 ASSERT_TRUE(bl_eq(expected
, bl
));
6001 r
= store
->read(cid
, hoid
, block_size
* 4, block_size
, bl
);
6002 ASSERT_EQ(r
, (int)block_size
);
6003 expected
.append(string(block_size
, 'f'));
6004 ASSERT_TRUE(bl_eq(expected
, bl
));
6008 r
= store
->read(cid
, hoid
, block_size
* 5, block_size
, bl
);
6009 ASSERT_EQ(r
, (int)block_size
);
6010 expected
.append(string(block_size
, 'd'));
6011 ASSERT_TRUE(bl_eq(expected
, bl
));
6015 r
= store
->read(cid
, hoid
, block_size
* 5, block_size
* 3, bl
);
6016 ASSERT_EQ(r
, (int)block_size
* 3);
6017 expected
.append(string(block_size
, 'd'));
6018 expected
.append(string(block_size
* 2, 'e'));
6019 ASSERT_TRUE(bl_eq(expected
, bl
));
6021 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 1u);
6022 ASSERT_EQ(logger
->get(l_bluestore_extents
), 1u);
6026 ObjectStore::Transaction t
;
6027 t
.remove(cid
, hoid
);
6028 t
.remove_collection(cid
);
6029 cerr
<< "Cleaning" << std::endl
;
6030 r
= apply_transaction(store
, &osr
, std::move(t
));
6033 g_conf
->set_val("bluestore_max_blob_size", "0");
6037 TEST_P(StoreTestSpecificAUSize
, BlobReuseOnOverwriteReverse
) {
6039 if (string(GetParam()) != "bluestore")
6042 size_t block_size
= 4096;
6043 StartDeferred(block_size
);
6044 g_conf
->set_val("bluestore_max_blob_size", "65536");
6046 g_conf
->apply_changes(NULL
);
6048 ObjectStore::Sequencer
osr("test");
6051 ghobject_t
hoid(hobject_t("test_hint", "", CEPH_NOSNAP
, 0, -1, ""));
6053 const PerfCounters
* logger
= store
->get_perf_counters();
6055 ObjectStore::Transaction t
;
6056 t
.create_collection(cid
, 0);
6057 r
= apply_transaction(store
, &osr
, std::move(t
));
6061 ObjectStore::Transaction t
;
6064 bl
.append(std::string(block_size
* 2, 'a'));
6065 t
.write(cid
, hoid
, block_size
* 10, bl
.length(), bl
,
6066 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6067 r
= apply_transaction(store
, &osr
, std::move(t
));
6072 ObjectStore::Transaction t
;
6075 bl
.append(std::string(block_size
, 'b'));
6076 t
.write(cid
, hoid
, block_size
* 9, bl
.length(), bl
,
6077 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6078 r
= apply_transaction(store
, &osr
, std::move(t
));
6082 // We need to issue a read to trigger cache stat update that refresh
6083 // perf counters. additionally we need to wait some time for mempool
6084 // thread to update stats.
6086 bufferlist bl
, expected
;
6087 r
= store
->read(cid
, hoid
, block_size
* 9, block_size
* 2, bl
);
6088 ASSERT_EQ(r
, (int)block_size
* 2);
6089 expected
.append(string(block_size
, 'b'));
6090 expected
.append(string(block_size
, 'a'));
6091 ASSERT_TRUE(bl_eq(expected
, bl
));
6092 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 1u);
6093 ASSERT_EQ(logger
->get(l_bluestore_extents
), 1u);
6098 // prepend existing with a gap
6099 ObjectStore::Transaction t
;
6102 bl
.append(std::string(block_size
, 'c'));
6103 t
.write(cid
, hoid
, block_size
* 7, bl
.length(), bl
,
6104 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6105 r
= apply_transaction(store
, &osr
, std::move(t
));
6109 // We need to issue a read to trigger cache stat update that refresh
6110 // perf counters. additionally we need to wait some time for mempool
6111 // thread to update stats.
6113 bufferlist bl
, expected
;
6114 r
= store
->read(cid
, hoid
, block_size
* 7, block_size
* 3, bl
);
6115 ASSERT_EQ(r
, (int)block_size
* 3);
6116 expected
.append(string(block_size
, 'c'));
6117 expected
.append(string(block_size
, 0));
6118 expected
.append(string(block_size
, 'b'));
6119 ASSERT_TRUE(bl_eq(expected
, bl
));
6120 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 1u);
6121 ASSERT_EQ(logger
->get(l_bluestore_extents
), 2u);
6125 // append after existing with a gap
6126 ObjectStore::Transaction t
;
6129 bl
.append(std::string(block_size
, 'd'));
6130 t
.write(cid
, hoid
, block_size
* 13, bl
.length(), bl
,
6131 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6132 r
= apply_transaction(store
, &osr
, std::move(t
));
6136 // We need to issue a read to trigger cache stat update that refresh
6137 // perf counters. additionally we need to wait some time for mempool
6138 // thread to update stats.
6140 bufferlist bl
, expected
;
6141 r
= store
->read(cid
, hoid
, block_size
* 11, block_size
* 3, bl
);
6142 ASSERT_EQ(r
, (int)block_size
* 3);
6143 expected
.append(string(block_size
, 'a'));
6144 expected
.append(string(block_size
, 0));
6145 expected
.append(string(block_size
, 'd'));
6146 ASSERT_TRUE(bl_eq(expected
, bl
));
6147 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 1u);
6148 ASSERT_EQ(logger
->get(l_bluestore_extents
), 3u);
6152 // append twice to the next max_blob slot
6153 ObjectStore::Transaction t
;
6156 bl
.append(std::string(block_size
, 'e'));
6157 t
.write(cid
, hoid
, block_size
* 17, bl
.length(), bl
,
6158 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6159 t
.write(cid
, hoid
, block_size
* 19, bl
.length(), bl
,
6160 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6161 r
= apply_transaction(store
, &osr
, std::move(t
));
6165 // We need to issue a read to trigger cache stat update that refresh
6166 // perf counters. additionally we need to wait some time for mempool
6167 // thread to update stats.
6169 bufferlist bl
, expected
;
6170 r
= store
->read(cid
, hoid
, block_size
* 17, block_size
* 3, bl
);
6171 ASSERT_EQ(r
, (int)block_size
* 3);
6172 expected
.append(string(block_size
, 'e'));
6173 expected
.append(string(block_size
, 0));
6174 expected
.append(string(block_size
, 'e'));
6175 ASSERT_TRUE(bl_eq(expected
, bl
));
6176 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 2u);
6177 ASSERT_EQ(logger
->get(l_bluestore_extents
), 5u);
6180 // fill gaps at the second slot
6181 ObjectStore::Transaction t
;
6184 bl
.append(std::string(block_size
, 'f'));
6185 t
.write(cid
, hoid
, block_size
* 16, bl
.length(), bl
,
6186 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6187 t
.write(cid
, hoid
, block_size
* 18, bl
.length(), bl
,
6188 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6189 r
= apply_transaction(store
, &osr
, std::move(t
));
6193 // We need to issue a read to trigger cache stat update that refresh
6194 // perf counters. additionally we need to wait some time for mempool
6195 // thread to update stats.
6197 bufferlist bl
, expected
;
6198 r
= store
->read(cid
, hoid
, block_size
* 16, block_size
* 4, bl
);
6199 ASSERT_EQ(r
, (int)block_size
* 4);
6200 expected
.append(string(block_size
, 'f'));
6201 expected
.append(string(block_size
, 'e'));
6202 expected
.append(string(block_size
, 'f'));
6203 expected
.append(string(block_size
, 'e'));
6204 ASSERT_TRUE(bl_eq(expected
, bl
));
6205 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 2u);
6206 ASSERT_EQ(logger
->get(l_bluestore_extents
), 4u);
6209 ObjectStore::Transaction t
;
6210 t
.remove(cid
, hoid
);
6211 t
.remove_collection(cid
);
6212 cerr
<< "Cleaning" << std::endl
;
6213 r
= apply_transaction(store
, &osr
, std::move(t
));
6216 g_conf
->set_val("bluestore_max_blob_size", "0");
6219 TEST_P(StoreTestSpecificAUSize
, BlobReuseOnSmallOverwrite
) {
6221 if (string(GetParam()) != "bluestore")
6224 size_t block_size
= 4096;
6225 StartDeferred(block_size
);
6226 g_conf
->set_val("bluestore_max_blob_size", "65536");
6227 g_conf
->apply_changes(NULL
);
6229 ObjectStore::Sequencer
osr("test");
6232 ghobject_t
hoid(hobject_t("test_hint", "", CEPH_NOSNAP
, 0, -1, ""));
6234 const PerfCounters
* logger
= store
->get_perf_counters();
6237 ObjectStore::Transaction t
;
6238 t
.create_collection(cid
, 0);
6239 r
= apply_transaction(store
, &osr
, std::move(t
));
6243 ObjectStore::Transaction t
;
6246 bl
.append(std::string(block_size
, 'a'));
6247 t
.write(cid
, hoid
, 0, bl
.length(), bl
, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6248 t
.write(cid
, hoid
, block_size
* 2, bl
.length(), bl
,
6249 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6250 r
= apply_transaction(store
, &osr
, std::move(t
));
6254 // write small into the gap
6255 ObjectStore::Transaction t
;
6258 bl
.append(std::string(3, 'b'));
6259 t
.write(cid
, hoid
, block_size
+ 1, bl
.length(), bl
,
6260 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6261 r
= apply_transaction(store
, &osr
, std::move(t
));
6265 // We need to issue a read to trigger cache stat update that refresh
6266 // perf counters. additionally we need to wait some time for mempool
6267 // thread to update stats.
6269 bufferlist bl
, expected
;
6270 r
= store
->read(cid
, hoid
, 0, block_size
* 3, bl
);
6271 ASSERT_EQ(r
, (int)block_size
* 3);
6272 expected
.append(string(block_size
, 'a'));
6273 expected
.append(string(1, 0));
6274 expected
.append(string(3, 'b'));
6275 expected
.append(string(block_size
- 4, 0));
6276 expected
.append(string(block_size
, 'a'));
6277 ASSERT_TRUE(bl_eq(expected
, bl
));
6279 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 1u);
6280 ASSERT_EQ(logger
->get(l_bluestore_extents
), 3u);
6283 ObjectStore::Transaction t
;
6284 t
.remove(cid
, hoid
);
6285 t
.remove_collection(cid
);
6286 cerr
<< "Cleaning" << std::endl
;
6287 r
= apply_transaction(store
, &osr
, std::move(t
));
6290 g_conf
->set_val("bluestore_max_blob_size", "0");
6293 // The test case to reproduce an issue when write happens
6294 // to a zero space between the extents sharing the same spanning blob
6295 // with unloaded shard map.
6296 // Second extent might be filled with zeros this way due to wrong result
6297 // returned by has_any_extents() call in do_write_small. The latter is caused
6298 // by incompletly loaded extent map.
6299 TEST_P(StoreTestSpecificAUSize
, SmallWriteOnShardedExtents
) {
6300 if (string(GetParam()) != "bluestore")
6303 size_t block_size
= 0x10000;
6304 StartDeferred(block_size
);
6306 g_conf
->set_val("bluestore_csum_type", "xxhash64");
6307 g_conf
->set_val("bluestore_max_target_blob", "524288"); // for sure
6309 g_conf
->apply_changes(NULL
);
6311 ObjectStore::Sequencer
osr("test");
6314 ghobject_t
hoid1(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
6317 ObjectStore::Transaction t
;
6318 t
.create_collection(cid
, 0);
6319 r
= apply_transaction(store
, &osr
, std::move(t
));
6323 //doing some tricks to have sharded extents/spanning objects
6324 ObjectStore::Transaction t
;
6327 bl
.append(std::string(0x80000, 'a'));
6328 t
.write(cid
, hoid1
, 0, bl
.length(), bl
, 0);
6329 t
.zero(cid
, hoid1
, 0x719e0, 0x75b0 );
6330 r
= apply_transaction(store
, &osr
, std::move(t
));
6333 bl2
.append(std::string(0x70000, 'b'));
6334 t
.write(cid
, hoid1
, 0, bl2
.length(), bl2
, 0);
6335 t
.zero(cid
, hoid1
, 0, 0x50000);
6336 r
= apply_transaction(store
, &osr
, std::move(t
));
6344 // do a write to zero space in between some extents sharing the same blob
6345 ObjectStore::Transaction t
;
6348 bl
.append(std::string(0x6520, 'c'));
6349 t
.write(cid
, hoid1
, 0x71c00, bl
.length(), bl
, 0);
6351 r
= apply_transaction(store
, &osr
, std::move(t
));
6356 ObjectStore::Transaction t
;
6357 bufferlist bl
, expected
;
6359 r
= store
->read(cid
, hoid1
, 0x70000, 0x9c00, bl
);
6360 ASSERT_EQ(r
, (int)0x9c00);
6361 expected
.append(string(0x19e0, 'a'));
6362 expected
.append(string(0x220, 0));
6363 expected
.append(string(0x6520, 'c'));
6364 expected
.append(string(0xe70, 0));
6365 expected
.append(string(0xc70, 'a'));
6366 ASSERT_TRUE(bl_eq(expected
, bl
));
6372 ObjectStore::Transaction t
;
6373 t
.remove(cid
, hoid1
);
6374 t
.remove_collection(cid
);
6375 cerr
<< "Cleaning" << std::endl
;
6376 r
= apply_transaction(store
, &osr
, std::move(t
));
6379 g_conf
->set_val("bluestore_max_target_blob", "524288");
6380 g_conf
->set_val("bluestore_csum_type", "crc32c");
6383 #endif //#if defined(HAVE_LIBAIO)
6385 TEST_P(StoreTest
, KVDBHistogramTest
) {
6386 if (string(GetParam()) != "bluestore")
6389 ObjectStore::Sequencer
osr("test");
6393 string
base("testobj.");
6395 bufferptr
ap(0x1000);
6396 memset(ap
.c_str(), 'a', 0x1000);
6399 ObjectStore::Transaction t
;
6400 t
.create_collection(cid
, 0);
6401 r
= apply_transaction(store
, &osr
, std::move(t
));
6404 for (int i
= 0; i
< NUM_OBJS
; ++i
) {
6405 ObjectStore::Transaction t
;
6407 snprintf(buf
, sizeof(buf
), "%d", i
);
6408 ghobject_t
hoid(hobject_t(sobject_t(base
+ string(buf
), CEPH_NOSNAP
)));
6409 t
.write(cid
, hoid
, 0, 0x1000, a
);
6410 r
= apply_transaction(store
, &osr
, std::move(t
));
6414 Formatter
*f
= Formatter::create("store_test", "json-pretty", "json-pretty");
6415 store
->generate_db_histogram(f
);
6420 TEST_P(StoreTest
, KVDBStatsTest
) {
6421 if (string(GetParam()) != "bluestore")
6424 g_conf
->set_val("rocksdb_perf", "true");
6425 g_conf
->set_val("rocksdb_collect_compaction_stats", "true");
6426 g_conf
->set_val("rocksdb_collect_extended_stats","true");
6427 g_conf
->set_val("rocksdb_collect_memory_stats","true");
6428 g_ceph_context
->_conf
->apply_changes(NULL
);
6429 int r
= store
->umount();
6431 r
= store
->mount(); //to force rocksdb stats
6434 ObjectStore::Sequencer
osr("test");
6437 string
base("testobj.");
6439 bufferptr
ap(0x1000);
6440 memset(ap
.c_str(), 'a', 0x1000);
6443 ObjectStore::Transaction t
;
6444 t
.create_collection(cid
, 0);
6445 r
= apply_transaction(store
, &osr
, std::move(t
));
6448 for (int i
= 0; i
< NUM_OBJS
; ++i
) {
6449 ObjectStore::Transaction t
;
6451 snprintf(buf
, sizeof(buf
), "%d", i
);
6452 ghobject_t
hoid(hobject_t(sobject_t(base
+ string(buf
), CEPH_NOSNAP
)));
6453 t
.write(cid
, hoid
, 0, 0x1000, a
);
6454 r
= apply_transaction(store
, &osr
, std::move(t
));
6458 Formatter
*f
= Formatter::create("store_test", "json-pretty", "json-pretty");
6459 store
->get_db_statistics(f
);
6462 g_conf
->set_val("rocksdb_perf", "false");
6463 g_conf
->set_val("rocksdb_collect_compaction_stats", "false");
6464 g_conf
->set_val("rocksdb_collect_extended_stats","false");
6465 g_conf
->set_val("rocksdb_collect_memory_stats","false");
6468 #if defined(HAVE_LIBAIO)
6469 TEST_P(StoreTestSpecificAUSize
, garbageCollection
) {
6470 ObjectStore::Sequencer
osr("test");
6473 int buf_len
= 256 * 1024;
6474 int overlap_offset
= 64 * 1024;
6475 int write_offset
= buf_len
;
6476 if (string(GetParam()) != "bluestore")
6479 #define WRITE_AT(offset, _length) {\
6480 ObjectStore::Transaction t;\
6481 if ((uint64_t)_length != bl.length()) { \
6482 buffer::ptr p(bl.c_str(), _length);\
6484 bl_tmp.push_back(p);\
6485 t.write(cid, hoid, offset, bl_tmp.length(), bl_tmp);\
6487 t.write(cid, hoid, offset, bl.length(), bl);\
6489 r = apply_transaction(store, &osr, std::move(t));\
6493 StartDeferred(65536);
6495 g_conf
->set_val("bluestore_compression_max_blob_size", "524288");
6496 g_conf
->set_val("bluestore_compression_min_blob_size", "262144");
6497 g_conf
->set_val("bluestore_compression_mode", "force");
6498 g_conf
->apply_changes(NULL
);
6500 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
6503 r
= store
->read(cid
, hoid
, 0, 5, in
);
6504 ASSERT_EQ(-ENOENT
, r
);
6507 ObjectStore::Transaction t
;
6508 t
.create_collection(cid
, 0);
6509 cerr
<< "Creating collection " << cid
<< std::endl
;
6510 r
= apply_transaction(store
, &osr
, std::move(t
));
6515 data
.resize(buf_len
);
6519 bool exists
= store
->exists(cid
, hoid
);
6520 ASSERT_TRUE(!exists
);
6522 ObjectStore::Transaction t
;
6524 cerr
<< "Creating object " << hoid
<< std::endl
;
6525 r
= apply_transaction(store
, &osr
, std::move(t
));
6528 exists
= store
->exists(cid
, hoid
);
6529 ASSERT_EQ(true, exists
);
6533 for(size_t i
= 0; i
< data
.size(); i
++)
6539 struct store_statfs_t statfs
;
6540 WRITE_AT(0, buf_len
);
6541 int r
= store
->statfs(&statfs
);
6543 ASSERT_EQ(statfs
.compressed_allocated
, 0x10000);
6546 struct store_statfs_t statfs
;
6547 WRITE_AT(write_offset
- 2 * overlap_offset
, buf_len
);
6548 int r
= store
->statfs(&statfs
);
6550 ASSERT_EQ(statfs
.compressed_allocated
, 0x20000);
6551 const PerfCounters
* counters
= store
->get_perf_counters();
6552 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0u);
6556 struct store_statfs_t statfs
;
6557 WRITE_AT(write_offset
- overlap_offset
, buf_len
);
6558 int r
= store
->statfs(&statfs
);
6560 ASSERT_EQ(statfs
.compressed_allocated
, 0x20000);
6561 const PerfCounters
* counters
= store
->get_perf_counters();
6562 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0x10000u
);
6565 struct store_statfs_t statfs
;
6566 WRITE_AT(write_offset
- 3 * overlap_offset
, buf_len
);
6567 int r
= store
->statfs(&statfs
);
6569 ASSERT_EQ(statfs
.compressed_allocated
, 0x20000);
6570 const PerfCounters
* counters
= store
->get_perf_counters();
6571 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0x20000u
);
6574 struct store_statfs_t statfs
;
6575 WRITE_AT(write_offset
+ 1, overlap_offset
-1);
6576 int r
= store
->statfs(&statfs
);
6578 ASSERT_EQ(statfs
.compressed_allocated
, 0x20000);
6579 const PerfCounters
* counters
= store
->get_perf_counters();
6580 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0x20000u
);
6583 struct store_statfs_t statfs
;
6584 WRITE_AT(write_offset
+ 1, overlap_offset
);
6585 int r
= store
->statfs(&statfs
);
6587 ASSERT_EQ(statfs
.compressed_allocated
, 0x10000);
6588 const PerfCounters
* counters
= store
->get_perf_counters();
6589 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0x3ffffu
);
6592 struct store_statfs_t statfs
;
6593 WRITE_AT(0, buf_len
-1);
6594 int r
= store
->statfs(&statfs
);
6596 ASSERT_EQ(statfs
.compressed_allocated
, 0x10000);
6597 const PerfCounters
* counters
= store
->get_perf_counters();
6598 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0x40001u
);
6600 g_conf
->set_val("bluestore_gc_enable_total_threshold", "1"); //forbid GC when saving = 0
6602 struct store_statfs_t statfs
;
6603 WRITE_AT(1, overlap_offset
-2);
6604 WRITE_AT(overlap_offset
* 2 + 1, overlap_offset
-2);
6605 int r
= store
->statfs(&statfs
);
6607 ASSERT_EQ(statfs
.compressed_allocated
, 0x10000);
6608 const PerfCounters
* counters
= store
->get_perf_counters();
6609 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0x40001u
);
6612 struct store_statfs_t statfs
;
6613 WRITE_AT(overlap_offset
+ 1, overlap_offset
-2);
6614 int r
= store
->statfs(&statfs
);
6616 ASSERT_EQ(statfs
.compressed_allocated
, 0x0);
6617 const PerfCounters
* counters
= store
->get_perf_counters();
6618 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0x40007u
);
6621 ObjectStore::Transaction t
;
6622 t
.remove(cid
, hoid
);
6623 cerr
<< "Cleaning" << std::endl
;
6624 r
= apply_transaction(store
, &osr
, std::move(t
));
6628 g_conf
->set_val("bluestore_gc_enable_total_threshold", "0");
6629 g_conf
->set_val("bluestore_compression_min_blob_size", "0");
6630 g_conf
->set_val("bluestore_compression_max_blob_size", "0");
6631 g_conf
->set_val("bluestore_compression_mode", "none");
6632 g_conf
->apply_changes(NULL
);
6636 int main(int argc
, char **argv
) {
6637 vector
<const char*> args
;
6638 argv_to_vec(argc
, (const char **)argv
, args
);
6641 auto cct
= global_init(NULL
, args
, CEPH_ENTITY_TYPE_CLIENT
,
6642 CODE_ENVIRONMENT_UTILITY
, 0);
6643 common_init_finish(g_ceph_context
);
6645 g_ceph_context
->_conf
->set_val("osd_journal_size", "400");
6646 g_ceph_context
->_conf
->set_val("filestore_index_retry_probability", "0.5");
6647 g_ceph_context
->_conf
->set_val("filestore_op_thread_timeout", "1000");
6648 g_ceph_context
->_conf
->set_val("filestore_op_thread_suicide_timeout", "10000");
6649 //g_ceph_context->_conf->set_val("filestore_fiemap", "true");
6650 g_ceph_context
->_conf
->set_val("bluestore_fsck_on_mount", "true");
6651 g_ceph_context
->_conf
->set_val("bluestore_fsck_on_umount", "true");
6652 g_ceph_context
->_conf
->set_val("bluestore_debug_misc", "true");
6653 g_ceph_context
->_conf
->set_val("bluestore_debug_small_allocations", "4");
6654 g_ceph_context
->_conf
->set_val("bluestore_debug_freelist", "true");
6655 g_ceph_context
->_conf
->set_val("bluestore_clone_cow", "true");
6656 g_ceph_context
->_conf
->set_val("bluestore_max_alloc_size", "196608");
6658 // set small cache sizes so we see trimming during Synthetic tests
6659 g_ceph_context
->_conf
->set_val("bluestore_cache_size", "4000000");
6661 // very short *_max prealloc so that we fall back to async submits
6662 g_ceph_context
->_conf
->set_val("bluestore_blobid_prealloc", "10");
6663 g_ceph_context
->_conf
->set_val("bluestore_nid_prealloc", "10");
6664 g_ceph_context
->_conf
->set_val("bluestore_debug_randomize_serial_transaction",
6667 g_ceph_context
->_conf
->set_val("bdev_debug_aio", "true");
6669 // specify device size
6670 g_ceph_context
->_conf
->set_val("bluestore_block_size", "10240000000");
6672 g_ceph_context
->_conf
->set_val(
6673 "enable_experimental_unrecoverable_data_corrupting_features", "*");
6674 g_ceph_context
->_conf
->apply_changes(NULL
);
6676 ::testing::InitGoogleTest(&argc
, argv
);
6677 return RUN_ALL_TESTS();
6682 * compile-command: "cd ../.. ; make ceph_test_objectstore &&
6683 * ./ceph_test_objectstore \
6684 * --gtest_filter=*.collect_metadata* --log-to-stderr=true --debug-filestore=20