1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
20 #include <sys/mount.h>
21 #include <boost/scoped_ptr.hpp>
22 #include <boost/random/mersenne_twister.hpp>
23 #include <boost/random/uniform_int.hpp>
24 #include <boost/random/binomial_distribution.hpp>
25 #include <gtest/gtest.h>
27 #include "os/ObjectStore.h"
28 #include "os/filestore/FileStore.h"
29 #if defined(HAVE_LIBAIO)
30 #include "os/bluestore/BlueStore.h"
32 #include "include/Context.h"
33 #include "common/ceph_argparse.h"
34 #include "global/global_init.h"
35 #include "common/Mutex.h"
36 #include "common/Cond.h"
37 #include "common/errno.h"
38 #include "include/stringify.h"
39 #include "include/coredumpctl.h"
41 #include "include/unordered_map.h"
42 #include "store_test_fixture.h"
45 typedef boost::mt11213b gen_type
;
47 #define dout_context g_ceph_context
49 #if GTEST_HAS_PARAM_TEST
51 static bool bl_eq(bufferlist
& expected
, bufferlist
& actual
)
53 if (expected
.contents_equal(actual
))
57 if(expected
.length() != actual
.length()) {
58 cout
<< "--- buffer lengths mismatch " << std::hex
59 << "expected 0x" << expected
.length() << " != actual 0x"
60 << actual
.length() << std::dec
<< std::endl
;
61 derr
<< "--- buffer lengths mismatch " << std::hex
62 << "expected 0x" << expected
.length() << " != actual 0x"
63 << actual
.length() << std::dec
<< dendl
;
65 auto len
= MIN(expected
.length(), actual
.length());
66 while ( first
<len
&& expected
[first
] == actual
[first
])
69 while (last
> 0 && expected
[last
-1] == actual
[last
-1])
72 cout
<< "--- buffer mismatch between offset 0x" << std::hex
<< first
73 << " and 0x" << last
<< ", total 0x" << len
<< std::dec
75 derr
<< "--- buffer mismatch between offset 0x" << std::hex
<< first
76 << " and 0x" << last
<< ", total 0x" << len
<< std::dec
78 cout
<< "--- expected:\n";
79 expected
.hexdump(cout
);
80 cout
<< "--- actual:\n";
89 int apply_transaction(
91 ObjectStore::Sequencer
*osr
,
92 ObjectStore::Transaction
&&t
) {
94 ObjectStore::Transaction t2
;
96 return store
->apply_transaction(osr
, std::move(t2
));
98 return store
->apply_transaction(osr
, std::move(t
));
103 bool sorted(const vector
<ghobject_t
> &in
) {
105 for (vector
<ghobject_t
>::const_iterator i
= in
.begin();
109 cout
<< start
<< " should follow " << *i
<< std::endl
;
117 class StoreTest
: public StoreTestFixture
,
118 public ::testing::WithParamInterface
<const char*> {
121 : StoreTestFixture(GetParam())
125 class StoreTestDeferredSetup
: public StoreTest
{
126 void SetUp() override
{
131 void DeferredSetup() {
138 class StoreTestSpecificAUSize
: public StoreTestDeferredSetup
{
143 boost::scoped_ptr
<ObjectStore
>& store
,
147 uint64_t align
)> MatrixTest
;
149 void StartDeferred(size_t min_alloc_size
) {
150 g_conf
->set_val("bluestore_min_alloc_size", stringify(min_alloc_size
));
154 void TearDown() override
{
155 g_conf
->set_val("bluestore_min_alloc_size", "0");
156 StoreTestDeferredSetup::TearDown();
160 // bluestore matrix testing
161 uint64_t max_write
= 40 * 1024;
162 uint64_t max_size
= 400 * 1024;
163 uint64_t alignment
= 0;
164 uint64_t num_ops
= 10000;
167 string
matrix_get(const char *k
) {
168 if (string(k
) == "max_write") {
169 return stringify(max_write
);
170 } else if (string(k
) == "max_size") {
171 return stringify(max_size
);
172 } else if (string(k
) == "alignment") {
173 return stringify(alignment
);
174 } else if (string(k
) == "num_ops") {
175 return stringify(num_ops
);
178 g_conf
->get_val(k
, &buf
, -1);
185 void matrix_set(const char *k
, const char *v
) {
186 if (string(k
) == "max_write") {
187 max_write
= atoll(v
);
188 } else if (string(k
) == "max_size") {
190 } else if (string(k
) == "alignment") {
191 alignment
= atoll(v
);
192 } else if (string(k
) == "num_ops") {
195 g_conf
->set_val(k
, v
);
199 void do_matrix_choose(const char *matrix
[][10],
200 int i
, int pos
, int num
,
201 boost::scoped_ptr
<ObjectStore
>& store
,
205 for (count
= 0; matrix
[i
][count
+1]; ++count
) ;
206 for (int j
= 1; matrix
[i
][j
]; ++j
) {
207 matrix_set(matrix
[i
][0], matrix
[i
][j
]);
208 do_matrix_choose(matrix
,
216 cout
<< "---------------------- " << (pos
+ 1) << " / " << num
217 << " ----------------------" << std::endl
;
218 for (unsigned k
=0; matrix
[k
][0]; ++k
) {
219 cout
<< " " << matrix
[k
][0] << " = " << matrix_get(matrix
[k
][0])
222 g_ceph_context
->_conf
->apply_changes(NULL
);
223 fn(store
, num_ops
, max_size
, max_write
, alignment
);
227 void do_matrix(const char *matrix
[][10],
228 boost::scoped_ptr
<ObjectStore
>& store
,
230 map
<string
,string
> old
;
231 for (unsigned i
=0; matrix
[i
][0]; ++i
) {
232 old
[matrix
[i
][0]] = matrix_get(matrix
[i
][0]);
234 cout
<< "saved config options " << old
<< std::endl
;
236 if (strcmp(matrix
[0][0], "bluestore_min_alloc_size") == 0) {
238 for (count
= 0; matrix
[0][count
+1]; ++count
) ;
239 for (size_t j
= 1; matrix
[0][j
]; ++j
) {
243 StartDeferred(strtoll(matrix
[0][j
], NULL
, 10));
244 do_matrix_choose(matrix
, 1, j
- 1, count
, store
, fn
);
248 do_matrix_choose(matrix
, 0, 0, 1, store
, fn
);
251 cout
<< "restoring config options " << old
<< std::endl
;
253 cout
<< " " << p
.first
<< " = " << p
.second
<< std::endl
;
254 matrix_set(p
.first
.c_str(), p
.second
.c_str());
256 g_ceph_context
->_conf
->apply_changes(NULL
);
261 TEST_P(StoreTest
, collect_metadata
) {
262 map
<string
,string
> pm
;
263 store
->collect_metadata(&pm
);
264 if (GetParam() == string("filestore")) {
265 ASSERT_NE(pm
.count("filestore_backend"), 0u);
266 ASSERT_NE(pm
.count("filestore_f_type"), 0u);
267 ASSERT_NE(pm
.count("backend_filestore_partition_path"), 0u);
268 ASSERT_NE(pm
.count("backend_filestore_dev_node"), 0u);
272 TEST_P(StoreTest
, Trivial
) {
275 TEST_P(StoreTest
, TrivialRemount
) {
276 int r
= store
->umount();
282 TEST_P(StoreTest
, SimpleRemount
) {
283 ObjectStore::Sequencer
osr("test");
285 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
286 ghobject_t
hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP
)));
288 bl
.append("1234512345");
291 cerr
<< "create collection + write" << std::endl
;
292 ObjectStore::Transaction t
;
293 t
.create_collection(cid
, 0);
294 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
295 r
= apply_transaction(store
, &osr
, std::move(t
));
303 ObjectStore::Transaction t
;
304 t
.write(cid
, hoid2
, 0, bl
.length(), bl
);
305 r
= apply_transaction(store
, &osr
, std::move(t
));
309 ObjectStore::Transaction t
;
311 t
.remove(cid
, hoid2
);
312 t
.remove_collection(cid
);
313 cerr
<< "remove collection" << std::endl
;
314 r
= apply_transaction(store
, &osr
, std::move(t
));
322 ObjectStore::Transaction t
;
323 t
.create_collection(cid
, 0);
324 r
= apply_transaction(store
, &osr
, std::move(t
));
326 bool exists
= store
->exists(cid
, hoid
);
327 ASSERT_TRUE(!exists
);
330 ObjectStore::Transaction t
;
331 t
.remove_collection(cid
);
332 cerr
<< "remove collection" << std::endl
;
333 r
= apply_transaction(store
, &osr
, std::move(t
));
338 TEST_P(StoreTest
, IORemount
) {
339 ObjectStore::Sequencer
osr("test");
342 bl
.append("1234512345");
345 cerr
<< "create collection + objects" << std::endl
;
346 ObjectStore::Transaction t
;
347 t
.create_collection(cid
, 0);
348 for (int n
=1; n
<=100; ++n
) {
349 ghobject_t
hoid(hobject_t(sobject_t("Object " + stringify(n
), CEPH_NOSNAP
)));
350 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
352 r
= apply_transaction(store
, &osr
, std::move(t
));
357 cout
<< "overwrites" << std::endl
;
358 for (int n
=1; n
<=100; ++n
) {
359 ObjectStore::Transaction t
;
360 ghobject_t
hoid(hobject_t(sobject_t("Object " + stringify(n
), CEPH_NOSNAP
)));
361 t
.write(cid
, hoid
, 1, bl
.length(), bl
);
362 r
= apply_transaction(store
, &osr
, std::move(t
));
371 ObjectStore::Transaction t
;
372 for (int n
=1; n
<=100; ++n
) {
373 ghobject_t
hoid(hobject_t(sobject_t("Object " + stringify(n
), CEPH_NOSNAP
)));
376 t
.remove_collection(cid
);
377 r
= apply_transaction(store
, &osr
, std::move(t
));
382 TEST_P(StoreTest
, UnprintableCharsName
) {
383 ObjectStore::Sequencer
osr("test");
385 string name
= "funnychars_";
386 for (unsigned i
= 0; i
< 256; ++i
) {
389 ghobject_t
oid(hobject_t(sobject_t(name
, CEPH_NOSNAP
)));
392 cerr
<< "create collection + object" << std::endl
;
393 ObjectStore::Transaction t
;
394 t
.create_collection(cid
, 0);
396 r
= apply_transaction(store
, &osr
, std::move(t
));
404 cout
<< "removing" << std::endl
;
405 ObjectStore::Transaction t
;
407 t
.remove_collection(cid
);
408 r
= apply_transaction(store
, &osr
, std::move(t
));
413 TEST_P(StoreTest
, FiemapEmpty
) {
414 ObjectStore::Sequencer
osr("test");
417 ghobject_t
oid(hobject_t(sobject_t("fiemap_object", CEPH_NOSNAP
)));
419 ObjectStore::Transaction t
;
420 t
.create_collection(cid
, 0);
422 t
.truncate(cid
, oid
, 100000);
423 r
= apply_transaction(store
, &osr
, std::move(t
));
428 store
->fiemap(cid
, oid
, 0, 100000, bl
);
429 map
<uint64_t,uint64_t> m
, e
;
430 bufferlist::iterator p
= bl
.begin();
432 cout
<< " got " << m
<< std::endl
;
434 EXPECT_TRUE(m
== e
|| m
.empty());
437 ObjectStore::Transaction t
;
439 t
.remove_collection(cid
);
440 cerr
<< "remove collection" << std::endl
;
441 r
= apply_transaction(store
, &osr
, std::move(t
));
446 TEST_P(StoreTest
, FiemapHoles
) {
447 ObjectStore::Sequencer
osr("test");
448 const uint64_t MAX_EXTENTS
= 4000;
449 const uint64_t SKIP_STEP
= 65536;
452 ghobject_t
oid(hobject_t(sobject_t("fiemap_object", CEPH_NOSNAP
)));
456 ObjectStore::Transaction t
;
457 t
.create_collection(cid
, 0);
459 for (uint64_t i
= 0; i
< MAX_EXTENTS
; i
++)
460 t
.write(cid
, oid
, SKIP_STEP
* i
, 3, bl
);
461 r
= apply_transaction(store
, &osr
, std::move(t
));
465 //fiemap test from 0 to SKIP_STEP * (MAX_EXTENTS - 1) + 3
467 store
->fiemap(cid
, oid
, 0, SKIP_STEP
* (MAX_EXTENTS
- 1) + 3, bl
);
468 map
<uint64_t,uint64_t> m
, e
;
469 bufferlist::iterator p
= bl
.begin();
471 cout
<< " got " << m
<< std::endl
;
472 ASSERT_TRUE(!m
.empty());
474 bool extents_exist
= true;
475 if (m
.size() == MAX_EXTENTS
) {
476 for (uint64_t i
= 0; i
< MAX_EXTENTS
; i
++)
477 extents_exist
= extents_exist
&& m
.count(SKIP_STEP
*i
);
479 ASSERT_TRUE((m
.size() == 1 &&
480 m
[0] > SKIP_STEP
* (MAX_EXTENTS
- 1)) ||
481 (m
.size() == MAX_EXTENTS
&& extents_exist
));
483 // fiemap test from SKIP_STEP to SKIP_STEP * (MAX_EXTENTS - 2) + 3
484 // reset bufferlist and map
488 store
->fiemap(cid
, oid
, SKIP_STEP
, SKIP_STEP
* (MAX_EXTENTS
- 2) + 3, bl
);
491 cout
<< " got " << m
<< std::endl
;
492 ASSERT_TRUE(!m
.empty());
493 ASSERT_GE(m
[SKIP_STEP
], 3u);
494 extents_exist
= true;
495 if (m
.size() == (MAX_EXTENTS
- 2)) {
496 for (uint64_t i
= 1; i
< MAX_EXTENTS
- 1; i
++)
497 extents_exist
= extents_exist
&& m
.count(SKIP_STEP
*i
);
499 ASSERT_TRUE((m
.size() == 1 &&
500 m
[SKIP_STEP
] > SKIP_STEP
* (MAX_EXTENTS
- 2)) ||
501 (m
.size() == (MAX_EXTENTS
- 1) && extents_exist
));
504 ObjectStore::Transaction t
;
506 t
.remove_collection(cid
);
507 cerr
<< "remove collection" << std::endl
;
508 r
= apply_transaction(store
, &osr
, std::move(t
));
513 TEST_P(StoreTest
, SimpleMetaColTest
) {
514 ObjectStore::Sequencer
osr("test");
518 ObjectStore::Transaction t
;
519 t
.create_collection(cid
, 0);
520 cerr
<< "create collection" << std::endl
;
521 r
= apply_transaction(store
, &osr
, std::move(t
));
525 ObjectStore::Transaction t
;
526 t
.remove_collection(cid
);
527 cerr
<< "remove collection" << std::endl
;
528 r
= apply_transaction(store
, &osr
, std::move(t
));
532 ObjectStore::Transaction t
;
533 t
.create_collection(cid
, 0);
534 cerr
<< "add collection" << std::endl
;
535 r
= apply_transaction(store
, &osr
, std::move(t
));
539 ObjectStore::Transaction t
;
540 t
.remove_collection(cid
);
541 cerr
<< "remove collection" << std::endl
;
542 r
= apply_transaction(store
, &osr
, std::move(t
));
547 TEST_P(StoreTest
, SimplePGColTest
) {
548 ObjectStore::Sequencer
osr("test");
549 coll_t
cid(spg_t(pg_t(1,2), shard_id_t::NO_SHARD
));
552 ObjectStore::Transaction t
;
553 t
.create_collection(cid
, 4);
554 cerr
<< "create collection" << std::endl
;
555 r
= apply_transaction(store
, &osr
, std::move(t
));
559 ObjectStore::Transaction t
;
560 t
.remove_collection(cid
);
561 cerr
<< "remove collection" << std::endl
;
562 r
= apply_transaction(store
, &osr
, std::move(t
));
566 ObjectStore::Transaction t
;
567 t
.create_collection(cid
, 4);
568 cerr
<< "add collection" << std::endl
;
569 r
= apply_transaction(store
, &osr
, std::move(t
));
573 ObjectStore::Transaction t
;
574 t
.remove_collection(cid
);
575 cerr
<< "remove collection" << std::endl
;
576 r
= apply_transaction(store
, &osr
, std::move(t
));
581 TEST_P(StoreTest
, SimpleColPreHashTest
) {
582 ObjectStore::Sequencer
osr("test");
583 // Firstly we will need to revert the value making sure
584 // collection hint actually works
585 int merge_threshold
= g_ceph_context
->_conf
->filestore_merge_threshold
;
586 std::ostringstream oss
;
587 if (merge_threshold
> 0) {
588 oss
<< "-" << merge_threshold
;
589 g_ceph_context
->_conf
->set_val("filestore_merge_threshold", oss
.str().c_str());
592 uint32_t pg_num
= 128;
594 boost::uniform_int
<> pg_id_range(0, pg_num
);
595 gen_type
rng(time(NULL
));
596 int pg_id
= pg_id_range(rng
);
598 int objs_per_folder
= abs(merge_threshold
) * 16 * g_ceph_context
->_conf
->filestore_split_multiple
;
599 boost::uniform_int
<> folders_range(5, 256);
600 uint64_t expected_num_objs
= (uint64_t)objs_per_folder
* (uint64_t)folders_range(rng
);
602 coll_t
cid(spg_t(pg_t(pg_id
, 15), shard_id_t::NO_SHARD
));
605 // Create a collection along with a hint
606 ObjectStore::Transaction t
;
607 t
.create_collection(cid
, 5);
608 cerr
<< "create collection" << std::endl
;
610 ::encode(pg_num
, hint
);
611 ::encode(expected_num_objs
, hint
);
612 t
.collection_hint(cid
, ObjectStore::Transaction::COLL_HINT_EXPECTED_NUM_OBJECTS
, hint
);
613 cerr
<< "collection hint" << std::endl
;
614 r
= apply_transaction(store
, &osr
, std::move(t
));
618 // Remove the collection
619 ObjectStore::Transaction t
;
620 t
.remove_collection(cid
);
621 cerr
<< "remove collection" << std::endl
;
622 r
= apply_transaction(store
, &osr
, std::move(t
));
625 // Revert the config change so that it does not affect the split/merge tests
626 if (merge_threshold
> 0) {
628 oss
<< merge_threshold
;
629 g_ceph_context
->_conf
->set_val("filestore_merge_threshold", oss
.str().c_str());
633 TEST_P(StoreTest
, SmallBlockWrites
) {
634 ObjectStore::Sequencer
osr("test");
637 ghobject_t
hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP
)));
639 ObjectStore::Transaction t
;
640 t
.create_collection(cid
, 0);
641 cerr
<< "Creating collection " << cid
<< std::endl
;
642 r
= apply_transaction(store
, &osr
, std::move(t
));
646 bufferptr
ap(0x1000);
647 memset(ap
.c_str(), 'a', 0x1000);
650 bufferptr
bp(0x1000);
651 memset(bp
.c_str(), 'b', 0x1000);
654 bufferptr
cp(0x1000);
655 memset(cp
.c_str(), 'c', 0x1000);
657 bufferptr
zp(0x1000);
662 ObjectStore::Transaction t
;
663 t
.write(cid
, hoid
, 0, 0x1000, a
);
664 r
= apply_transaction(store
, &osr
, std::move(t
));
668 r
= store
->read(cid
, hoid
, 0, 0x4000, in
);
669 ASSERT_EQ(0x1000, r
);
671 ASSERT_TRUE(bl_eq(exp
, in
));
674 ObjectStore::Transaction t
;
675 t
.write(cid
, hoid
, 0x1000, 0x1000, b
);
676 r
= apply_transaction(store
, &osr
, std::move(t
));
680 r
= store
->read(cid
, hoid
, 0, 0x4000, in
);
681 ASSERT_EQ(0x2000, r
);
684 ASSERT_TRUE(bl_eq(exp
, in
));
687 ObjectStore::Transaction t
;
688 t
.write(cid
, hoid
, 0x3000, 0x1000, c
);
689 r
= apply_transaction(store
, &osr
, std::move(t
));
693 r
= store
->read(cid
, hoid
, 0, 0x4000, in
);
694 ASSERT_EQ(0x4000, r
);
699 ASSERT_TRUE(bl_eq(exp
, in
));
702 ObjectStore::Transaction t
;
703 t
.write(cid
, hoid
, 0x2000, 0x1000, a
);
704 r
= apply_transaction(store
, &osr
, std::move(t
));
708 r
= store
->read(cid
, hoid
, 0, 0x4000, in
);
709 ASSERT_EQ(0x4000, r
);
714 ASSERT_TRUE(bl_eq(exp
, in
));
717 ObjectStore::Transaction t
;
718 t
.write(cid
, hoid
, 0, 0x1000, c
);
719 r
= apply_transaction(store
, &osr
, std::move(t
));
724 r
= store
->read(cid
, hoid
, 0, 0x4000, in
);
725 ASSERT_EQ(0x4000, r
);
730 ASSERT_TRUE(bl_eq(exp
, in
));
733 ObjectStore::Transaction t
;
735 t
.remove_collection(cid
);
736 cerr
<< "Cleaning" << std::endl
;
737 r
= apply_transaction(store
, &osr
, std::move(t
));
742 TEST_P(StoreTest
, BufferCacheReadTest
) {
743 ObjectStore::Sequencer
osr("test");
746 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
749 r
= store
->read(cid
, hoid
, 0, 5, in
);
750 ASSERT_EQ(-ENOENT
, r
);
753 ObjectStore::Transaction t
;
754 t
.create_collection(cid
, 0);
755 cerr
<< "Creating collection " << cid
<< std::endl
;
756 r
= apply_transaction(store
, &osr
, std::move(t
));
760 bool exists
= store
->exists(cid
, hoid
);
761 ASSERT_TRUE(!exists
);
763 ObjectStore::Transaction t
;
765 cerr
<< "Creating object " << hoid
<< std::endl
;
766 r
= apply_transaction(store
, &osr
, std::move(t
));
769 exists
= store
->exists(cid
, hoid
);
770 ASSERT_EQ(true, exists
);
773 ObjectStore::Transaction t
;
774 bufferlist bl
, newdata
;
776 t
.write(cid
, hoid
, 0, 5, bl
);
777 t
.write(cid
, hoid
, 10, 5, bl
);
778 cerr
<< "TwinWrite" << std::endl
;
779 r
= apply_transaction(store
, &osr
, std::move(t
));
782 r
= store
->read(cid
, hoid
, 0, 15, newdata
);
787 expected
.append_zero(5);
789 ASSERT_TRUE(bl_eq(expected
, newdata
));
792 //overwrite over the same extents
794 ObjectStore::Transaction t
;
795 bufferlist bl
, newdata
;
797 t
.write(cid
, hoid
, 0, 5, bl
);
798 t
.write(cid
, hoid
, 10, 5, bl
);
799 cerr
<< "TwinWrite" << std::endl
;
800 r
= apply_transaction(store
, &osr
, std::move(t
));
803 r
= store
->read(cid
, hoid
, 0, 15, newdata
);
808 expected
.append_zero(5);
810 ASSERT_TRUE(bl_eq(expected
, newdata
));
813 //additional write to an unused region of some blob
815 ObjectStore::Transaction t
;
816 bufferlist bl2
, newdata
;
817 bl2
.append("1234567890");
819 t
.write(cid
, hoid
, 20, bl2
.length(), bl2
);
820 cerr
<< "Append" << std::endl
;
821 r
= apply_transaction(store
, &osr
, std::move(t
));
824 r
= store
->read(cid
, hoid
, 0, 30, newdata
);
828 expected
.append("edcba");
829 expected
.append_zero(5);
830 expected
.append("edcba");
831 expected
.append_zero(5);
832 expected
.append(bl2
);
834 ASSERT_TRUE(bl_eq(expected
, newdata
));
837 //additional write to an unused region of some blob and partial owerite over existing extents
839 ObjectStore::Transaction t
;
840 bufferlist bl
, bl2
, bl3
, newdata
;
842 bl2
.append("1234567890");
845 t
.write(cid
, hoid
, 30, bl2
.length(), bl2
);
846 t
.write(cid
, hoid
, 1, bl
.length(), bl
);
847 t
.write(cid
, hoid
, 13, bl3
.length(), bl3
);
848 cerr
<< "TripleWrite" << std::endl
;
849 r
= apply_transaction(store
, &osr
, std::move(t
));
852 r
= store
->read(cid
, hoid
, 0, 40, newdata
);
856 expected
.append("eDCBa");
857 expected
.append_zero(5);
858 expected
.append("edcBA");
859 expected
.append_zero(5);
860 expected
.append(bl2
);
861 expected
.append(bl2
);
863 ASSERT_TRUE(bl_eq(expected
, newdata
));
868 void doCompressionTest( boost::scoped_ptr
<ObjectStore
>& store
)
870 ObjectStore::Sequencer
osr("test");
873 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
876 r
= store
->read(cid
, hoid
, 0, 5, in
);
877 ASSERT_EQ(-ENOENT
, r
);
880 ObjectStore::Transaction t
;
881 t
.create_collection(cid
, 0);
882 cerr
<< "Creating collection " << cid
<< std::endl
;
883 r
= apply_transaction(store
, &osr
, std::move(t
));
887 bool exists
= store
->exists(cid
, hoid
);
888 ASSERT_TRUE(!exists
);
890 ObjectStore::Transaction t
;
892 cerr
<< "Creating object " << hoid
<< std::endl
;
893 r
= apply_transaction(store
, &osr
, std::move(t
));
896 exists
= store
->exists(cid
, hoid
);
897 ASSERT_EQ(true, exists
);
900 data
.resize(0x10000 * 4);
901 for(size_t i
= 0;i
< data
.size(); i
++)
904 ObjectStore::Transaction t
;
905 bufferlist bl
, newdata
;
907 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
908 cerr
<< "CompressibleData (4xAU) Write" << std::endl
;
909 r
= apply_transaction(store
, &osr
, std::move(t
));
912 r
= store
->read(cid
, hoid
, 0, data
.size() , newdata
);
914 ASSERT_EQ(r
, (int)data
.size());
917 expected
.append(data
);
918 ASSERT_TRUE(bl_eq(expected
, newdata
));
921 r
= store
->read(cid
, hoid
, 0, 711 , newdata
);
925 expected
.append(data
.substr(0,711));
926 ASSERT_TRUE(bl_eq(expected
, newdata
));
929 r
= store
->read(cid
, hoid
, 0xf00f, data
.size(), newdata
);
930 ASSERT_EQ(r
, int(data
.size() - 0xf00f) );
933 expected
.append(data
.substr(0xf00f));
934 ASSERT_TRUE(bl_eq(expected
, newdata
));
937 struct store_statfs_t statfs
;
938 int r
= store
->statfs(&statfs
);
940 ASSERT_EQ(statfs
.stored
, (unsigned)data
.size());
941 ASSERT_LE(statfs
.compressed
, (unsigned)data
.size());
942 ASSERT_EQ(statfs
.compressed_original
, (unsigned)data
.size());
943 ASSERT_LE(statfs
.compressed_allocated
, (unsigned)data
.size());
947 data2
.resize(0x10000 * 4 - 0x9000);
948 for(size_t i
= 0;i
< data2
.size(); i
++)
949 data2
[i
] = (i
+1) / 256;
951 ObjectStore::Transaction t
;
952 bufferlist bl
, newdata
;
954 t
.write(cid
, hoid
, 0x8000, bl
.length(), bl
);
955 cerr
<< "CompressibleData partial overwrite" << std::endl
;
956 r
= apply_transaction(store
, &osr
, std::move(t
));
959 r
= store
->read(cid
, hoid
, 0, 0x10000, newdata
);
960 ASSERT_EQ(r
, (int)0x10000);
963 expected
.append(data
.substr(0, 0x8000));
964 expected
.append(data2
.substr(0, 0x8000));
965 ASSERT_TRUE(bl_eq(expected
, newdata
));
968 r
= store
->read(cid
, hoid
, 0x9000, 711 , newdata
);
972 expected
.append(data2
.substr(0x1000,711));
973 ASSERT_TRUE(bl_eq(expected
, newdata
));
976 r
= store
->read(cid
, hoid
, 0x0, 0x40000, newdata
);
977 ASSERT_EQ(r
, int(0x40000) );
980 expected
.append(data
.substr(0, 0x8000));
981 expected
.append(data2
.substr(0, 0x37000));
982 expected
.append(data
.substr(0x3f000, 0x1000));
983 ASSERT_TRUE(bl_eq(expected
, newdata
));
986 data2
.resize(0x3f000);
987 for(size_t i
= 0;i
< data2
.size(); i
++)
988 data2
[i
] = (i
+2) / 256;
990 ObjectStore::Transaction t
;
991 bufferlist bl
, newdata
;
993 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
994 cerr
<< "CompressibleData partial overwrite, two extents overlapped, single one to be removed" << std::endl
;
995 r
= apply_transaction(store
, &osr
, std::move(t
));
998 r
= store
->read(cid
, hoid
, 0, 0x3e000 - 1, newdata
);
999 ASSERT_EQ(r
, (int)0x3e000 - 1);
1001 bufferlist expected
;
1002 expected
.append(data2
.substr(0, 0x3e000 - 1));
1003 ASSERT_TRUE(bl_eq(expected
, newdata
));
1006 r
= store
->read(cid
, hoid
, 0x3e000-1, 0x2001, newdata
);
1007 ASSERT_EQ(r
, 0x2001);
1009 bufferlist expected
;
1010 expected
.append(data2
.substr(0x3e000-1, 0x1001));
1011 expected
.append(data
.substr(0x3f000, 0x1000));
1012 ASSERT_TRUE(bl_eq(expected
, newdata
));
1015 r
= store
->read(cid
, hoid
, 0x0, 0x40000, newdata
);
1016 ASSERT_EQ(r
, int(0x40000) );
1018 bufferlist expected
;
1019 expected
.append(data2
.substr(0, 0x3f000));
1020 expected
.append(data
.substr(0x3f000, 0x1000));
1021 ASSERT_TRUE(bl_eq(expected
, newdata
));
1024 data
.resize(0x1001);
1025 for(size_t i
= 0;i
< data
.size(); i
++)
1026 data
[i
] = (i
+3) / 256;
1028 ObjectStore::Transaction t
;
1029 bufferlist bl
, newdata
;
1031 t
.write(cid
, hoid
, 0x3f000-1, bl
.length(), bl
);
1032 cerr
<< "Small chunk partial overwrite, two extents overlapped, single one to be removed" << std::endl
;
1033 r
= apply_transaction(store
, &osr
, std::move(t
));
1036 r
= store
->read(cid
, hoid
, 0x3e000, 0x2000, newdata
);
1037 ASSERT_EQ(r
, (int)0x2000);
1039 bufferlist expected
;
1040 expected
.append(data2
.substr(0x3e000, 0x1000 - 1));
1041 expected
.append(data
.substr(0, 0x1001));
1042 ASSERT_TRUE(bl_eq(expected
, newdata
));
1046 ObjectStore::Transaction t
;
1047 t
.remove(cid
, hoid
);
1048 cerr
<< "Cleaning object" << std::endl
;
1049 r
= apply_transaction(store
, &osr
, std::move(t
));
1053 EXPECT_EQ(store
->umount(), 0);
1054 EXPECT_EQ(store
->mount(), 0);
1055 auto orig_min_blob_size
= g_conf
->bluestore_compression_min_blob_size
;
1057 g_conf
->set_val("bluestore_compression_min_blob_size", "262144");
1058 g_ceph_context
->_conf
->apply_changes(NULL
);
1059 data
.resize(0x10000*6);
1061 for(size_t i
= 0;i
< data
.size(); i
++)
1063 ObjectStore::Transaction t
;
1064 bufferlist bl
, newdata
;
1066 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
1067 cerr
<< "CompressibleData large blob" << std::endl
;
1068 r
= apply_transaction(store
, &osr
, std::move(t
));
1072 EXPECT_EQ(store
->umount(), 0);
1073 EXPECT_EQ(store
->mount(), 0);
1076 ObjectStore::Transaction t
;
1077 t
.remove(cid
, hoid
);
1078 t
.remove_collection(cid
);
1079 cerr
<< "Cleaning" << std::endl
;
1080 r
= apply_transaction(store
, &osr
, std::move(t
));
1083 g_conf
->set_val("bluestore_compression_min_blob_size", stringify(orig_min_blob_size
));
1084 g_ceph_context
->_conf
->apply_changes(NULL
);
1087 TEST_P(StoreTest
, CompressionTest
) {
1088 if (string(GetParam()) != "bluestore")
1091 g_conf
->set_val("bluestore_compression_algorithm", "snappy");
1092 g_conf
->set_val("bluestore_compression_mode", "force");
1094 g_ceph_context
->_conf
->apply_changes(NULL
);
1096 doCompressionTest(store
);
1098 g_conf
->set_val("bluestore_compression_algorithm", "zlib");
1099 g_conf
->set_val("bluestore_compression_mode", "force");
1100 g_ceph_context
->_conf
->apply_changes(NULL
);
1102 doCompressionTest(store
);
1104 g_conf
->set_val("bluestore_compression_algorithm", "snappy");
1105 g_conf
->set_val("bluestore_compression_mode", "none");
1106 g_ceph_context
->_conf
->apply_changes(NULL
);
1109 TEST_P(StoreTest
, SimpleObjectTest
) {
1110 ObjectStore::Sequencer
osr("test");
1113 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
1116 r
= store
->read(cid
, hoid
, 0, 5, in
);
1117 ASSERT_EQ(-ENOENT
, r
);
1120 ObjectStore::Transaction t
;
1121 t
.create_collection(cid
, 0);
1122 cerr
<< "Creating collection " << cid
<< std::endl
;
1123 r
= apply_transaction(store
, &osr
, std::move(t
));
1127 bool exists
= store
->exists(cid
, hoid
);
1128 ASSERT_TRUE(!exists
);
1130 ObjectStore::Transaction t
;
1132 cerr
<< "Creating object " << hoid
<< std::endl
;
1133 r
= apply_transaction(store
, &osr
, std::move(t
));
1136 exists
= store
->exists(cid
, hoid
);
1137 ASSERT_EQ(true, exists
);
1140 ObjectStore::Transaction t
;
1141 t
.remove(cid
, hoid
);
1143 cerr
<< "Remove then create" << std::endl
;
1144 r
= apply_transaction(store
, &osr
, std::move(t
));
1148 ObjectStore::Transaction t
;
1149 bufferlist bl
, orig
;
1152 t
.remove(cid
, hoid
);
1153 t
.write(cid
, hoid
, 0, 5, bl
);
1154 cerr
<< "Remove then create" << std::endl
;
1155 r
= apply_transaction(store
, &osr
, std::move(t
));
1159 r
= store
->read(cid
, hoid
, 0, 5, in
);
1161 ASSERT_TRUE(bl_eq(orig
, in
));
1164 ObjectStore::Transaction t
;
1169 t
.write(cid
, hoid
, 5, 5, bl
);
1170 cerr
<< "Append" << std::endl
;
1171 r
= apply_transaction(store
, &osr
, std::move(t
));
1175 r
= store
->read(cid
, hoid
, 0, 10, in
);
1177 ASSERT_TRUE(bl_eq(exp
, in
));
1180 ObjectStore::Transaction t
;
1182 bl
.append("abcdeabcde");
1184 t
.write(cid
, hoid
, 0, 10, bl
);
1185 cerr
<< "Full overwrite" << std::endl
;
1186 r
= apply_transaction(store
, &osr
, std::move(t
));
1190 r
= store
->read(cid
, hoid
, 0, 10, in
);
1192 ASSERT_TRUE(bl_eq(exp
, in
));
1195 ObjectStore::Transaction t
;
1198 t
.write(cid
, hoid
, 3, 5, bl
);
1199 cerr
<< "Partial overwrite" << std::endl
;
1200 r
= apply_transaction(store
, &osr
, std::move(t
));
1204 exp
.append("abcabcdede");
1205 r
= store
->read(cid
, hoid
, 0, 10, in
);
1208 ASSERT_TRUE(bl_eq(exp
, in
));
1212 ObjectStore::Transaction t
;
1215 t
.truncate(cid
, hoid
, 0);
1216 t
.write(cid
, hoid
, 5, 5, bl
);
1217 cerr
<< "Truncate + hole" << std::endl
;
1218 r
= apply_transaction(store
, &osr
, std::move(t
));
1222 ObjectStore::Transaction t
;
1225 t
.write(cid
, hoid
, 0, 5, bl
);
1226 cerr
<< "Reverse fill-in" << std::endl
;
1227 r
= apply_transaction(store
, &osr
, std::move(t
));
1232 exp
.append("abcdefghij");
1233 r
= store
->read(cid
, hoid
, 0, 10, in
);
1236 ASSERT_TRUE(bl_eq(exp
, in
));
1239 ObjectStore::Transaction t
;
1241 bl
.append("abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234");
1242 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
1243 cerr
<< "larger overwrite" << std::endl
;
1244 r
= apply_transaction(store
, &osr
, std::move(t
));
1248 r
= store
->read(cid
, hoid
, 0, bl
.length(), in
);
1249 ASSERT_EQ((int)bl
.length(), r
);
1251 ASSERT_TRUE(bl_eq(bl
, in
));
1255 bl
.append("abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234");
1257 //test: offset=len=0 mean read all data
1259 r
= store
->read(cid
, hoid
, 0, 0, in
);
1260 ASSERT_EQ((int)bl
.length(), r
);
1262 ASSERT_TRUE(bl_eq(bl
, in
));
1265 //verifying unaligned csums
1266 std::string
s1("1"), s2(0x1000, '2'), s3("00");
1268 ObjectStore::Transaction t
;
1272 t
.truncate(cid
, hoid
, 0);
1273 t
.write(cid
, hoid
, 0x1000-1, bl
.length(), bl
);
1274 cerr
<< "Write unaligned csum, stage 1" << std::endl
;
1275 r
= apply_transaction(store
, &osr
, std::move(t
));
1279 bufferlist in
, exp1
, exp2
, exp3
;
1283 r
= store
->read(cid
, hoid
, 0x1000-1, 1, in
);
1285 ASSERT_TRUE(bl_eq(exp1
, in
));
1287 r
= store
->read(cid
, hoid
, 0x1000, 0x1000, in
);
1288 ASSERT_EQ(0x1000, r
);
1289 ASSERT_TRUE(bl_eq(exp2
, in
));
1292 ObjectStore::Transaction t
;
1295 t
.write(cid
, hoid
, 1, bl
.length(), bl
);
1296 cerr
<< "Write unaligned csum, stage 2" << std::endl
;
1297 r
= apply_transaction(store
, &osr
, std::move(t
));
1301 r
= store
->read(cid
, hoid
, 1, 2, in
);
1303 ASSERT_TRUE(bl_eq(exp3
, in
));
1305 r
= store
->read(cid
, hoid
, 0x1000-1, 1, in
);
1307 ASSERT_TRUE(bl_eq(exp1
, in
));
1309 r
= store
->read(cid
, hoid
, 0x1000, 0x1000, in
);
1310 ASSERT_EQ(0x1000, r
);
1311 ASSERT_TRUE(bl_eq(exp2
, in
));
1316 ObjectStore::Transaction t
;
1317 t
.remove(cid
, hoid
);
1318 t
.remove_collection(cid
);
1319 cerr
<< "Cleaning" << std::endl
;
1320 r
= apply_transaction(store
, &osr
, std::move(t
));
1325 #if defined(HAVE_LIBAIO)
1326 TEST_P(StoreTestSpecificAUSize
, BluestoreStatFSTest
) {
1327 if(string(GetParam()) != "bluestore")
1329 StartDeferred(65536);
1330 g_conf
->set_val("bluestore_compression_mode", "force");
1332 // just a big number to disble gc
1333 g_conf
->set_val("bluestore_gc_enable_total_threshold", "100000");
1334 g_conf
->apply_changes(NULL
);
1337 ObjectStore::Sequencer
osr("test");
1339 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
1340 ghobject_t hoid2
= hoid
;
1341 hoid2
.hobj
.snap
= 1;
1344 r
= store
->read(cid
, hoid
, 0, 5, in
);
1345 ASSERT_EQ(-ENOENT
, r
);
1348 ObjectStore::Transaction t
;
1349 t
.create_collection(cid
, 0);
1350 cerr
<< "Creating collection " << cid
<< std::endl
;
1351 r
= apply_transaction(store
, &osr
, std::move(t
));
1355 bool exists
= store
->exists(cid
, hoid
);
1356 ASSERT_TRUE(!exists
);
1358 ObjectStore::Transaction t
;
1360 cerr
<< "Creating object " << hoid
<< std::endl
;
1361 r
= apply_transaction(store
, &osr
, std::move(t
));
1364 exists
= store
->exists(cid
, hoid
);
1365 ASSERT_EQ(true, exists
);
1368 struct store_statfs_t statfs
;
1369 int r
= store
->statfs(&statfs
);
1371 ASSERT_EQ( 0u, statfs
.allocated
);
1372 ASSERT_EQ( 0u, statfs
.stored
);
1373 ASSERT_EQ(g_conf
->bluestore_block_size
, statfs
.total
);
1374 ASSERT_TRUE(statfs
.available
> 0u && statfs
.available
< g_conf
->bluestore_block_size
);
1376 EXPECT_EQ(store
->umount(), 0);
1377 EXPECT_EQ(store
->mount(), 0);
1380 ObjectStore::Transaction t
;
1383 t
.write(cid
, hoid
, 0, 5, bl
);
1384 cerr
<< "Append 5 bytes" << std::endl
;
1385 r
= apply_transaction(store
, &osr
, std::move(t
));
1388 struct store_statfs_t statfs
;
1389 int r
= store
->statfs(&statfs
);
1391 ASSERT_EQ(5, statfs
.stored
);
1392 ASSERT_EQ(0x10000, statfs
.allocated
);
1393 ASSERT_EQ(0, statfs
.compressed
);
1394 ASSERT_EQ(0, statfs
.compressed_original
);
1395 ASSERT_EQ(0, statfs
.compressed_allocated
);
1397 EXPECT_EQ(store
->umount(), 0);
1398 EXPECT_EQ(store
->mount(), 0);
1401 ObjectStore::Transaction t
;
1402 std::string
s(0x30000, 'a');
1405 t
.write(cid
, hoid
, 0x10000, bl
.length(), bl
);
1406 cerr
<< "Append 0x30000 compressible bytes" << std::endl
;
1407 r
= apply_transaction(store
, &osr
, std::move(t
));
1410 struct store_statfs_t statfs
;
1411 int r
= store
->statfs(&statfs
);
1413 ASSERT_EQ(0x30005, statfs
.stored
);
1414 ASSERT_EQ(0x30000, statfs
.allocated
);
1415 ASSERT_LE(statfs
.compressed
, 0x10000);
1416 ASSERT_EQ(0x20000, statfs
.compressed_original
);
1417 ASSERT_EQ(statfs
.compressed_allocated
, 0x10000);
1419 EXPECT_EQ(store
->umount(), 0);
1420 EXPECT_EQ(store
->mount(), 0);
1423 ObjectStore::Transaction t
;
1424 t
.zero(cid
, hoid
, 1, 3);
1425 t
.zero(cid
, hoid
, 0x20000, 9);
1426 cerr
<< "Punch hole at 1~3, 0x20000~9" << std::endl
;
1427 r
= apply_transaction(store
, &osr
, std::move(t
));
1430 struct store_statfs_t statfs
;
1431 int r
= store
->statfs(&statfs
);
1433 ASSERT_EQ(0x30005 - 3 - 9, statfs
.stored
);
1434 ASSERT_EQ(0x30000, statfs
.allocated
);
1435 ASSERT_LE(statfs
.compressed
, 0x10000);
1436 ASSERT_EQ(0x20000 - 9, statfs
.compressed_original
);
1437 ASSERT_EQ(statfs
.compressed_allocated
, 0x10000);
1439 EXPECT_EQ(store
->umount(), 0);
1440 EXPECT_EQ(store
->mount(), 0);
1443 ObjectStore::Transaction t
;
1444 std::string
s(0x1000, 'b');
1447 t
.write(cid
, hoid
, 1, bl
.length(), bl
);
1448 t
.write(cid
, hoid
, 0x10001, bl
.length(), bl
);
1449 cerr
<< "Overwrite first and second(compressible) extents" << std::endl
;
1450 r
= apply_transaction(store
, &osr
, std::move(t
));
1453 struct store_statfs_t statfs
;
1454 int r
= store
->statfs(&statfs
);
1456 ASSERT_EQ(0x30001 - 9 + 0x1000, statfs
.stored
);
1457 ASSERT_EQ(0x40000, statfs
.allocated
);
1458 ASSERT_LE(statfs
.compressed
, 0x10000);
1459 ASSERT_EQ(0x20000 - 9 - 0x1000, statfs
.compressed_original
);
1460 ASSERT_EQ(statfs
.compressed_allocated
, 0x10000);
1462 EXPECT_EQ(store
->umount(), 0);
1463 EXPECT_EQ(store
->mount(), 0);
1466 ObjectStore::Transaction t
;
1467 std::string
s(0x10000, 'c');
1470 t
.write(cid
, hoid
, 0x10000, bl
.length(), bl
);
1471 t
.write(cid
, hoid
, 0x20000, bl
.length(), bl
);
1472 t
.write(cid
, hoid
, 0x30000, bl
.length(), bl
);
1473 cerr
<< "Overwrite compressed extent with 3 uncompressible ones" << std::endl
;
1474 r
= apply_transaction(store
, &osr
, std::move(t
));
1477 struct store_statfs_t statfs
;
1478 int r
= store
->statfs(&statfs
);
1480 ASSERT_EQ(0x30000 + 0x1001, statfs
.stored
);
1481 ASSERT_EQ(0x40000, statfs
.allocated
);
1482 ASSERT_LE(statfs
.compressed
, 0);
1483 ASSERT_EQ(0, statfs
.compressed_original
);
1484 ASSERT_EQ(0, statfs
.compressed_allocated
);
1486 EXPECT_EQ(store
->umount(), 0);
1487 EXPECT_EQ(store
->mount(), 0);
1490 ObjectStore::Transaction t
;
1491 t
.zero(cid
, hoid
, 0, 0x40000);
1492 cerr
<< "Zero object" << std::endl
;
1493 r
= apply_transaction(store
, &osr
, std::move(t
));
1495 struct store_statfs_t statfs
;
1496 int r
= store
->statfs(&statfs
);
1498 ASSERT_EQ(0u, statfs
.allocated
);
1499 ASSERT_EQ(0u, statfs
.stored
);
1500 ASSERT_EQ(0u, statfs
.compressed_original
);
1501 ASSERT_EQ(0u, statfs
.compressed
);
1502 ASSERT_EQ(0u, statfs
.compressed_allocated
);
1504 EXPECT_EQ(store
->umount(), 0);
1505 EXPECT_EQ(store
->mount(), 0);
1508 ObjectStore::Transaction t
;
1509 std::string
s(0x10000, 'c');
1514 bl
.append(s
.substr(0, 0x10000-2));
1515 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
1516 cerr
<< "Yet another compressible write" << std::endl
;
1517 r
= apply_transaction(store
, &osr
, std::move(t
));
1519 struct store_statfs_t statfs
;
1520 r
= store
->statfs(&statfs
);
1522 ASSERT_EQ(0x40000 - 2, statfs
.stored
);
1523 ASSERT_EQ(0x30000, statfs
.allocated
);
1524 ASSERT_LE(statfs
.compressed
, 0x10000);
1525 ASSERT_EQ(0x20000, statfs
.compressed_original
);
1526 ASSERT_EQ(0x10000, statfs
.compressed_allocated
);
1528 EXPECT_EQ(store
->umount(), 0);
1529 EXPECT_EQ(store
->mount(), 0);
1532 struct store_statfs_t statfs
;
1533 r
= store
->statfs(&statfs
);
1536 ObjectStore::Transaction t
;
1537 t
.clone(cid
, hoid
, hoid2
);
1538 cerr
<< "Clone compressed objecte" << std::endl
;
1539 r
= apply_transaction(store
, &osr
, std::move(t
));
1541 struct store_statfs_t statfs2
;
1542 r
= store
->statfs(&statfs2
);
1544 ASSERT_GT(statfs2
.stored
, statfs
.stored
);
1545 ASSERT_EQ(statfs2
.allocated
, statfs
.allocated
);
1546 ASSERT_GT(statfs2
.compressed
, statfs
.compressed
);
1547 ASSERT_GT(statfs2
.compressed_original
, statfs
.compressed_original
);
1548 ASSERT_EQ(statfs2
.compressed_allocated
, statfs
.compressed_allocated
);
1552 ObjectStore::Transaction t
;
1553 t
.remove(cid
, hoid
);
1554 t
.remove(cid
, hoid2
);
1555 t
.remove_collection(cid
);
1556 cerr
<< "Cleaning" << std::endl
;
1557 r
= apply_transaction(store
, &osr
, std::move(t
));
1560 struct store_statfs_t statfs
;
1561 r
= store
->statfs(&statfs
);
1563 ASSERT_EQ( 0u, statfs
.allocated
);
1564 ASSERT_EQ( 0u, statfs
.stored
);
1565 ASSERT_EQ( 0u, statfs
.compressed_original
);
1566 ASSERT_EQ( 0u, statfs
.compressed
);
1567 ASSERT_EQ( 0u, statfs
.compressed_allocated
);
1569 g_conf
->set_val("bluestore_gc_enable_total_threshold", "0");
1570 g_conf
->set_val("bluestore_compression_mode", "none");
1571 g_ceph_context
->_conf
->apply_changes(NULL
);
1574 TEST_P(StoreTestSpecificAUSize
, BluestoreFragmentedBlobTest
) {
1575 if(string(GetParam()) != "bluestore")
1577 StartDeferred(0x10000);
1579 ObjectStore::Sequencer
osr("test");
1582 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
1584 ObjectStore::Transaction t
;
1585 t
.create_collection(cid
, 0);
1586 cerr
<< "Creating collection " << cid
<< std::endl
;
1587 r
= apply_transaction(store
, &osr
, std::move(t
));
1591 bool exists
= store
->exists(cid
, hoid
);
1592 ASSERT_TRUE(!exists
);
1594 ObjectStore::Transaction t
;
1596 cerr
<< "Creating object " << hoid
<< std::endl
;
1597 r
= apply_transaction(store
, &osr
, std::move(t
));
1600 exists
= store
->exists(cid
, hoid
);
1601 ASSERT_EQ(true, exists
);
1604 struct store_statfs_t statfs
;
1605 int r
= store
->statfs(&statfs
);
1607 ASSERT_EQ(g_conf
->bluestore_block_size
, statfs
.total
);
1608 ASSERT_EQ(0u, statfs
.allocated
);
1609 ASSERT_EQ(0u, statfs
.stored
);
1610 ASSERT_TRUE(statfs
.available
> 0u && statfs
.available
< g_conf
->bluestore_block_size
);
1613 data
.resize(0x10000 * 3);
1615 ObjectStore::Transaction t
;
1616 for(size_t i
= 0;i
< data
.size(); i
++)
1617 data
[i
] = i
/ 256 + 1;
1618 bufferlist bl
, newdata
;
1620 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
1621 t
.zero(cid
, hoid
, 0x10000, 0x10000);
1622 cerr
<< "Append 3*0x10000 bytes and punch a hole 0x10000~10000" << std::endl
;
1623 r
= apply_transaction(store
, &osr
, std::move(t
));
1626 struct store_statfs_t statfs
;
1627 int r
= store
->statfs(&statfs
);
1629 ASSERT_EQ(0x20000, statfs
.stored
);
1630 ASSERT_EQ(0x20000, statfs
.allocated
);
1632 r
= store
->read(cid
, hoid
, 0, data
.size(), newdata
);
1633 ASSERT_EQ(r
, (int)data
.size());
1635 bufferlist expected
;
1636 expected
.append(data
.substr(0, 0x10000));
1637 expected
.append(string(0x10000, 0));
1638 expected
.append(data
.substr(0x20000, 0x10000));
1639 ASSERT_TRUE(bl_eq(expected
, newdata
));
1643 r
= store
->read(cid
, hoid
, 1, data
.size()-2, newdata
);
1644 ASSERT_EQ(r
, (int)data
.size()-2);
1646 bufferlist expected
;
1647 expected
.append(data
.substr(1, 0x10000-1));
1648 expected
.append(string(0x10000, 0));
1649 expected
.append(data
.substr(0x20000, 0x10000 - 1));
1650 ASSERT_TRUE(bl_eq(expected
, newdata
));
1655 EXPECT_EQ(store
->umount(), 0);
1656 EXPECT_EQ(store
->mount(), 0);
1659 ObjectStore::Transaction t
;
1660 std::string
data2(3, 'b');
1661 bufferlist bl
, newdata
;
1663 t
.write(cid
, hoid
, 0x20000, bl
.length(), bl
);
1664 cerr
<< "Write 3 bytes after the hole" << std::endl
;
1665 r
= apply_transaction(store
, &osr
, std::move(t
));
1668 struct store_statfs_t statfs
;
1669 int r
= store
->statfs(&statfs
);
1671 ASSERT_EQ(0x20000, statfs
.allocated
);
1672 ASSERT_EQ(0x20000, statfs
.stored
);
1674 r
= store
->read(cid
, hoid
, 0x20000-1, 21, newdata
);
1675 ASSERT_EQ(r
, (int)21);
1677 bufferlist expected
;
1678 expected
.append(string(0x1, 0));
1679 expected
.append(string(data2
));
1680 expected
.append(data
.substr(0x20003, 21-4));
1681 ASSERT_TRUE(bl_eq(expected
, newdata
));
1686 EXPECT_EQ(store
->umount(), 0);
1687 EXPECT_EQ(store
->mount(), 0);
1690 ObjectStore::Transaction t
;
1691 std::string
data2(3, 'a');
1692 bufferlist bl
, newdata
;
1694 t
.write(cid
, hoid
, 0x10000+1, bl
.length(), bl
);
1695 cerr
<< "Write 3 bytes to the hole" << std::endl
;
1696 r
= apply_transaction(store
, &osr
, std::move(t
));
1699 struct store_statfs_t statfs
;
1700 int r
= store
->statfs(&statfs
);
1702 ASSERT_EQ(0x30000, statfs
.allocated
);
1703 ASSERT_EQ(0x20003, statfs
.stored
);
1705 r
= store
->read(cid
, hoid
, 0x10000-1, 0x10000+22, newdata
);
1706 ASSERT_EQ(r
, (int)0x10000+22);
1708 bufferlist expected
;
1709 expected
.append(data
.substr(0x10000-1, 1));
1710 expected
.append(string(0x1, 0));
1711 expected
.append(data2
);
1712 expected
.append(string(0x10000-4, 0));
1713 expected
.append(string(0x3, 'b'));
1714 expected
.append(data
.substr(0x20004, 21-3));
1715 ASSERT_TRUE(bl_eq(expected
, newdata
));
1720 ObjectStore::Transaction t
;
1721 bufferlist bl
, newdata
;
1722 bl
.append(string(0x30000, 'c'));
1723 t
.write(cid
, hoid
, 0, 0x30000, bl
);
1724 t
.zero(cid
, hoid
, 0, 0x10000);
1725 t
.zero(cid
, hoid
, 0x20000, 0x10000);
1726 cerr
<< "Rewrite an object and create two holes at the begining and the end" << std::endl
;
1727 r
= apply_transaction(store
, &osr
, std::move(t
));
1730 struct store_statfs_t statfs
;
1731 int r
= store
->statfs(&statfs
);
1733 ASSERT_EQ(0x10000, statfs
.allocated
);
1734 ASSERT_EQ(0x10000, statfs
.stored
);
1736 r
= store
->read(cid
, hoid
, 0, 0x30000, newdata
);
1737 ASSERT_EQ(r
, (int)0x30000);
1739 bufferlist expected
;
1740 expected
.append(string(0x10000, 0));
1741 expected
.append(string(0x10000, 'c'));
1742 expected
.append(string(0x10000, 0));
1743 ASSERT_TRUE(bl_eq(expected
, newdata
));
1749 EXPECT_EQ(store
->umount(), 0);
1750 EXPECT_EQ(store
->mount(), 0);
1753 ObjectStore::Transaction t
;
1754 t
.remove(cid
, hoid
);
1755 t
.remove_collection(cid
);
1756 cerr
<< "Cleaning" << std::endl
;
1757 r
= apply_transaction(store
, &osr
, std::move(t
));
1760 struct store_statfs_t statfs
;
1761 r
= store
->statfs(&statfs
);
1763 ASSERT_EQ( 0u, statfs
.allocated
);
1764 ASSERT_EQ( 0u, statfs
.stored
);
1765 ASSERT_EQ( 0u, statfs
.compressed_original
);
1766 ASSERT_EQ( 0u, statfs
.compressed
);
1767 ASSERT_EQ( 0u, statfs
.compressed_allocated
);
1772 TEST_P(StoreTest
, ManySmallWrite
) {
1773 ObjectStore::Sequencer
osr("test");
1776 ghobject_t
a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
1777 ghobject_t
b(hobject_t(sobject_t("Object 2", CEPH_NOSNAP
)));
1779 ObjectStore::Transaction t
;
1780 t
.create_collection(cid
, 0);
1781 cerr
<< "Creating collection " << cid
<< std::endl
;
1782 r
= apply_transaction(store
, &osr
, std::move(t
));
1789 for (int i
=0; i
<100; ++i
) {
1790 ObjectStore::Transaction t
;
1791 t
.write(cid
, a
, i
*4096, 4096, bl
, 0);
1792 r
= apply_transaction(store
, &osr
, std::move(t
));
1795 for (int i
=0; i
<100; ++i
) {
1796 ObjectStore::Transaction t
;
1797 t
.write(cid
, b
, (rand() % 1024)*4096, 4096, bl
, 0);
1798 r
= apply_transaction(store
, &osr
, std::move(t
));
1802 ObjectStore::Transaction t
;
1805 t
.remove_collection(cid
);
1806 cerr
<< "Cleaning" << std::endl
;
1807 r
= apply_transaction(store
, &osr
, std::move(t
));
1812 TEST_P(StoreTest
, MultiSmallWriteSameBlock
) {
1813 ObjectStore::Sequencer
osr("test");
1816 ghobject_t
a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
1818 ObjectStore::Transaction t
;
1819 t
.create_collection(cid
, 0);
1820 cerr
<< "Creating collection " << cid
<< std::endl
;
1821 r
= apply_transaction(store
, &osr
, std::move(t
));
1827 // touch same block in both same transaction, tls, and pipelined txns
1829 ObjectStore::Transaction t
, u
;
1830 t
.write(cid
, a
, 0, 5, bl
, 0);
1831 t
.write(cid
, a
, 5, 5, bl
, 0);
1832 t
.write(cid
, a
, 4094, 5, bl
, 0);
1833 t
.write(cid
, a
, 9000, 5, bl
, 0);
1834 u
.write(cid
, a
, 10, 5, bl
, 0);
1835 u
.write(cid
, a
, 7000, 5, bl
, 0);
1836 vector
<ObjectStore::Transaction
> v
= {t
, u
};
1837 store
->queue_transactions(&osr
, v
, nullptr, &c
);
1840 ObjectStore::Transaction t
, u
;
1841 t
.write(cid
, a
, 40, 5, bl
, 0);
1842 t
.write(cid
, a
, 45, 5, bl
, 0);
1843 t
.write(cid
, a
, 4094, 5, bl
, 0);
1844 t
.write(cid
, a
, 6000, 5, bl
, 0);
1845 u
.write(cid
, a
, 610, 5, bl
, 0);
1846 u
.write(cid
, a
, 11000, 5, bl
, 0);
1847 vector
<ObjectStore::Transaction
> v
= {t
, u
};
1848 store
->queue_transactions(&osr
, v
, nullptr, &d
);
1854 r
= store
->read(cid
, a
, 0, 16000, bl2
);
1858 ObjectStore::Transaction t
;
1860 t
.remove_collection(cid
);
1861 cerr
<< "Cleaning" << std::endl
;
1862 r
= apply_transaction(store
, &osr
, std::move(t
));
1867 TEST_P(StoreTest
, SmallSkipFront
) {
1868 ObjectStore::Sequencer
osr("test");
1871 ghobject_t
a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
1873 ObjectStore::Transaction t
;
1874 t
.create_collection(cid
, 0);
1875 cerr
<< "Creating collection " << cid
<< std::endl
;
1876 r
= apply_transaction(store
, &osr
, std::move(t
));
1880 ObjectStore::Transaction t
;
1882 t
.truncate(cid
, a
, 3000);
1883 r
= apply_transaction(store
, &osr
, std::move(t
));
1889 memset(bp
.c_str(), 1, 4096);
1891 ObjectStore::Transaction t
;
1892 t
.write(cid
, a
, 4096, 4096, bl
);
1893 r
= apply_transaction(store
, &osr
, std::move(t
));
1898 ASSERT_EQ(8192, store
->read(cid
, a
, 0, 8192, bl
));
1899 for (unsigned i
=0; i
<4096; ++i
)
1900 ASSERT_EQ(0, bl
[i
]);
1901 for (unsigned i
=4096; i
<8192; ++i
)
1902 ASSERT_EQ(1, bl
[i
]);
1905 ObjectStore::Transaction t
;
1907 t
.remove_collection(cid
);
1908 cerr
<< "Cleaning" << std::endl
;
1909 r
= apply_transaction(store
, &osr
, std::move(t
));
1914 TEST_P(StoreTest
, AppendDeferredVsTailCache
) {
1915 ObjectStore::Sequencer
osr("test");
1918 ghobject_t
a(hobject_t(sobject_t("fooo", CEPH_NOSNAP
)));
1920 ObjectStore::Transaction t
;
1921 t
.create_collection(cid
, 0);
1922 cerr
<< "Creating collection " << cid
<< std::endl
;
1923 r
= store
->apply_transaction(&osr
, std::move(t
));
1926 unsigned min_alloc
= g_conf
->bluestore_min_alloc_size
;
1927 g_conf
->set_val("bluestore_inject_deferred_apply_delay", "1.0");
1928 g_ceph_context
->_conf
->apply_changes(NULL
);
1929 unsigned size
= min_alloc
/ 3;
1930 bufferptr
bpa(size
);
1931 memset(bpa
.c_str(), 1, bpa
.length());
1935 ObjectStore::Transaction t
;
1936 t
.write(cid
, a
, 0, bla
.length(), bla
, 0);
1937 r
= store
->apply_transaction(&osr
, std::move(t
));
1941 // force cached tail to clear ...
1943 int r
= store
->umount();
1949 bufferptr
bpb(size
);
1950 memset(bpb
.c_str(), 2, bpb
.length());
1954 ObjectStore::Transaction t
;
1955 t
.write(cid
, a
, bla
.length(), blb
.length(), blb
, 0);
1956 r
= store
->apply_transaction(&osr
, std::move(t
));
1959 bufferptr
bpc(size
);
1960 memset(bpc
.c_str(), 3, bpc
.length());
1964 ObjectStore::Transaction t
;
1965 t
.write(cid
, a
, bla
.length() + blb
.length(), blc
.length(), blc
, 0);
1966 r
= store
->apply_transaction(&osr
, std::move(t
));
1975 ASSERT_EQ((int)final
.length(),
1976 store
->read(cid
, a
, 0, final
.length(), actual
));
1977 ASSERT_TRUE(bl_eq(final
, actual
));
1980 ObjectStore::Transaction t
;
1982 t
.remove_collection(cid
);
1983 cerr
<< "Cleaning" << std::endl
;
1984 r
= store
->apply_transaction(&osr
, std::move(t
));
1987 g_conf
->set_val("bluestore_inject_deferred_apply_delay", "0");
1988 g_ceph_context
->_conf
->apply_changes(NULL
);
1991 TEST_P(StoreTest
, AppendZeroTrailingSharedBlock
) {
1992 ObjectStore::Sequencer
osr("test");
1995 ghobject_t
a(hobject_t(sobject_t("fooo", CEPH_NOSNAP
)));
1999 ObjectStore::Transaction t
;
2000 t
.create_collection(cid
, 0);
2001 cerr
<< "Creating collection " << cid
<< std::endl
;
2002 r
= store
->apply_transaction(&osr
, std::move(t
));
2005 unsigned min_alloc
= g_conf
->bluestore_min_alloc_size
;
2006 unsigned size
= min_alloc
/ 3;
2007 bufferptr
bpa(size
);
2008 memset(bpa
.c_str(), 1, bpa
.length());
2011 // make sure there is some trailing gunk in the last block
2015 bt
.append("BADBADBADBAD");
2016 ObjectStore::Transaction t
;
2017 t
.write(cid
, a
, 0, bt
.length(), bt
, 0);
2018 r
= store
->apply_transaction(&osr
, std::move(t
));
2022 ObjectStore::Transaction t
;
2023 t
.truncate(cid
, a
, size
);
2024 r
= store
->apply_transaction(&osr
, std::move(t
));
2030 ObjectStore::Transaction t
;
2032 r
= store
->apply_transaction(&osr
, std::move(t
));
2036 // append with implicit zeroing
2037 bufferptr
bpb(size
);
2038 memset(bpb
.c_str(), 2, bpb
.length());
2042 ObjectStore::Transaction t
;
2043 t
.write(cid
, a
, min_alloc
* 3, blb
.length(), blb
, 0);
2044 r
= store
->apply_transaction(&osr
, std::move(t
));
2050 zeros
.append_zero(min_alloc
* 3 - size
);
2051 final
.append(zeros
);
2055 ASSERT_EQ((int)final
.length(),
2056 store
->read(cid
, a
, 0, final
.length(), actual
));
2057 final
.hexdump(cout
);
2058 actual
.hexdump(cout
);
2059 ASSERT_TRUE(bl_eq(final
, actual
));
2062 ObjectStore::Transaction t
;
2065 t
.remove_collection(cid
);
2066 cerr
<< "Cleaning" << std::endl
;
2067 r
= store
->apply_transaction(&osr
, std::move(t
));
2072 TEST_P(StoreTest
, SmallSequentialUnaligned
) {
2073 ObjectStore::Sequencer
osr("test");
2076 ghobject_t
a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
2078 ObjectStore::Transaction t
;
2079 t
.create_collection(cid
, 0);
2080 cerr
<< "Creating collection " << cid
<< std::endl
;
2081 r
= apply_transaction(store
, &osr
, std::move(t
));
2089 for (int i
=0; i
<1000; ++i
) {
2090 ObjectStore::Transaction t
;
2091 t
.write(cid
, a
, i
*len
, len
, bl
, 0);
2092 r
= apply_transaction(store
, &osr
, std::move(t
));
2096 ObjectStore::Transaction t
;
2098 t
.remove_collection(cid
);
2099 cerr
<< "Cleaning" << std::endl
;
2100 r
= apply_transaction(store
, &osr
, std::move(t
));
2105 TEST_P(StoreTest
, ManyBigWrite
) {
2106 ObjectStore::Sequencer
osr("test");
2109 ghobject_t
a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
2110 ghobject_t
b(hobject_t(sobject_t("Object 2", CEPH_NOSNAP
)));
2112 ObjectStore::Transaction t
;
2113 t
.create_collection(cid
, 0);
2114 cerr
<< "Creating collection " << cid
<< std::endl
;
2115 r
= apply_transaction(store
, &osr
, std::move(t
));
2119 bufferptr
bp(4 * 1048576);
2122 for (int i
=0; i
<10; ++i
) {
2123 ObjectStore::Transaction t
;
2124 t
.write(cid
, a
, i
*4*1048586, 4*1048576, bl
, 0);
2125 r
= apply_transaction(store
, &osr
, std::move(t
));
2129 for (int i
=0; i
<10; ++i
) {
2130 ObjectStore::Transaction t
;
2131 t
.write(cid
, b
, (rand() % 256)*4*1048576, 4*1048576, bl
, 0);
2132 r
= apply_transaction(store
, &osr
, std::move(t
));
2136 for (int i
=0; i
<10; ++i
) {
2137 ObjectStore::Transaction t
;
2138 t
.write(cid
, b
, (rand() % (256*4096))*1024, 4*1048576, bl
, 0);
2139 r
= apply_transaction(store
, &osr
, std::move(t
));
2143 for (int i
=0; i
<10; ++i
) {
2144 ObjectStore::Transaction t
;
2145 t
.zero(cid
, b
, (rand() % (256*4096))*1024, 16*1048576);
2146 r
= apply_transaction(store
, &osr
, std::move(t
));
2150 ObjectStore::Transaction t
;
2153 t
.remove_collection(cid
);
2154 cerr
<< "Cleaning" << std::endl
;
2155 r
= apply_transaction(store
, &osr
, std::move(t
));
2160 TEST_P(StoreTest
, BigWriteBigZero
) {
2161 ObjectStore::Sequencer
osr("test");
2164 ghobject_t
a(hobject_t(sobject_t("foo", CEPH_NOSNAP
)));
2166 ObjectStore::Transaction t
;
2167 t
.create_collection(cid
, 0);
2168 r
= apply_transaction(store
, &osr
, std::move(t
));
2172 bufferptr
bp(1048576);
2173 memset(bp
.c_str(), 'b', bp
.length());
2177 memset(sp
.c_str(), 's', sp
.length());
2180 ObjectStore::Transaction t
;
2181 t
.write(cid
, a
, 0, bl
.length(), bl
);
2182 r
= apply_transaction(store
, &osr
, std::move(t
));
2186 ObjectStore::Transaction t
;
2187 t
.zero(cid
, a
, bl
.length() / 4, bl
.length() / 2);
2188 r
= apply_transaction(store
, &osr
, std::move(t
));
2192 ObjectStore::Transaction t
;
2193 t
.write(cid
, a
, bl
.length() / 2, s
.length(), s
);
2194 r
= apply_transaction(store
, &osr
, std::move(t
));
2198 ObjectStore::Transaction t
;
2200 t
.remove_collection(cid
);
2201 r
= apply_transaction(store
, &osr
, std::move(t
));
2206 TEST_P(StoreTest
, MiscFragmentTests
) {
2207 ObjectStore::Sequencer
osr("test");
2210 ghobject_t
a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
2212 ObjectStore::Transaction t
;
2213 t
.create_collection(cid
, 0);
2214 cerr
<< "Creating collection " << cid
<< std::endl
;
2215 r
= apply_transaction(store
, &osr
, std::move(t
));
2219 bufferptr
bp(524288);
2223 ObjectStore::Transaction t
;
2224 t
.write(cid
, a
, 0, 524288, bl
, 0);
2225 r
= apply_transaction(store
, &osr
, std::move(t
));
2229 ObjectStore::Transaction t
;
2230 t
.write(cid
, a
, 1048576, 524288, bl
, 0);
2231 r
= apply_transaction(store
, &osr
, std::move(t
));
2236 int r
= store
->read(cid
, a
, 524288 + 131072, 1024, inbl
);
2238 ASSERT_EQ(inbl
.length(), 1024u);
2239 ASSERT_TRUE(inbl
.is_zero());
2242 ObjectStore::Transaction t
;
2243 t
.write(cid
, a
, 1048576 - 4096, 524288, bl
, 0);
2244 r
= apply_transaction(store
, &osr
, std::move(t
));
2248 ObjectStore::Transaction t
;
2250 t
.remove_collection(cid
);
2251 cerr
<< "Cleaning" << std::endl
;
2252 r
= apply_transaction(store
, &osr
, std::move(t
));
2258 TEST_P(StoreTest
, ZeroVsObjectSize
) {
2259 ObjectStore::Sequencer
osr("test");
2263 ghobject_t
hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP
)));
2265 ObjectStore::Transaction t
;
2266 t
.create_collection(cid
, 0);
2267 cerr
<< "Creating collection " << cid
<< std::endl
;
2268 r
= apply_transaction(store
, &osr
, std::move(t
));
2274 ObjectStore::Transaction t
;
2275 t
.write(cid
, hoid
, 0, 5, a
);
2276 r
= apply_transaction(store
, &osr
, std::move(t
));
2279 ASSERT_EQ(0, store
->stat(cid
, hoid
, &stat
));
2280 ASSERT_EQ(5, stat
.st_size
);
2282 ObjectStore::Transaction t
;
2283 t
.zero(cid
, hoid
, 1, 2);
2284 r
= apply_transaction(store
, &osr
, std::move(t
));
2287 ASSERT_EQ(0, store
->stat(cid
, hoid
, &stat
));
2288 ASSERT_EQ(5, stat
.st_size
);
2290 ObjectStore::Transaction t
;
2291 t
.zero(cid
, hoid
, 3, 200);
2292 r
= apply_transaction(store
, &osr
, std::move(t
));
2295 ASSERT_EQ(0, store
->stat(cid
, hoid
, &stat
));
2296 ASSERT_EQ(203, stat
.st_size
);
2298 ObjectStore::Transaction t
;
2299 t
.zero(cid
, hoid
, 100000, 200);
2300 r
= apply_transaction(store
, &osr
, std::move(t
));
2303 ASSERT_EQ(0, store
->stat(cid
, hoid
, &stat
));
2304 ASSERT_EQ(100200, stat
.st_size
);
2307 TEST_P(StoreTest
, ZeroLengthWrite
) {
2308 ObjectStore::Sequencer
osr("test");
2311 ghobject_t
hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP
)));
2313 ObjectStore::Transaction t
;
2314 t
.create_collection(cid
, 0);
2316 r
= apply_transaction(store
, &osr
, std::move(t
));
2320 ObjectStore::Transaction t
;
2322 t
.write(cid
, hoid
, 1048576, 0, empty
);
2323 r
= apply_transaction(store
, &osr
, std::move(t
));
2327 r
= store
->stat(cid
, hoid
, &stat
);
2329 ASSERT_EQ(0, stat
.st_size
);
2332 r
= store
->read(cid
, hoid
, 0, 1048576, newdata
);
2336 TEST_P(StoreTest
, ZeroLengthZero
) {
2337 ObjectStore::Sequencer
osr("test");
2340 ghobject_t
hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP
)));
2342 ObjectStore::Transaction t
;
2343 t
.create_collection(cid
, 0);
2345 r
= apply_transaction(store
, &osr
, std::move(t
));
2349 ObjectStore::Transaction t
;
2350 t
.zero(cid
, hoid
, 1048576, 0);
2351 r
= apply_transaction(store
, &osr
, std::move(t
));
2355 r
= store
->stat(cid
, hoid
, &stat
);
2357 ASSERT_EQ(0, stat
.st_size
);
2360 r
= store
->read(cid
, hoid
, 0, 1048576, newdata
);
2364 TEST_P(StoreTest
, SimpleAttrTest
) {
2365 ObjectStore::Sequencer
osr("test");
2368 ghobject_t
hoid(hobject_t(sobject_t("attr object 1", CEPH_NOSNAP
)));
2369 bufferlist val
, val2
;
2370 val
.append("value");
2371 val
.append("value2");
2374 map
<string
,bufferptr
> aset
;
2375 r
= store
->getattr(cid
, hoid
, "nofoo", bp
);
2376 ASSERT_EQ(-ENOENT
, r
);
2377 r
= store
->getattrs(cid
, hoid
, aset
);
2378 ASSERT_EQ(-ENOENT
, r
);
2381 ObjectStore::Transaction t
;
2382 t
.create_collection(cid
, 0);
2383 r
= apply_transaction(store
, &osr
, std::move(t
));
2389 int r
= store
->collection_empty(cid
, &empty
);
2395 r
= store
->getattr(cid
, hoid
, "nofoo", bp
);
2396 ASSERT_EQ(-ENOENT
, r
);
2399 ObjectStore::Transaction t
;
2401 t
.setattr(cid
, hoid
, "foo", val
);
2402 t
.setattr(cid
, hoid
, "bar", val2
);
2403 r
= apply_transaction(store
, &osr
, std::move(t
));
2409 int r
= store
->collection_empty(cid
, &empty
);
2411 ASSERT_TRUE(!empty
);
2415 r
= store
->getattr(cid
, hoid
, "nofoo", bp
);
2416 ASSERT_EQ(-ENODATA
, r
);
2418 r
= store
->getattr(cid
, hoid
, "foo", bp
);
2422 ASSERT_TRUE(bl_eq(val
, bl
));
2424 map
<string
,bufferptr
> bm
;
2425 r
= store
->getattrs(cid
, hoid
, bm
);
2430 ObjectStore::Transaction t
;
2431 t
.remove(cid
, hoid
);
2432 t
.remove_collection(cid
);
2433 r
= apply_transaction(store
, &osr
, std::move(t
));
2438 TEST_P(StoreTest
, SimpleListTest
) {
2439 ObjectStore::Sequencer
osr("test");
2441 coll_t
cid(spg_t(pg_t(0, 1), shard_id_t(1)));
2443 ObjectStore::Transaction t
;
2444 t
.create_collection(cid
, 0);
2445 cerr
<< "Creating collection " << cid
<< std::endl
;
2446 r
= apply_transaction(store
, &osr
, std::move(t
));
2449 set
<ghobject_t
> all
;
2451 ObjectStore::Transaction t
;
2452 for (int i
=0; i
<200; ++i
) {
2453 string
name("object_");
2454 name
+= stringify(i
);
2455 ghobject_t
hoid(hobject_t(sobject_t(name
, CEPH_NOSNAP
)),
2456 ghobject_t::NO_GEN
, shard_id_t(1));
2460 cerr
<< "Creating object " << hoid
<< std::endl
;
2462 r
= apply_transaction(store
, &osr
, std::move(t
));
2466 set
<ghobject_t
> saw
;
2467 vector
<ghobject_t
> objects
;
2468 ghobject_t next
, current
;
2469 while (!next
.is_max()) {
2470 int r
= store
->collection_list(cid
, current
, ghobject_t::get_max(),
2474 ASSERT_TRUE(sorted(objects
));
2475 cout
<< " got " << objects
.size() << " next " << next
<< std::endl
;
2476 for (vector
<ghobject_t
>::iterator p
= objects
.begin(); p
!= objects
.end();
2478 if (saw
.count(*p
)) {
2479 cout
<< "got DUP " << *p
<< std::endl
;
2481 //cout << "got new " << *p << std::endl;
2488 ASSERT_EQ(saw
.size(), all
.size());
2489 ASSERT_EQ(saw
, all
);
2492 ObjectStore::Transaction t
;
2493 for (set
<ghobject_t
>::iterator p
= all
.begin(); p
!= all
.end(); ++p
)
2495 t
.remove_collection(cid
);
2496 cerr
<< "Cleaning" << std::endl
;
2497 r
= apply_transaction(store
, &osr
, std::move(t
));
2502 TEST_P(StoreTest
, ListEndTest
) {
2503 ObjectStore::Sequencer
osr("test");
2505 coll_t
cid(spg_t(pg_t(0, 1), shard_id_t(1)));
2507 ObjectStore::Transaction t
;
2508 t
.create_collection(cid
, 0);
2509 cerr
<< "Creating collection " << cid
<< std::endl
;
2510 r
= apply_transaction(store
, &osr
, std::move(t
));
2513 set
<ghobject_t
> all
;
2515 ObjectStore::Transaction t
;
2516 for (int i
=0; i
<200; ++i
) {
2517 string
name("object_");
2518 name
+= stringify(i
);
2519 ghobject_t
hoid(hobject_t(sobject_t(name
, CEPH_NOSNAP
)),
2520 ghobject_t::NO_GEN
, shard_id_t(1));
2524 cerr
<< "Creating object " << hoid
<< std::endl
;
2526 r
= apply_transaction(store
, &osr
, std::move(t
));
2530 ghobject_t
end(hobject_t(sobject_t("object_100", CEPH_NOSNAP
)),
2531 ghobject_t::NO_GEN
, shard_id_t(1));
2533 vector
<ghobject_t
> objects
;
2535 int r
= store
->collection_list(cid
, ghobject_t(), end
, 500,
2538 for (auto &p
: objects
) {
2543 ObjectStore::Transaction t
;
2544 for (set
<ghobject_t
>::iterator p
= all
.begin(); p
!= all
.end(); ++p
)
2546 t
.remove_collection(cid
);
2547 cerr
<< "Cleaning" << std::endl
;
2548 r
= apply_transaction(store
, &osr
, std::move(t
));
2553 TEST_P(StoreTest
, Sort
) {
2555 hobject_t
a(sobject_t("a", CEPH_NOSNAP
));
2568 ghobject_t
a(hobject_t(sobject_t("a", CEPH_NOSNAP
)));
2569 ghobject_t
b(hobject_t(sobject_t("b", CEPH_NOSNAP
)));
2581 TEST_P(StoreTest
, MultipoolListTest
) {
2582 ObjectStore::Sequencer
osr("test");
2585 coll_t cid
= coll_t(spg_t(pg_t(0, poolid
), shard_id_t::NO_SHARD
));
2587 ObjectStore::Transaction t
;
2588 t
.create_collection(cid
, 0);
2589 cerr
<< "Creating collection " << cid
<< std::endl
;
2590 r
= apply_transaction(store
, &osr
, std::move(t
));
2593 set
<ghobject_t
> all
, saw
;
2595 ObjectStore::Transaction t
;
2596 for (int i
=0; i
<200; ++i
) {
2597 string
name("object_");
2598 name
+= stringify(i
);
2599 ghobject_t
hoid(hobject_t(sobject_t(name
, CEPH_NOSNAP
)));
2601 hoid
.hobj
.pool
= -2 - poolid
;
2603 hoid
.hobj
.pool
= poolid
;
2606 cerr
<< "Creating object " << hoid
<< std::endl
;
2608 r
= apply_transaction(store
, &osr
, std::move(t
));
2612 vector
<ghobject_t
> objects
;
2613 ghobject_t next
, current
;
2614 while (!next
.is_max()) {
2615 int r
= store
->collection_list(cid
, current
, ghobject_t::get_max(), 50,
2618 cout
<< " got " << objects
.size() << " next " << next
<< std::endl
;
2619 for (vector
<ghobject_t
>::iterator p
= objects
.begin(); p
!= objects
.end();
2626 ASSERT_EQ(saw
, all
);
2629 ObjectStore::Transaction t
;
2630 for (set
<ghobject_t
>::iterator p
= all
.begin(); p
!= all
.end(); ++p
)
2632 t
.remove_collection(cid
);
2633 cerr
<< "Cleaning" << std::endl
;
2634 r
= apply_transaction(store
, &osr
, std::move(t
));
2639 TEST_P(StoreTest
, SimpleCloneTest
) {
2640 ObjectStore::Sequencer
osr("test");
2644 ObjectStore::Transaction t
;
2645 t
.create_collection(cid
, 0);
2646 cerr
<< "Creating collection " << cid
<< std::endl
;
2647 r
= apply_transaction(store
, &osr
, std::move(t
));
2650 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
),
2651 "key", 123, -1, ""));
2652 bufferlist small
, large
, xlarge
, newdata
, attr
;
2653 small
.append("small");
2654 large
.append("large");
2655 xlarge
.append("xlarge");
2657 ObjectStore::Transaction t
;
2659 t
.setattr(cid
, hoid
, "attr1", small
);
2660 t
.setattr(cid
, hoid
, "attr2", large
);
2661 t
.setattr(cid
, hoid
, "attr3", xlarge
);
2662 t
.write(cid
, hoid
, 0, small
.length(), small
);
2663 t
.write(cid
, hoid
, 10, small
.length(), small
);
2664 cerr
<< "Creating object and set attr " << hoid
<< std::endl
;
2665 r
= apply_transaction(store
, &osr
, std::move(t
));
2669 ghobject_t
hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP
),
2670 "key", 123, -1, ""));
2671 ghobject_t
hoid3(hobject_t(sobject_t("Object 3", CEPH_NOSNAP
)));
2673 ObjectStore::Transaction t
;
2674 t
.clone(cid
, hoid
, hoid2
);
2675 t
.setattr(cid
, hoid2
, "attr2", small
);
2676 t
.rmattr(cid
, hoid2
, "attr1");
2677 t
.write(cid
, hoid
, 10, large
.length(), large
);
2678 t
.setattr(cid
, hoid
, "attr1", large
);
2679 t
.setattr(cid
, hoid
, "attr2", small
);
2680 cerr
<< "Clone object and rm attr" << std::endl
;
2681 r
= apply_transaction(store
, &osr
, std::move(t
));
2684 r
= store
->read(cid
, hoid
, 10, 5, newdata
);
2686 ASSERT_TRUE(bl_eq(large
, newdata
));
2689 r
= store
->read(cid
, hoid
, 0, 5, newdata
);
2691 ASSERT_TRUE(bl_eq(small
, newdata
));
2694 r
= store
->read(cid
, hoid2
, 10, 5, newdata
);
2696 ASSERT_TRUE(bl_eq(small
, newdata
));
2698 r
= store
->getattr(cid
, hoid2
, "attr2", attr
);
2700 ASSERT_TRUE(bl_eq(small
, attr
));
2703 r
= store
->getattr(cid
, hoid2
, "attr3", attr
);
2705 ASSERT_TRUE(bl_eq(xlarge
, attr
));
2708 r
= store
->getattr(cid
, hoid
, "attr1", attr
);
2710 ASSERT_TRUE(bl_eq(large
, attr
));
2713 ObjectStore::Transaction t
;
2714 t
.remove(cid
, hoid
);
2715 t
.remove(cid
, hoid2
);
2716 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2721 memset(p
.c_str(), 1, p
.length());
2725 ObjectStore::Transaction t
;
2726 t
.write(cid
, hoid
, 0, pl
.length(), pl
);
2727 t
.clone(cid
, hoid
, hoid2
);
2729 memset(a
.c_str(), 2, a
.length());
2733 t
.write(cid
, hoid
, pl
.length(), a
.length(), al
);
2734 r
= apply_transaction(store
, &osr
, std::move(t
));
2737 ASSERT_EQ((int)final
.length(),
2738 store
->read(cid
, hoid
, 0, final
.length(), rl
));
2739 ASSERT_TRUE(bl_eq(rl
, final
));
2742 ObjectStore::Transaction t
;
2743 t
.remove(cid
, hoid
);
2744 t
.remove(cid
, hoid2
);
2745 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2750 memset(p
.c_str(), 111, p
.length());
2754 ObjectStore::Transaction t
;
2755 t
.write(cid
, hoid
, 0, pl
.length(), pl
);
2756 t
.clone(cid
, hoid
, hoid2
);
2761 memset(a
.c_str(), 112, a
.length());
2765 t
.write(cid
, hoid
, pl
.length() + z
.length(), a
.length(), al
);
2766 r
= apply_transaction(store
, &osr
, std::move(t
));
2769 ASSERT_EQ((int)final
.length(),
2770 store
->read(cid
, hoid
, 0, final
.length(), rl
));
2771 ASSERT_TRUE(bl_eq(rl
, final
));
2774 ObjectStore::Transaction t
;
2775 t
.remove(cid
, hoid
);
2776 t
.remove(cid
, hoid2
);
2777 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2782 memset(p
.c_str(), 5, p
.length());
2786 ObjectStore::Transaction t
;
2787 t
.write(cid
, hoid
, 0, pl
.length(), pl
);
2788 t
.clone(cid
, hoid
, hoid2
);
2793 memset(a
.c_str(), 6, a
.length());
2797 t
.write(cid
, hoid
, 17000, a
.length(), al
);
2798 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2800 ASSERT_EQ((int)final
.length(),
2801 store
->read(cid
, hoid
, 0, final
.length(), rl
));
2802 /*cout << "expected:\n";
2803 final.hexdump(cout);
2806 ASSERT_TRUE(bl_eq(rl
, final
));
2809 ObjectStore::Transaction t
;
2810 t
.remove(cid
, hoid
);
2811 t
.remove(cid
, hoid2
);
2812 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2815 bufferptr
p(1048576);
2816 memset(p
.c_str(), 3, p
.length());
2819 ObjectStore::Transaction t
;
2820 t
.write(cid
, hoid
, 0, pl
.length(), pl
);
2821 t
.clone(cid
, hoid
, hoid2
);
2823 memset(a
.c_str(), 4, a
.length());
2826 t
.write(cid
, hoid
, a
.length(), a
.length(), al
);
2827 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2830 final
.substr_of(pl
, 0, al
.length());
2833 end
.substr_of(pl
, al
.length()*2, pl
.length() - al
.length()*2);
2835 ASSERT_EQ((int)final
.length(),
2836 store
->read(cid
, hoid
, 0, final
.length(), rl
));
2837 /*cout << "expected:\n";
2838 final.hexdump(cout);
2841 ASSERT_TRUE(bl_eq(rl
, final
));
2844 ObjectStore::Transaction t
;
2845 t
.remove(cid
, hoid
);
2846 t
.remove(cid
, hoid2
);
2847 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2851 memset(p
.c_str(), 7, p
.length());
2854 ObjectStore::Transaction t
;
2855 t
.write(cid
, hoid
, 0, pl
.length(), pl
);
2856 t
.clone(cid
, hoid
, hoid2
);
2858 memset(a
.c_str(), 8, a
.length());
2861 t
.write(cid
, hoid
, 32768, a
.length(), al
);
2862 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2865 final
.substr_of(pl
, 0, 32768);
2868 end
.substr_of(pl
, final
.length(), pl
.length() - final
.length());
2870 ASSERT_EQ((int)final
.length(),
2871 store
->read(cid
, hoid
, 0, final
.length(), rl
));
2872 /*cout << "expected:\n";
2873 final.hexdump(cout);
2876 ASSERT_TRUE(bl_eq(rl
, final
));
2879 ObjectStore::Transaction t
;
2880 t
.remove(cid
, hoid
);
2881 t
.remove(cid
, hoid2
);
2882 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2886 memset(p
.c_str(), 9, p
.length());
2889 ObjectStore::Transaction t
;
2890 t
.write(cid
, hoid
, 0, pl
.length(), pl
);
2891 t
.clone(cid
, hoid
, hoid2
);
2893 memset(a
.c_str(), 10, a
.length());
2896 t
.write(cid
, hoid
, 33768, a
.length(), al
);
2897 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2900 final
.substr_of(pl
, 0, 33768);
2903 end
.substr_of(pl
, final
.length(), pl
.length() - final
.length());
2905 ASSERT_EQ((int)final
.length(),
2906 store
->read(cid
, hoid
, 0, final
.length(), rl
));
2907 /*cout << "expected:\n";
2908 final.hexdump(cout);
2911 ASSERT_TRUE(bl_eq(rl
, final
));
2914 //Unfortunately we need a workaround for filestore since EXPECT_DEATH
2915 // macro has potential issues when using /in multithread environments.
2916 //It works well for all stores but filestore for now.
2917 //A fix setting gtest_death_test_style = "threadsafe" doesn't help as well -
2918 // test app clone asserts on store folder presence.
2920 if (string(GetParam()) != "filestore") {
2921 //verify if non-empty collection is properly handled after store reload
2922 r
= store
->umount();
2927 ObjectStore::Transaction t
;
2928 t
.remove_collection(cid
);
2929 cerr
<< "Invalid rm coll" << std::endl
;
2930 PrCtl unset_dumpable
;
2931 EXPECT_DEATH(apply_transaction(store
, &osr
, std::move(t
)), "");
2934 ObjectStore::Transaction t
;
2935 t
.touch(cid
, hoid3
); //new record in db
2936 r
= apply_transaction(store
, &osr
, std::move(t
));
2939 //See comment above for "filestore" check explanation.
2940 if (string(GetParam()) != "filestore") {
2941 ObjectStore::Transaction t
;
2942 //verify if non-empty collection is properly handled when there are some pending removes and live records in db
2943 cerr
<< "Invalid rm coll again" << std::endl
;
2944 r
= store
->umount();
2949 t
.remove(cid
, hoid
);
2950 t
.remove(cid
, hoid2
);
2951 t
.remove_collection(cid
);
2952 PrCtl unset_dumpable
;
2953 EXPECT_DEATH(apply_transaction(store
, &osr
, std::move(t
)), "");
2956 ObjectStore::Transaction t
;
2957 t
.remove(cid
, hoid
);
2958 t
.remove(cid
, hoid2
);
2959 t
.remove(cid
, hoid3
);
2960 t
.remove_collection(cid
);
2961 cerr
<< "Cleaning" << std::endl
;
2962 r
= apply_transaction(store
, &osr
, std::move(t
));
2967 TEST_P(StoreTest
, OmapSimple
) {
2968 ObjectStore::Sequencer
osr("test");
2972 ObjectStore::Transaction t
;
2973 t
.create_collection(cid
, 0);
2974 cerr
<< "Creating collection " << cid
<< std::endl
;
2975 r
= apply_transaction(store
, &osr
, std::move(t
));
2978 ghobject_t
hoid(hobject_t(sobject_t("omap_obj", CEPH_NOSNAP
),
2979 "key", 123, -1, ""));
2981 small
.append("small");
2982 map
<string
,bufferlist
> km
;
2984 km
["bar"].append("asdfjkasdkjdfsjkafskjsfdj");
2986 header
.append("this is a header");
2988 ObjectStore::Transaction t
;
2990 t
.omap_setkeys(cid
, hoid
, km
);
2991 t
.omap_setheader(cid
, hoid
, header
);
2992 cerr
<< "Creating object and set omap " << hoid
<< std::endl
;
2993 r
= apply_transaction(store
, &osr
, std::move(t
));
2999 map
<string
,bufferlist
> r
;
3000 store
->omap_get(cid
, hoid
, &h
, &r
);
3001 ASSERT_TRUE(bl_eq(header
, h
));
3002 ASSERT_EQ(r
.size(), km
.size());
3003 cout
<< "r: " << r
<< std::endl
;
3005 // test iterator with seek_to_first
3007 map
<string
,bufferlist
> r
;
3008 ObjectMap::ObjectMapIterator iter
= store
->get_omap_iterator(cid
, hoid
);
3009 for (iter
->seek_to_first(); iter
->valid(); iter
->next(false)) {
3010 r
[iter
->key()] = iter
->value();
3012 cout
<< "r: " << r
<< std::endl
;
3013 ASSERT_EQ(r
.size(), km
.size());
3015 // test iterator with initial lower_bound
3017 map
<string
,bufferlist
> r
;
3018 ObjectMap::ObjectMapIterator iter
= store
->get_omap_iterator(cid
, hoid
);
3019 for (iter
->lower_bound(string()); iter
->valid(); iter
->next(false)) {
3020 r
[iter
->key()] = iter
->value();
3022 cout
<< "r: " << r
<< std::endl
;
3023 ASSERT_EQ(r
.size(), km
.size());
3026 ObjectStore::Transaction t
;
3027 t
.remove(cid
, hoid
);
3028 t
.remove_collection(cid
);
3029 cerr
<< "Cleaning" << std::endl
;
3030 r
= apply_transaction(store
, &osr
, std::move(t
));
3035 TEST_P(StoreTest
, OmapCloneTest
) {
3036 ObjectStore::Sequencer
osr("test");
3040 ObjectStore::Transaction t
;
3041 t
.create_collection(cid
, 0);
3042 cerr
<< "Creating collection " << cid
<< std::endl
;
3043 r
= apply_transaction(store
, &osr
, std::move(t
));
3046 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
),
3047 "key", 123, -1, ""));
3049 small
.append("small");
3050 map
<string
,bufferlist
> km
;
3052 km
["bar"].append("asdfjkasdkjdfsjkafskjsfdj");
3054 header
.append("this is a header");
3056 ObjectStore::Transaction t
;
3058 t
.omap_setkeys(cid
, hoid
, km
);
3059 t
.omap_setheader(cid
, hoid
, header
);
3060 cerr
<< "Creating object and set omap " << hoid
<< std::endl
;
3061 r
= apply_transaction(store
, &osr
, std::move(t
));
3064 ghobject_t
hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP
),
3065 "key", 123, -1, ""));
3067 ObjectStore::Transaction t
;
3068 t
.clone(cid
, hoid
, hoid2
);
3069 cerr
<< "Clone object" << std::endl
;
3070 r
= apply_transaction(store
, &osr
, std::move(t
));
3074 map
<string
,bufferlist
> r
;
3076 store
->omap_get(cid
, hoid2
, &h
, &r
);
3077 ASSERT_TRUE(bl_eq(header
, h
));
3078 ASSERT_EQ(r
.size(), km
.size());
3081 ObjectStore::Transaction t
;
3082 t
.remove(cid
, hoid
);
3083 t
.remove(cid
, hoid2
);
3084 t
.remove_collection(cid
);
3085 cerr
<< "Cleaning" << std::endl
;
3086 r
= apply_transaction(store
, &osr
, std::move(t
));
3091 TEST_P(StoreTest
, SimpleCloneRangeTest
) {
3092 ObjectStore::Sequencer
osr("test");
3096 ObjectStore::Transaction t
;
3097 t
.create_collection(cid
, 0);
3098 cerr
<< "Creating collection " << cid
<< std::endl
;
3099 r
= apply_transaction(store
, &osr
, std::move(t
));
3102 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
3103 hoid
.hobj
.pool
= -1;
3104 bufferlist small
, newdata
;
3105 small
.append("small");
3107 ObjectStore::Transaction t
;
3108 t
.write(cid
, hoid
, 10, 5, small
);
3109 cerr
<< "Creating object and write bl " << hoid
<< std::endl
;
3110 r
= apply_transaction(store
, &osr
, std::move(t
));
3113 ghobject_t
hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP
)));
3114 hoid2
.hobj
.pool
= -1;
3116 ObjectStore::Transaction t
;
3117 t
.clone_range(cid
, hoid
, hoid2
, 10, 5, 10);
3118 cerr
<< "Clone range object" << std::endl
;
3119 r
= apply_transaction(store
, &osr
, std::move(t
));
3121 r
= store
->read(cid
, hoid2
, 10, 5, newdata
);
3123 ASSERT_TRUE(bl_eq(small
, newdata
));
3126 ObjectStore::Transaction t
;
3127 t
.truncate(cid
, hoid
, 1024*1024);
3128 t
.clone_range(cid
, hoid
, hoid2
, 0, 1024*1024, 0);
3129 cerr
<< "Clone range object" << std::endl
;
3130 r
= apply_transaction(store
, &osr
, std::move(t
));
3132 struct stat stat
, stat2
;
3133 r
= store
->stat(cid
, hoid
, &stat
);
3134 r
= store
->stat(cid
, hoid2
, &stat2
);
3135 ASSERT_EQ(stat
.st_size
, stat2
.st_size
);
3136 ASSERT_EQ(1024*1024, stat2
.st_size
);
3139 ObjectStore::Transaction t
;
3140 t
.remove(cid
, hoid
);
3141 t
.remove(cid
, hoid2
);
3142 t
.remove_collection(cid
);
3143 cerr
<< "Cleaning" << std::endl
;
3144 r
= apply_transaction(store
, &osr
, std::move(t
));
3150 TEST_P(StoreTest
, SimpleObjectLongnameTest
) {
3151 ObjectStore::Sequencer
osr("test");
3155 ObjectStore::Transaction t
;
3156 t
.create_collection(cid
, 0);
3157 cerr
<< "Creating collection " << cid
<< std::endl
;
3158 r
= apply_transaction(store
, &osr
, std::move(t
));
3161 ghobject_t
hoid(hobject_t(sobject_t("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaObjectaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 1", CEPH_NOSNAP
)));
3163 ObjectStore::Transaction t
;
3165 cerr
<< "Creating object " << hoid
<< std::endl
;
3166 r
= apply_transaction(store
, &osr
, std::move(t
));
3170 ObjectStore::Transaction t
;
3171 t
.remove(cid
, hoid
);
3172 t
.remove_collection(cid
);
3173 cerr
<< "Cleaning" << std::endl
;
3174 r
= apply_transaction(store
, &osr
, std::move(t
));
3179 ghobject_t
generate_long_name(unsigned i
)
3182 name
<< "object id " << i
<< " ";
3183 for (unsigned j
= 0; j
< 500; ++j
) name
<< 'a';
3184 ghobject_t
hoid(hobject_t(sobject_t(name
.str(), CEPH_NOSNAP
)));
3185 hoid
.hobj
.set_hash(i
% 2);
3189 TEST_P(StoreTest
, LongnameSplitTest
) {
3190 ObjectStore::Sequencer
osr("test");
3194 ObjectStore::Transaction t
;
3195 t
.create_collection(cid
, 0);
3196 cerr
<< "Creating collection " << cid
<< std::endl
;
3197 r
= apply_transaction(store
, &osr
, std::move(t
));
3200 for (unsigned i
= 0; i
< 320; ++i
) {
3201 ObjectStore::Transaction t
;
3202 ghobject_t hoid
= generate_long_name(i
);
3204 cerr
<< "Creating object " << hoid
<< std::endl
;
3205 r
= apply_transaction(store
, &osr
, std::move(t
));
3209 ghobject_t test_obj
= generate_long_name(319);
3210 ghobject_t test_obj_2
= test_obj
;
3211 test_obj_2
.generation
= 0;
3213 ObjectStore::Transaction t
;
3214 // should cause a split
3215 t
.collection_move_rename(
3218 r
= apply_transaction(store
, &osr
, std::move(t
));
3222 for (unsigned i
= 0; i
< 319; ++i
) {
3223 ObjectStore::Transaction t
;
3224 ghobject_t hoid
= generate_long_name(i
);
3225 t
.remove(cid
, hoid
);
3226 cerr
<< "Removing object " << hoid
<< std::endl
;
3227 r
= apply_transaction(store
, &osr
, std::move(t
));
3231 ObjectStore::Transaction t
;
3232 t
.remove(cid
, test_obj_2
);
3233 t
.remove_collection(cid
);
3234 cerr
<< "Cleaning" << std::endl
;
3235 r
= apply_transaction(store
, &osr
, std::move(t
));
3241 TEST_P(StoreTest
, ManyObjectTest
) {
3242 ObjectStore::Sequencer
osr("test");
3243 int NUM_OBJS
= 2000;
3247 for (int i
= 0; i
< 100; ++i
) base
.append("aaaaa");
3248 set
<ghobject_t
> created
;
3250 ObjectStore::Transaction t
;
3251 t
.create_collection(cid
, 0);
3252 r
= apply_transaction(store
, &osr
, std::move(t
));
3255 for (int i
= 0; i
< NUM_OBJS
; ++i
) {
3257 cerr
<< "Object " << i
<< std::endl
;
3259 ObjectStore::Transaction t
;
3261 snprintf(buf
, sizeof(buf
), "%d", i
);
3262 ghobject_t
hoid(hobject_t(sobject_t(string(buf
) + base
, CEPH_NOSNAP
)));
3264 created
.insert(hoid
);
3265 r
= apply_transaction(store
, &osr
, std::move(t
));
3269 for (set
<ghobject_t
>::iterator i
= created
.begin();
3273 ASSERT_TRUE(!store
->stat(cid
, *i
, &buf
));
3276 set
<ghobject_t
> listed
, listed2
;
3277 vector
<ghobject_t
> objects
;
3278 r
= store
->collection_list(cid
, ghobject_t(), ghobject_t::get_max(), INT_MAX
, &objects
, 0);
3281 cerr
<< "objects.size() is " << objects
.size() << std::endl
;
3282 for (vector
<ghobject_t
> ::iterator i
= objects
.begin();
3286 ASSERT_TRUE(created
.count(*i
));
3288 ASSERT_TRUE(listed
.size() == created
.size());
3290 ghobject_t start
, next
;
3292 r
= store
->collection_list(
3294 ghobject_t::get_max(),
3295 ghobject_t::get_max(),
3301 ASSERT_TRUE(objects
.empty());
3305 ghobject_t start2
, next2
;
3307 r
= store
->collection_list(cid
, start
, ghobject_t::get_max(),
3311 ASSERT_TRUE(sorted(objects
));
3313 listed
.insert(objects
.begin(), objects
.end());
3314 if (objects
.size() < 50) {
3315 ASSERT_TRUE(next
.is_max());
3322 cerr
<< "listed.size() is " << listed
.size() << std::endl
;
3323 ASSERT_TRUE(listed
.size() == created
.size());
3324 if (listed2
.size()) {
3325 ASSERT_EQ(listed
.size(), listed2
.size());
3327 for (set
<ghobject_t
>::iterator i
= listed
.begin();
3330 ASSERT_TRUE(created
.count(*i
));
3333 for (set
<ghobject_t
>::iterator i
= created
.begin();
3336 ObjectStore::Transaction t
;
3338 r
= apply_transaction(store
, &osr
, std::move(t
));
3341 cerr
<< "cleaning up" << std::endl
;
3343 ObjectStore::Transaction t
;
3344 t
.remove_collection(cid
);
3345 r
= apply_transaction(store
, &osr
, std::move(t
));
3351 class ObjectGenerator
{
3353 virtual ghobject_t
create_object(gen_type
*gen
) = 0;
3354 virtual ~ObjectGenerator() {}
3357 class MixedGenerator
: public ObjectGenerator
{
3361 explicit MixedGenerator(int64_t p
) : seq(0), poolid(p
) {}
3362 ghobject_t
create_object(gen_type
*gen
) override
{
3364 snprintf(buf
, sizeof(buf
), "OBJ_%u", seq
);
3367 for (unsigned i
= 0; i
< 300; ++i
) {
3368 name
.push_back('a');
3374 name
, string(), rand() & 2 ? CEPH_NOSNAP
: rand(),
3375 (((seq
/ 1024) % 2) * 0xF00 ) +
3381 class SyntheticWorkloadState
{
3384 map
<string
, bufferlist
> attrs
;
3387 static const unsigned max_in_flight
= 16;
3388 static const unsigned max_objects
= 3000;
3389 static const unsigned max_attr_size
= 5;
3390 static const unsigned max_attr_name_len
= 100;
3391 static const unsigned max_attr_value_len
= 1024 * 64;
3393 unsigned write_alignment
;
3394 unsigned max_object_len
, max_write_len
;
3396 map
<ghobject_t
, Object
> contents
;
3397 set
<ghobject_t
> available_objects
;
3398 set
<ghobject_t
> in_flight_objects
;
3399 ObjectGenerator
*object_gen
;
3402 ObjectStore::Sequencer
*osr
;
3409 explicit EnterExit(const char *m
) : msg(m
) {
3410 //cout << pthread_self() << " enter " << msg << std::endl;
3413 //cout << pthread_self() << " exit " << msg << std::endl;
3417 class C_SyntheticOnReadable
: public Context
{
3419 SyntheticWorkloadState
*state
;
3421 C_SyntheticOnReadable(SyntheticWorkloadState
*state
, ghobject_t hoid
)
3422 : state(state
), hoid(hoid
) {}
3424 void finish(int r
) override
{
3425 Mutex::Locker
locker(state
->lock
);
3426 EnterExit
ee("onreadable finish");
3427 ASSERT_TRUE(state
->in_flight_objects
.count(hoid
));
3429 state
->in_flight_objects
.erase(hoid
);
3430 if (state
->contents
.count(hoid
))
3431 state
->available_objects
.insert(hoid
);
3432 --(state
->in_flight
);
3433 state
->cond
.Signal();
3436 r
= state
->store
->read(state
->cid
, hoid
, 0, state
->contents
[hoid
].data
.length(), r2
);
3437 assert(bl_eq(state
->contents
[hoid
].data
, r2
));
3438 state
->cond
.Signal();
3442 class C_SyntheticOnStash
: public Context
{
3444 SyntheticWorkloadState
*state
;
3445 ghobject_t oid
, noid
;
3447 C_SyntheticOnStash(SyntheticWorkloadState
*state
,
3448 ghobject_t oid
, ghobject_t noid
)
3449 : state(state
), oid(oid
), noid(noid
) {}
3451 void finish(int r
) override
{
3452 Mutex::Locker
locker(state
->lock
);
3453 EnterExit
ee("stash finish");
3454 ASSERT_TRUE(state
->in_flight_objects
.count(oid
));
3456 state
->in_flight_objects
.erase(oid
);
3457 if (state
->contents
.count(noid
))
3458 state
->available_objects
.insert(noid
);
3459 --(state
->in_flight
);
3461 r
= state
->store
->read(
3462 state
->cid
, noid
, 0,
3463 state
->contents
[noid
].data
.length(), r2
);
3464 assert(bl_eq(state
->contents
[noid
].data
, r2
));
3465 state
->cond
.Signal();
3469 class C_SyntheticOnClone
: public Context
{
3471 SyntheticWorkloadState
*state
;
3472 ghobject_t oid
, noid
;
3474 C_SyntheticOnClone(SyntheticWorkloadState
*state
,
3475 ghobject_t oid
, ghobject_t noid
)
3476 : state(state
), oid(oid
), noid(noid
) {}
3478 void finish(int r
) override
{
3479 Mutex::Locker
locker(state
->lock
);
3480 EnterExit
ee("clone finish");
3481 ASSERT_TRUE(state
->in_flight_objects
.count(oid
));
3483 state
->in_flight_objects
.erase(oid
);
3484 if (state
->contents
.count(oid
))
3485 state
->available_objects
.insert(oid
);
3486 if (state
->contents
.count(noid
))
3487 state
->available_objects
.insert(noid
);
3488 --(state
->in_flight
);
3490 r
= state
->store
->read(state
->cid
, noid
, 0, state
->contents
[noid
].data
.length(), r2
);
3491 assert(bl_eq(state
->contents
[noid
].data
, r2
));
3492 state
->cond
.Signal();
3496 static void filled_byte_array(bufferlist
& bl
, size_t size
)
3498 static const char alphanum
[] = "0123456789"
3499 "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
3500 "abcdefghijklmnopqrstuvwxyz";
3505 for (unsigned int i
= 0; i
< size
- 1; i
++) {
3506 // severely limit entropy so we can compress...
3507 bp
[i
] = alphanum
[rand() % 10]; //(sizeof(alphanum) - 1)];
3509 bp
[size
- 1] = '\0';
3514 SyntheticWorkloadState(ObjectStore
*store
,
3515 ObjectGenerator
*gen
,
3517 ObjectStore::Sequencer
*osr
,
3522 : cid(cid
), write_alignment(alignment
), max_object_len(max_size
),
3523 max_write_len(max_write
), in_flight(0), object_gen(gen
),
3524 rng(rng
), store(store
), osr(osr
), lock("State lock") {}
3527 ObjectStore::Transaction t
;
3528 t
.create_collection(cid
, 0);
3529 return apply_transaction(store
, osr
, std::move(t
));
3533 vector
<ghobject_t
> objects
;
3534 int r
= store
->collection_list(cid
, ghobject_t(), ghobject_t::get_max(),
3537 if (objects
.empty())
3539 ObjectStore::Transaction t
;
3540 for (vector
<ghobject_t
>::iterator p
= objects
.begin();
3541 p
!= objects
.end(); ++p
) {
3544 apply_transaction(store
, osr
, std::move(t
));
3546 ObjectStore::Transaction t
;
3547 t
.remove_collection(cid
);
3548 apply_transaction(store
, osr
, std::move(t
));
3550 void statfs(store_statfs_t
& stat
) {
3551 store
->statfs(&stat
);
3554 ghobject_t
get_uniform_random_object() {
3555 while (in_flight
>= max_in_flight
|| available_objects
.empty())
3557 boost::uniform_int
<> choose(0, available_objects
.size() - 1);
3558 int index
= choose(*rng
);
3559 set
<ghobject_t
>::iterator i
= available_objects
.begin();
3560 for ( ; index
> 0; --index
, ++i
) ;
3561 ghobject_t ret
= *i
;
3565 void wait_for_ready() {
3566 while (in_flight
>= max_in_flight
)
3570 void wait_for_done() {
3572 Mutex::Locker
locker(lock
);
3578 return (available_objects
.size() + in_flight_objects
.size()) < max_objects
;
3582 return (available_objects
.size() + in_flight_objects
.size()) > 0;
3585 unsigned get_random_alloc_hints() {
3588 boost::uniform_int
<> u(0, 3);
3591 f
|= CEPH_OSD_ALLOC_HINT_FLAG_SEQUENTIAL_WRITE
;
3594 f
|= CEPH_OSD_ALLOC_HINT_FLAG_RANDOM_WRITE
;
3599 boost::uniform_int
<> u(0, 3);
3602 f
|= CEPH_OSD_ALLOC_HINT_FLAG_SEQUENTIAL_READ
;
3605 f
|= CEPH_OSD_ALLOC_HINT_FLAG_RANDOM_READ
;
3610 // append_only, immutable
3611 boost::uniform_int
<> u(0, 4);
3615 boost::uniform_int
<> u(0, 3);
3618 f
|= CEPH_OSD_ALLOC_HINT_FLAG_SHORTLIVED
;
3621 f
|= CEPH_OSD_ALLOC_HINT_FLAG_LONGLIVED
;
3626 boost::uniform_int
<> u(0, 3);
3629 f
|= CEPH_OSD_ALLOC_HINT_FLAG_COMPRESSIBLE
;
3632 f
|= CEPH_OSD_ALLOC_HINT_FLAG_INCOMPRESSIBLE
;
3640 Mutex::Locker
locker(lock
);
3641 EnterExit
ee("touch");
3645 ghobject_t new_obj
= object_gen
->create_object(rng
);
3646 available_objects
.erase(new_obj
);
3647 ObjectStore::Transaction t
;
3648 t
.touch(cid
, new_obj
);
3649 boost::uniform_int
<> u(17, 22);
3650 boost::uniform_int
<> v(12, 17);
3651 t
.set_alloc_hint(cid
, new_obj
,
3654 get_random_alloc_hints());
3656 in_flight_objects
.insert(new_obj
);
3657 if (!contents
.count(new_obj
))
3658 contents
[new_obj
] = Object();
3659 int status
= store
->queue_transaction(osr
, std::move(t
), new C_SyntheticOnReadable(this, new_obj
));
3664 Mutex::Locker
locker(lock
);
3665 EnterExit
ee("stash");
3675 old_obj
= get_uniform_random_object();
3676 } while (--max
&& !contents
[old_obj
].data
.length());
3677 available_objects
.erase(old_obj
);
3678 ghobject_t new_obj
= old_obj
;
3679 new_obj
.generation
++;
3680 available_objects
.erase(new_obj
);
3682 ObjectStore::Transaction t
;
3683 t
.collection_move_rename(cid
, old_obj
, cid
, new_obj
);
3685 in_flight_objects
.insert(old_obj
);
3687 contents
[new_obj
].attrs
= contents
[old_obj
].attrs
;
3688 contents
[new_obj
].data
= contents
[old_obj
].data
;
3689 contents
.erase(old_obj
);
3690 int status
= store
->queue_transaction(
3692 new C_SyntheticOnStash(this, old_obj
, new_obj
));
3697 Mutex::Locker
locker(lock
);
3698 EnterExit
ee("clone");
3708 old_obj
= get_uniform_random_object();
3709 } while (--max
&& !contents
[old_obj
].data
.length());
3710 available_objects
.erase(old_obj
);
3711 ghobject_t new_obj
= object_gen
->create_object(rng
);
3712 // make the hash match
3713 new_obj
.hobj
.set_hash(old_obj
.hobj
.get_hash());
3714 available_objects
.erase(new_obj
);
3716 ObjectStore::Transaction t
;
3717 t
.clone(cid
, old_obj
, new_obj
);
3719 in_flight_objects
.insert(old_obj
);
3721 contents
[new_obj
].attrs
= contents
[old_obj
].attrs
;
3722 contents
[new_obj
].data
= contents
[old_obj
].data
;
3724 int status
= store
->queue_transaction(
3726 new C_SyntheticOnClone(this, old_obj
, new_obj
));
3731 Mutex::Locker
locker(lock
);
3732 EnterExit
ee("clone_range");
3742 old_obj
= get_uniform_random_object();
3743 } while (--max
&& !contents
[old_obj
].data
.length());
3744 bufferlist
&srcdata
= contents
[old_obj
].data
;
3745 if (srcdata
.length() == 0) {
3748 available_objects
.erase(old_obj
);
3749 ghobject_t new_obj
= get_uniform_random_object();
3750 available_objects
.erase(new_obj
);
3752 boost::uniform_int
<> u1(0, max_object_len
- max_write_len
);
3753 boost::uniform_int
<> u2(0, max_write_len
);
3754 uint64_t srcoff
= u1(*rng
);
3755 // make src and dst offsets match, since that's what the osd does
3756 uint64_t dstoff
= srcoff
; //u1(*rng);
3757 uint64_t len
= u2(*rng
);
3758 if (write_alignment
) {
3759 srcoff
= ROUND_UP_TO(srcoff
, write_alignment
);
3760 dstoff
= ROUND_UP_TO(dstoff
, write_alignment
);
3761 len
= ROUND_UP_TO(len
, write_alignment
);
3764 if (srcoff
> srcdata
.length() - 1) {
3765 srcoff
= srcdata
.length() - 1;
3767 if (srcoff
+ len
> srcdata
.length()) {
3768 len
= srcdata
.length() - srcoff
;
3771 cout
<< __func__
<< " from " << srcoff
<< "~" << len
3772 << " (size " << srcdata
.length() << ") to "
3773 << dstoff
<< "~" << len
<< std::endl
;
3775 ObjectStore::Transaction t
;
3776 t
.clone_range(cid
, old_obj
, new_obj
, srcoff
, len
, dstoff
);
3778 in_flight_objects
.insert(old_obj
);
3781 if (srcoff
< srcdata
.length()) {
3782 if (srcoff
+ len
> srcdata
.length()) {
3783 bl
.substr_of(srcdata
, srcoff
, srcdata
.length() - srcoff
);
3785 bl
.substr_of(srcdata
, srcoff
, len
);
3789 bufferlist
& dstdata
= contents
[new_obj
].data
;
3790 if (dstdata
.length() <= dstoff
) {
3791 if (bl
.length() > 0) {
3792 dstdata
.append_zero(dstoff
- dstdata
.length());
3797 assert(dstdata
.length() > dstoff
);
3798 dstdata
.copy(0, dstoff
, value
);
3800 if (value
.length() < dstdata
.length())
3801 dstdata
.copy(value
.length(),
3802 dstdata
.length() - value
.length(), value
);
3803 value
.swap(dstdata
);
3806 int status
= store
->queue_transaction(
3807 osr
, std::move(t
), new C_SyntheticOnClone(this, old_obj
, new_obj
));
3813 Mutex::Locker
locker(lock
);
3814 EnterExit
ee("write");
3819 ghobject_t new_obj
= get_uniform_random_object();
3820 available_objects
.erase(new_obj
);
3821 ObjectStore::Transaction t
;
3823 boost::uniform_int
<> u1(0, max_object_len
- max_write_len
);
3824 boost::uniform_int
<> u2(0, max_write_len
);
3825 uint64_t offset
= u1(*rng
);
3826 uint64_t len
= u2(*rng
);
3828 if (write_alignment
) {
3829 offset
= ROUND_UP_TO(offset
, write_alignment
);
3830 len
= ROUND_UP_TO(len
, write_alignment
);
3833 filled_byte_array(bl
, len
);
3835 bufferlist
& data
= contents
[new_obj
].data
;
3836 if (data
.length() <= offset
) {
3838 data
.append_zero(offset
-data
.length());
3843 assert(data
.length() > offset
);
3844 data
.copy(0, offset
, value
);
3846 if (value
.length() < data
.length())
3847 data
.copy(value
.length(),
3848 data
.length()-value
.length(), value
);
3852 t
.write(cid
, new_obj
, offset
, len
, bl
);
3854 in_flight_objects
.insert(new_obj
);
3855 int status
= store
->queue_transaction(
3856 osr
, std::move(t
), new C_SyntheticOnReadable(this, new_obj
));
3861 Mutex::Locker
locker(lock
);
3862 EnterExit
ee("truncate");
3867 ghobject_t obj
= get_uniform_random_object();
3868 available_objects
.erase(obj
);
3869 ObjectStore::Transaction t
;
3871 boost::uniform_int
<> choose(0, max_object_len
);
3872 size_t len
= choose(*rng
);
3873 if (write_alignment
) {
3874 len
= ROUND_UP_TO(len
, write_alignment
);
3877 t
.truncate(cid
, obj
, len
);
3879 in_flight_objects
.insert(obj
);
3880 bufferlist
& data
= contents
[obj
].data
;
3881 if (data
.length() <= len
) {
3882 data
.append_zero(len
- data
.length());
3885 data
.copy(0, len
, bl
);
3889 int status
= store
->queue_transaction(
3890 osr
, std::move(t
), new C_SyntheticOnReadable(this, obj
));
3895 Mutex::Locker
locker(lock
);
3896 EnterExit
ee("zero");
3901 ghobject_t new_obj
= get_uniform_random_object();
3902 available_objects
.erase(new_obj
);
3903 ObjectStore::Transaction t
;
3905 boost::uniform_int
<> u1(0, max_object_len
- max_write_len
);
3906 boost::uniform_int
<> u2(0, max_write_len
);
3907 uint64_t offset
= u1(*rng
);
3908 uint64_t len
= u2(*rng
);
3909 if (write_alignment
) {
3910 offset
= ROUND_UP_TO(offset
, write_alignment
);
3911 len
= ROUND_UP_TO(len
, write_alignment
);
3915 auto& data
= contents
[new_obj
].data
;
3916 if (data
.length() < offset
+ len
) {
3917 data
.append_zero(offset
+len
-data
.length());
3920 n
.substr_of(data
, 0, offset
);
3922 if (data
.length() > offset
+ len
)
3923 data
.copy(offset
+ len
, data
.length() - offset
- len
, n
);
3927 t
.zero(cid
, new_obj
, offset
, len
);
3929 in_flight_objects
.insert(new_obj
);
3930 int status
= store
->queue_transaction(
3931 osr
, std::move(t
), new C_SyntheticOnReadable(this, new_obj
));
3936 EnterExit
ee("read");
3937 boost::uniform_int
<> u1(0, max_object_len
/2);
3938 boost::uniform_int
<> u2(0, max_object_len
);
3939 uint64_t offset
= u1(*rng
);
3940 uint64_t len
= u2(*rng
);
3945 bufferlist expected
;
3948 Mutex::Locker
locker(lock
);
3949 EnterExit
ee("read locked");
3954 obj
= get_uniform_random_object();
3955 expected
= contents
[obj
].data
;
3957 bufferlist bl
, result
;
3958 if (0) cout
<< " obj " << obj
3959 << " size " << expected
.length()
3960 << " offset " << offset
3961 << " len " << len
<< std::endl
;
3962 r
= store
->read(cid
, obj
, offset
, len
, result
);
3963 if (offset
>= expected
.length()) {
3966 size_t max_len
= expected
.length() - offset
;
3969 assert(len
== result
.length());
3970 ASSERT_EQ(len
, result
.length());
3971 expected
.copy(offset
, len
, bl
);
3972 ASSERT_EQ(r
, (int)len
);
3973 ASSERT_TRUE(bl_eq(bl
, result
));
3978 Mutex::Locker
locker(lock
);
3979 EnterExit
ee("setattrs");
3984 ghobject_t obj
= get_uniform_random_object();
3985 available_objects
.erase(obj
);
3986 ObjectStore::Transaction t
;
3988 boost::uniform_int
<> u0(1, max_attr_size
);
3989 boost::uniform_int
<> u1(4, max_attr_name_len
);
3990 boost::uniform_int
<> u2(4, max_attr_value_len
);
3991 boost::uniform_int
<> u3(0, 100);
3992 uint64_t size
= u0(*rng
);
3994 map
<string
, bufferlist
> attrs
;
3996 for (map
<string
, bufferlist
>::iterator it
= contents
[obj
].attrs
.begin();
3997 it
!= contents
[obj
].attrs
.end(); ++it
)
3998 keys
.insert(it
->first
);
4001 bufferlist name
, value
;
4002 uint64_t get_exist
= u3(*rng
);
4003 uint64_t value_len
= u2(*rng
);
4004 filled_byte_array(value
, value_len
);
4005 if (get_exist
< 50 && keys
.size()) {
4006 set
<string
>::iterator k
= keys
.begin();
4008 contents
[obj
].attrs
[*k
] = value
;
4011 name_len
= u1(*rng
);
4012 filled_byte_array(name
, name_len
);
4013 attrs
[name
.c_str()] = value
;
4014 contents
[obj
].attrs
[name
.c_str()] = value
;
4017 t
.setattrs(cid
, obj
, attrs
);
4019 in_flight_objects
.insert(obj
);
4020 int status
= store
->queue_transaction(
4021 osr
, std::move(t
), new C_SyntheticOnReadable(this, obj
));
4026 EnterExit
ee("getattrs");
4028 map
<string
, bufferlist
> expected
;
4030 Mutex::Locker
locker(lock
);
4031 EnterExit
ee("getattrs locked");
4038 obj
= get_uniform_random_object();
4041 } while (contents
[obj
].attrs
.empty());
4042 expected
= contents
[obj
].attrs
;
4044 map
<string
, bufferlist
> attrs
;
4045 int r
= store
->getattrs(cid
, obj
, attrs
);
4046 ASSERT_TRUE(r
== 0);
4047 ASSERT_TRUE(attrs
.size() == expected
.size());
4048 for (map
<string
, bufferlist
>::iterator it
= expected
.begin();
4049 it
!= expected
.end(); ++it
) {
4050 ASSERT_TRUE(bl_eq(attrs
[it
->first
], it
->second
));
4055 EnterExit
ee("getattr");
4059 map
<string
, bufferlist
> expected
;
4061 Mutex::Locker
locker(lock
);
4062 EnterExit
ee("getattr locked");
4069 obj
= get_uniform_random_object();
4072 } while (contents
[obj
].attrs
.empty());
4073 expected
= contents
[obj
].attrs
;
4075 boost::uniform_int
<> u(0, expected
.size()-1);
4077 map
<string
, bufferlist
>::iterator it
= expected
.begin();
4084 r
= store
->getattr(cid
, obj
, it
->first
, bl
);
4086 ASSERT_TRUE(bl_eq(it
->second
, bl
));
4090 Mutex::Locker
locker(lock
);
4091 EnterExit
ee("rmattr");
4099 obj
= get_uniform_random_object();
4102 } while (contents
[obj
].attrs
.empty());
4104 boost::uniform_int
<> u(0, contents
[obj
].attrs
.size()-1);
4106 map
<string
, bufferlist
>::iterator it
= contents
[obj
].attrs
.begin();
4112 available_objects
.erase(obj
);
4113 ObjectStore::Transaction t
;
4114 t
.rmattr(cid
, obj
, it
->first
);
4116 contents
[obj
].attrs
.erase(it
->first
);
4118 in_flight_objects
.insert(obj
);
4119 int status
= store
->queue_transaction(
4120 osr
, std::move(t
), new C_SyntheticOnReadable(this, obj
));
4124 void fsck(bool deep
) {
4125 Mutex::Locker
locker(lock
);
4126 EnterExit
ee("fsck");
4130 int r
= store
->fsck(deep
);
4131 assert(r
== 0 || r
== -EOPNOTSUPP
);
4136 Mutex::Locker
locker(lock
);
4137 EnterExit
ee("scan");
4140 vector
<ghobject_t
> objects
;
4141 set
<ghobject_t
> objects_set
, objects_set2
;
4142 ghobject_t next
, current
;
4144 //cerr << "scanning..." << std::endl;
4145 int r
= store
->collection_list(cid
, current
, ghobject_t::get_max(), 100,
4148 ASSERT_TRUE(sorted(objects
));
4149 objects_set
.insert(objects
.begin(), objects
.end());
4151 if (next
.is_max()) break;
4154 if (objects_set
.size() != available_objects
.size()) {
4155 for (set
<ghobject_t
>::iterator p
= objects_set
.begin();
4156 p
!= objects_set
.end();
4158 if (available_objects
.count(*p
) == 0) {
4159 cerr
<< "+ " << *p
<< std::endl
;
4162 for (set
<ghobject_t
>::iterator p
= available_objects
.begin();
4163 p
!= available_objects
.end();
4165 if (objects_set
.count(*p
) == 0)
4166 cerr
<< "- " << *p
<< std::endl
;
4167 //cerr << " objects_set: " << objects_set << std::endl;
4168 //cerr << " available_set: " << available_objects << std::endl;
4169 assert(0 == "badness");
4172 ASSERT_EQ(objects_set
.size(), available_objects
.size());
4173 for (set
<ghobject_t
>::iterator i
= objects_set
.begin();
4174 i
!= objects_set
.end();
4176 ASSERT_GT(available_objects
.count(*i
), (unsigned)0);
4179 int r
= store
->collection_list(cid
, ghobject_t(), ghobject_t::get_max(),
4180 INT_MAX
, &objects
, 0);
4182 objects_set2
.insert(objects
.begin(), objects
.end());
4183 ASSERT_EQ(objects_set2
.size(), available_objects
.size());
4184 for (set
<ghobject_t
>::iterator i
= objects_set2
.begin();
4185 i
!= objects_set2
.end();
4187 ASSERT_GT(available_objects
.count(*i
), (unsigned)0);
4188 if (available_objects
.count(*i
) == 0) {
4189 cerr
<< "+ " << *i
<< std::endl
;
4195 EnterExit
ee("stat");
4199 Mutex::Locker
locker(lock
);
4200 EnterExit
ee("stat lock1");
4203 hoid
= get_uniform_random_object();
4204 in_flight_objects
.insert(hoid
);
4205 available_objects
.erase(hoid
);
4207 expected
= contents
[hoid
].data
.length();
4210 int r
= store
->stat(cid
, hoid
, &buf
);
4212 assert((uint64_t)buf
.st_size
== expected
);
4213 ASSERT_TRUE((uint64_t)buf
.st_size
== expected
);
4215 Mutex::Locker
locker(lock
);
4216 EnterExit
ee("stat lock2");
4219 in_flight_objects
.erase(hoid
);
4220 available_objects
.insert(hoid
);
4225 Mutex::Locker
locker(lock
);
4226 EnterExit
ee("unlink");
4229 ghobject_t to_remove
= get_uniform_random_object();
4230 ObjectStore::Transaction t
;
4231 t
.remove(cid
, to_remove
);
4233 available_objects
.erase(to_remove
);
4234 in_flight_objects
.insert(to_remove
);
4235 contents
.erase(to_remove
);
4236 int status
= store
->queue_transaction(osr
, std::move(t
), new C_SyntheticOnReadable(this, to_remove
));
4240 void print_internal_state() {
4241 Mutex::Locker
locker(lock
);
4242 cerr
<< "available_objects: " << available_objects
.size()
4243 << " in_flight_objects: " << in_flight_objects
.size()
4244 << " total objects: " << in_flight_objects
.size() + available_objects
.size()
4245 << " in_flight " << in_flight
<< std::endl
;
4250 void doSyntheticTest(boost::scoped_ptr
<ObjectStore
>& store
,
4252 uint64_t max_obj
, uint64_t max_wr
, uint64_t align
)
4254 ObjectStore::Sequencer
osr("test");
4255 MixedGenerator
gen(555);
4256 gen_type
rng(time(NULL
));
4257 coll_t
cid(spg_t(pg_t(0,555), shard_id_t::NO_SHARD
));
4259 g_ceph_context
->_conf
->set_val("bluestore_fsck_on_mount", "false");
4260 g_ceph_context
->_conf
->set_val("bluestore_fsck_on_umount", "false");
4261 g_ceph_context
->_conf
->apply_changes(NULL
);
4263 SyntheticWorkloadState
test_obj(store
.get(), &gen
, &rng
, &osr
, cid
,
4264 max_obj
, max_wr
, align
);
4266 for (int i
= 0; i
< num_ops
/10; ++i
) {
4267 if (!(i
% 500)) cerr
<< "seeding object " << i
<< std::endl
;
4270 for (int i
= 0; i
< num_ops
; ++i
) {
4272 cerr
<< "Op " << i
<< std::endl
;
4273 test_obj
.print_internal_state();
4275 boost::uniform_int
<> true_false(0, 999);
4276 int val
= true_false(rng
);
4278 test_obj
.fsck(true);
4279 } else if (val
> 997) {
4280 test_obj
.fsck(false);
4281 } else if (val
> 970) {
4283 } else if (val
> 950) {
4285 } else if (val
> 850) {
4287 } else if (val
> 800) {
4289 } else if (val
> 550) {
4291 } else if (val
> 500) {
4293 } else if (val
> 450) {
4294 test_obj
.clone_range();
4295 } else if (val
> 300) {
4297 } else if (val
> 100) {
4300 test_obj
.truncate();
4303 test_obj
.wait_for_done();
4304 test_obj
.shutdown();
4306 g_ceph_context
->_conf
->set_val("bluestore_fsck_on_mount", "true");
4307 g_ceph_context
->_conf
->set_val("bluestore_fsck_on_umount", "true");
4308 g_ceph_context
->_conf
->apply_changes(NULL
);
4311 TEST_P(StoreTest
, Synthetic
) {
4312 doSyntheticTest(store
, 10000, 400*1024, 40*1024, 0);
4316 TEST_P(StoreTestSpecificAUSize
, SyntheticMatrixSharding
) {
4317 if (string(GetParam()) != "bluestore")
4320 const char *m
[][10] = {
4321 { "bluestore_min_alloc_size", "4096", 0 }, // must be the first!
4322 { "num_ops", "50000", 0 },
4323 { "max_write", "65536", 0 },
4324 { "max_size", "262144", 0 },
4325 { "alignment", "4096", 0 },
4326 { "bluestore_max_blob_size", "65536", 0 },
4327 { "bluestore_extent_map_shard_min_size", "60", 0 },
4328 { "bluestore_extent_map_shard_max_size", "300", 0 },
4329 { "bluestore_extent_map_shard_target_size", "150", 0 },
4330 { "bluestore_default_buffered_read", "true", 0 },
4331 { "bluestore_default_buffered_write", "true", 0 },
4334 do_matrix(m
, store
, doSyntheticTest
);
4337 TEST_P(StoreTestSpecificAUSize
, ZipperPatternSharded
) {
4338 if(string(GetParam()) != "bluestore")
4340 StartDeferred(4096);
4343 ObjectStore::Sequencer
osr("test");
4345 ghobject_t
a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
4347 ObjectStore::Transaction t
;
4348 t
.create_collection(cid
, 0);
4349 cerr
<< "Creating collection " << cid
<< std::endl
;
4350 r
= apply_transaction(store
, &osr
, std::move(t
));
4358 for (int i
=0; i
<1000; ++i
) {
4359 ObjectStore::Transaction t
;
4360 t
.write(cid
, a
, i
*2*len
, len
, bl
, 0);
4361 r
= apply_transaction(store
, &osr
, std::move(t
));
4364 for (int i
=0; i
<1000; ++i
) {
4365 ObjectStore::Transaction t
;
4366 t
.write(cid
, a
, i
*2*len
+ 1, len
, bl
, 0);
4367 r
= apply_transaction(store
, &osr
, std::move(t
));
4371 ObjectStore::Transaction t
;
4373 t
.remove_collection(cid
);
4374 cerr
<< "Cleaning" << std::endl
;
4375 r
= apply_transaction(store
, &osr
, std::move(t
));
4380 TEST_P(StoreTestSpecificAUSize
, SyntheticMatrixCsumAlgorithm
) {
4381 if (string(GetParam()) != "bluestore")
4384 const char *m
[][10] = {
4385 { "bluestore_min_alloc_size", "65536", 0 }, // must be the first!
4386 { "max_write", "65536", 0 },
4387 { "max_size", "1048576", 0 },
4388 { "alignment", "16", 0 },
4389 { "bluestore_csum_type", "crc32c", "crc32c_16", "crc32c_8", "xxhash32",
4390 "xxhash64", "none", 0 },
4391 { "bluestore_default_buffered_write", "false", 0 },
4394 do_matrix(m
, store
, doSyntheticTest
);
4397 TEST_P(StoreTestSpecificAUSize
, SyntheticMatrixCsumVsCompression
) {
4398 if (string(GetParam()) != "bluestore")
4401 const char *m
[][10] = {
4402 { "bluestore_min_alloc_size", "4096", "16384", 0 }, //to be the first!
4403 { "max_write", "131072", 0 },
4404 { "max_size", "262144", 0 },
4405 { "alignment", "512", 0 },
4406 { "bluestore_compression_mode", "force", 0},
4407 { "bluestore_compression_algorithm", "snappy", "zlib", 0 },
4408 { "bluestore_csum_type", "crc32c", 0 },
4409 { "bluestore_default_buffered_read", "true", "false", 0 },
4410 { "bluestore_default_buffered_write", "true", "false", 0 },
4411 { "bluestore_sync_submit_transaction", "false", 0 },
4414 do_matrix(m
, store
, doSyntheticTest
);
4417 TEST_P(StoreTestSpecificAUSize
, SyntheticMatrixCompression
) {
4418 if (string(GetParam()) != "bluestore")
4421 const char *m
[][10] = {
4422 { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first!
4423 { "max_write", "1048576", 0 },
4424 { "max_size", "4194304", 0 },
4425 { "alignment", "65536", 0 },
4426 { "bluestore_compression_mode", "force", "aggressive", "passive", "none", 0},
4427 { "bluestore_default_buffered_write", "false", 0 },
4428 { "bluestore_sync_submit_transaction", "true", 0 },
4431 do_matrix(m
, store
, doSyntheticTest
);
4434 TEST_P(StoreTestSpecificAUSize
, SyntheticMatrixCompressionAlgorithm
) {
4435 if (string(GetParam()) != "bluestore")
4438 const char *m
[][10] = {
4439 { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first!
4440 { "max_write", "1048576", 0 },
4441 { "max_size", "4194304", 0 },
4442 { "alignment", "65536", 0 },
4443 { "bluestore_compression_algorithm", "zlib", "snappy", 0 },
4444 { "bluestore_compression_mode", "force", 0 },
4445 { "bluestore_default_buffered_write", "false", 0 },
4448 do_matrix(m
, store
, doSyntheticTest
);
4451 TEST_P(StoreTestSpecificAUSize
, SyntheticMatrixNoCsum
) {
4452 if (string(GetParam()) != "bluestore")
4455 const char *m
[][10] = {
4456 { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first!
4457 { "max_write", "65536", 0 },
4458 { "max_size", "1048576", 0 },
4459 { "alignment", "512", 0 },
4460 { "bluestore_max_blob_size", "262144", 0 },
4461 { "bluestore_compression_mode", "force", "none", 0},
4462 { "bluestore_csum_type", "none", 0},
4463 { "bluestore_default_buffered_read", "true", "false", 0 },
4464 { "bluestore_default_buffered_write", "true", 0 },
4465 { "bluestore_sync_submit_transaction", "true", "false", 0 },
4468 do_matrix(m
, store
, doSyntheticTest
);
4471 TEST_P(StoreTestSpecificAUSize
, SyntheticMatrixPreferDeferred
) {
4472 if (string(GetParam()) != "bluestore")
4475 const char *m
[][10] = {
4476 { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first!
4477 { "max_write", "65536", 0 },
4478 { "max_size", "1048576", 0 },
4479 { "alignment", "512", 0 },
4480 { "bluestore_max_blob_size", "262144", 0 },
4481 { "bluestore_compression_mode", "force", "none", 0},
4482 { "bluestore_prefer_deferred_size", "32768", "0", 0},
4485 do_matrix(m
, store
, doSyntheticTest
);
4488 TEST_P(StoreTest
, AttrSynthetic
) {
4489 ObjectStore::Sequencer
osr("test");
4490 MixedGenerator
gen(447);
4491 gen_type
rng(time(NULL
));
4492 coll_t
cid(spg_t(pg_t(0,447),shard_id_t::NO_SHARD
));
4494 SyntheticWorkloadState
test_obj(store
.get(), &gen
, &rng
, &osr
, cid
, 40*1024, 4*1024, 0);
4496 for (int i
= 0; i
< 500; ++i
) {
4497 if (!(i
% 10)) cerr
<< "seeding object " << i
<< std::endl
;
4500 for (int i
= 0; i
< 1000; ++i
) {
4502 cerr
<< "Op " << i
<< std::endl
;
4503 test_obj
.print_internal_state();
4505 boost::uniform_int
<> true_false(0, 99);
4506 int val
= true_false(rng
);
4509 } else if (val
> 93) {
4511 } else if (val
> 75) {
4513 } else if (val
> 47) {
4514 test_obj
.setattrs();
4515 } else if (val
> 45) {
4517 } else if (val
> 37) {
4519 } else if (val
> 30) {
4520 test_obj
.getattrs();
4525 test_obj
.wait_for_done();
4526 test_obj
.shutdown();
4529 TEST_P(StoreTest
, HashCollisionTest
) {
4530 ObjectStore::Sequencer
osr("test");
4531 int64_t poolid
= 11;
4532 coll_t
cid(spg_t(pg_t(0,poolid
),shard_id_t::NO_SHARD
));
4535 ObjectStore::Transaction t
;
4536 t
.create_collection(cid
, 0);
4537 r
= apply_transaction(store
, &osr
, std::move(t
));
4541 for (int i
= 0; i
< 100; ++i
) base
.append("aaaaa");
4542 set
<ghobject_t
> created
;
4543 for (int n
= 0; n
< 10; ++n
) {
4545 sprintf(nbuf
, "n%d", n
);
4546 for (int i
= 0; i
< 1000; ++i
) {
4548 sprintf(buf
, "%d", i
);
4550 cerr
<< "Object n" << n
<< " "<< i
<< std::endl
;
4552 ghobject_t
hoid(hobject_t(string(buf
) + base
, string(), CEPH_NOSNAP
, 0, poolid
, string(nbuf
)));
4554 ObjectStore::Transaction t
;
4556 r
= apply_transaction(store
, &osr
, std::move(t
));
4559 created
.insert(hoid
);
4562 vector
<ghobject_t
> objects
;
4563 r
= store
->collection_list(cid
, ghobject_t(), ghobject_t::get_max(), INT_MAX
, &objects
, 0);
4565 set
<ghobject_t
> listed(objects
.begin(), objects
.end());
4566 cerr
<< "listed.size() is " << listed
.size() << " and created.size() is " << created
.size() << std::endl
;
4567 ASSERT_TRUE(listed
.size() == created
.size());
4570 ghobject_t current
, next
;
4572 r
= store
->collection_list(cid
, current
, ghobject_t::get_max(), 60,
4575 ASSERT_TRUE(sorted(objects
));
4576 for (vector
<ghobject_t
>::iterator i
= objects
.begin();
4579 if (listed
.count(*i
))
4580 cerr
<< *i
<< " repeated" << std::endl
;
4583 if (objects
.size() < 50) {
4584 ASSERT_TRUE(next
.is_max());
4590 cerr
<< "listed.size() is " << listed
.size() << std::endl
;
4591 ASSERT_TRUE(listed
.size() == created
.size());
4592 for (set
<ghobject_t
>::iterator i
= listed
.begin();
4595 ASSERT_TRUE(created
.count(*i
));
4598 for (set
<ghobject_t
>::iterator i
= created
.begin();
4601 ObjectStore::Transaction t
;
4603 r
= apply_transaction(store
, &osr
, std::move(t
));
4606 ObjectStore::Transaction t
;
4607 t
.remove_collection(cid
);
4608 r
= apply_transaction(store
, &osr
, std::move(t
));
4612 TEST_P(StoreTest
, ScrubTest
) {
4613 ObjectStore::Sequencer
osr("test");
4614 int64_t poolid
= 111;
4615 coll_t
cid(spg_t(pg_t(0, poolid
),shard_id_t(1)));
4618 ObjectStore::Transaction t
;
4619 t
.create_collection(cid
, 0);
4620 r
= apply_transaction(store
, &osr
, std::move(t
));
4623 string base
= "aaaaa";
4624 set
<ghobject_t
> created
;
4625 for (int i
= 0; i
< 1000; ++i
) {
4627 sprintf(buf
, "%d", i
);
4629 cerr
<< "Object " << i
<< std::endl
;
4631 ghobject_t
hoid(hobject_t(string(buf
) + base
, string(), CEPH_NOSNAP
, i
,
4633 ghobject_t::NO_GEN
, shard_id_t(1));
4635 ObjectStore::Transaction t
;
4637 r
= apply_transaction(store
, &osr
, std::move(t
));
4640 created
.insert(hoid
);
4643 // Add same hobject_t but different generation
4645 ghobject_t
hoid1(hobject_t("same-object", string(), CEPH_NOSNAP
, 0, poolid
, ""),
4646 ghobject_t::NO_GEN
, shard_id_t(1));
4647 ghobject_t
hoid2(hobject_t("same-object", string(), CEPH_NOSNAP
, 0, poolid
, ""), (gen_t
)1, shard_id_t(1));
4648 ghobject_t
hoid3(hobject_t("same-object", string(), CEPH_NOSNAP
, 0, poolid
, ""), (gen_t
)2, shard_id_t(1));
4649 ObjectStore::Transaction t
;
4650 t
.touch(cid
, hoid1
);
4651 t
.touch(cid
, hoid2
);
4652 t
.touch(cid
, hoid3
);
4653 r
= apply_transaction(store
, &osr
, std::move(t
));
4654 created
.insert(hoid1
);
4655 created
.insert(hoid2
);
4656 created
.insert(hoid3
);
4660 vector
<ghobject_t
> objects
;
4661 r
= store
->collection_list(cid
, ghobject_t(), ghobject_t::get_max(),
4662 INT_MAX
, &objects
, 0);
4664 set
<ghobject_t
> listed(objects
.begin(), objects
.end());
4665 cerr
<< "listed.size() is " << listed
.size() << " and created.size() is " << created
.size() << std::endl
;
4666 ASSERT_TRUE(listed
.size() == created
.size());
4669 ghobject_t current
, next
;
4671 r
= store
->collection_list(cid
, current
, ghobject_t::get_max(), 60,
4674 ASSERT_TRUE(sorted(objects
));
4675 for (vector
<ghobject_t
>::iterator i
= objects
.begin();
4676 i
!= objects
.end(); ++i
) {
4677 if (listed
.count(*i
))
4678 cerr
<< *i
<< " repeated" << std::endl
;
4681 if (objects
.size() < 50) {
4682 ASSERT_TRUE(next
.is_max());
4686 current
= next
.get_boundary();
4688 cerr
<< "listed.size() is " << listed
.size() << std::endl
;
4689 ASSERT_TRUE(listed
.size() == created
.size());
4690 for (set
<ghobject_t
>::iterator i
= listed
.begin();
4693 ASSERT_TRUE(created
.count(*i
));
4696 for (set
<ghobject_t
>::iterator i
= created
.begin();
4699 ObjectStore::Transaction t
;
4701 r
= apply_transaction(store
, &osr
, std::move(t
));
4704 ObjectStore::Transaction t
;
4705 t
.remove_collection(cid
);
4706 r
= apply_transaction(store
, &osr
, std::move(t
));
4711 TEST_P(StoreTest
, OMapTest
) {
4712 ObjectStore::Sequencer
osr("test");
4714 ghobject_t
hoid(hobject_t("tesomap", "", CEPH_NOSNAP
, 0, 0, ""));
4717 ObjectStore::Transaction t
;
4718 t
.create_collection(cid
, 0);
4719 r
= apply_transaction(store
, &osr
, std::move(t
));
4723 map
<string
, bufferlist
> attrs
;
4725 ObjectStore::Transaction t
;
4727 t
.omap_clear(cid
, hoid
);
4728 map
<string
, bufferlist
> start_set
;
4729 t
.omap_setkeys(cid
, hoid
, start_set
);
4730 r
= apply_transaction(store
, &osr
, std::move(t
));
4734 for (int i
= 0; i
< 100; i
++) {
4736 std::cout
<< "On iteration " << i
<< std::endl
;
4738 ObjectStore::Transaction t
;
4740 map
<string
, bufferlist
> cur_attrs
;
4741 r
= store
->omap_get(cid
, hoid
, &bl
, &cur_attrs
);
4743 for (map
<string
, bufferlist
>::iterator j
= attrs
.begin();
4746 bool correct
= cur_attrs
.count(j
->first
) && string(cur_attrs
[j
->first
].c_str()) == string(j
->second
.c_str());
4748 std::cout
<< j
->first
<< " is present in cur_attrs " << cur_attrs
.count(j
->first
) << " times " << std::endl
;
4749 if (cur_attrs
.count(j
->first
) > 0) {
4750 std::cout
<< j
->second
.c_str() << " : " << cur_attrs
[j
->first
].c_str() << std::endl
;
4753 ASSERT_EQ(correct
, true);
4755 ASSERT_EQ(attrs
.size(), cur_attrs
.size());
4758 snprintf(buf
, sizeof(buf
), "%d", i
);
4760 bufferptr
bp(buf
, strlen(buf
) + 1);
4762 map
<string
, bufferlist
> to_add
;
4763 to_add
.insert(pair
<string
, bufferlist
>("key-" + string(buf
), bl
));
4764 attrs
.insert(pair
<string
, bufferlist
>("key-" + string(buf
), bl
));
4765 t
.omap_setkeys(cid
, hoid
, to_add
);
4766 r
= apply_transaction(store
, &osr
, std::move(t
));
4771 while (attrs
.size()) {
4773 std::cout
<< "removal: On iteration " << i
<< std::endl
;
4775 ObjectStore::Transaction t
;
4777 map
<string
, bufferlist
> cur_attrs
;
4778 r
= store
->omap_get(cid
, hoid
, &bl
, &cur_attrs
);
4780 for (map
<string
, bufferlist
>::iterator j
= attrs
.begin();
4783 bool correct
= cur_attrs
.count(j
->first
) && string(cur_attrs
[j
->first
].c_str()) == string(j
->second
.c_str());
4785 std::cout
<< j
->first
<< " is present in cur_attrs " << cur_attrs
.count(j
->first
) << " times " << std::endl
;
4786 if (cur_attrs
.count(j
->first
) > 0) {
4787 std::cout
<< j
->second
.c_str() << " : " << cur_attrs
[j
->first
].c_str() << std::endl
;
4790 ASSERT_EQ(correct
, true);
4793 string to_remove
= attrs
.begin()->first
;
4794 set
<string
> keys_to_remove
;
4795 keys_to_remove
.insert(to_remove
);
4796 t
.omap_rmkeys(cid
, hoid
, keys_to_remove
);
4797 r
= apply_transaction(store
, &osr
, std::move(t
));
4800 attrs
.erase(to_remove
);
4807 bl1
.append("omap_header");
4808 ObjectStore::Transaction t
;
4809 t
.omap_setheader(cid
, hoid
, bl1
);
4810 r
= apply_transaction(store
, &osr
, std::move(t
));
4812 t
= ObjectStore::Transaction();
4815 bl2
.append("value");
4816 map
<string
, bufferlist
> to_add
;
4817 to_add
.insert(pair
<string
, bufferlist
>("key", bl2
));
4818 t
.omap_setkeys(cid
, hoid
, to_add
);
4819 r
= apply_transaction(store
, &osr
, std::move(t
));
4823 map
<string
, bufferlist
> cur_attrs
;
4824 r
= store
->omap_get(cid
, hoid
, &bl3
, &cur_attrs
);
4826 ASSERT_EQ(cur_attrs
.size(), size_t(1));
4827 ASSERT_TRUE(bl_eq(bl1
, bl3
));
4830 r
= store
->omap_get_keys(cid
, hoid
, &keys
);
4832 ASSERT_EQ(keys
.size(), size_t(1));
4835 // test omap_clear, omap_rmkey_range
4838 map
<string
,bufferlist
> to_set
;
4839 for (int n
=0; n
<10; ++n
) {
4840 to_set
[stringify(n
)].append("foo");
4844 ObjectStore::Transaction t
;
4845 t
.remove(cid
, hoid
);
4847 t
.omap_setheader(cid
, hoid
, h
);
4848 t
.omap_setkeys(cid
, hoid
, to_set
);
4849 r
= apply_transaction(store
, &osr
, std::move(t
));
4853 ObjectStore::Transaction t
;
4854 t
.omap_rmkeyrange(cid
, hoid
, "3", "7");
4855 r
= apply_transaction(store
, &osr
, std::move(t
));
4860 map
<string
,bufferlist
> m
;
4861 store
->omap_get(cid
, hoid
, &hdr
, &m
);
4862 ASSERT_EQ(6u, hdr
.length());
4863 ASSERT_TRUE(m
.count("2"));
4864 ASSERT_TRUE(!m
.count("3"));
4865 ASSERT_TRUE(!m
.count("6"));
4866 ASSERT_TRUE(m
.count("7"));
4867 ASSERT_TRUE(m
.count("8"));
4868 //cout << m << std::endl;
4869 ASSERT_EQ(6u, m
.size());
4872 ObjectStore::Transaction t
;
4873 t
.omap_clear(cid
, hoid
);
4874 r
= apply_transaction(store
, &osr
, std::move(t
));
4879 map
<string
,bufferlist
> m
;
4880 store
->omap_get(cid
, hoid
, &hdr
, &m
);
4881 ASSERT_EQ(0u, hdr
.length());
4882 ASSERT_EQ(0u, m
.size());
4886 ObjectStore::Transaction t
;
4887 t
.remove(cid
, hoid
);
4888 t
.remove_collection(cid
);
4889 r
= apply_transaction(store
, &osr
, std::move(t
));
4893 TEST_P(StoreTest
, OMapIterator
) {
4894 ObjectStore::Sequencer
osr("test");
4896 ghobject_t
hoid(hobject_t("tesomap", "", CEPH_NOSNAP
, 0, 0, ""));
4900 ObjectStore::Transaction t
;
4901 t
.create_collection(cid
, 0);
4902 r
= apply_transaction(store
, &osr
, std::move(t
));
4906 map
<string
, bufferlist
> attrs
;
4908 ObjectStore::Transaction t
;
4910 t
.omap_clear(cid
, hoid
);
4911 map
<string
, bufferlist
> start_set
;
4912 t
.omap_setkeys(cid
, hoid
, start_set
);
4913 r
= apply_transaction(store
, &osr
, std::move(t
));
4916 ObjectMap::ObjectMapIterator iter
;
4919 for (int i
= 0; i
< 100; i
++) {
4921 std::cout
<< "On iteration " << i
<< std::endl
;
4925 // FileStore may deadlock two active iterators over the same data
4926 iter
= ObjectMap::ObjectMapIterator();
4928 iter
= store
->get_omap_iterator(cid
, hoid
);
4929 for (iter
->seek_to_first(), count
=0; iter
->valid(); iter
->next(), count
++) {
4930 string key
= iter
->key();
4931 bufferlist value
= iter
->value();
4932 correct
= attrs
.count(key
) && (string(value
.c_str()) == string(attrs
[key
].c_str()));
4934 if (attrs
.count(key
) > 0) {
4935 std::cout
<< "key " << key
<< "in omap , " << value
.c_str() << " : " << attrs
[key
].c_str() << std::endl
;
4938 std::cout
<< "key " << key
<< "should not exists in omap" << std::endl
;
4940 ASSERT_EQ(correct
, true);
4942 ASSERT_EQ((int)attrs
.size(), count
);
4944 // FileStore may deadlock an active iterator vs apply_transaction
4945 iter
= ObjectMap::ObjectMapIterator();
4948 snprintf(buf
, sizeof(buf
), "%d", i
);
4950 bufferptr
bp(buf
, strlen(buf
) + 1);
4952 map
<string
, bufferlist
> to_add
;
4953 to_add
.insert(pair
<string
, bufferlist
>("key-" + string(buf
), bl
));
4954 attrs
.insert(pair
<string
, bufferlist
>("key-" + string(buf
), bl
));
4955 ObjectStore::Transaction t
;
4956 t
.omap_setkeys(cid
, hoid
, to_add
);
4957 r
= apply_transaction(store
, &osr
, std::move(t
));
4961 iter
= store
->get_omap_iterator(cid
, hoid
);
4963 string bound_key
= "key-5";
4964 iter
->lower_bound(bound_key
);
4965 correct
= bound_key
<= iter
->key();
4967 std::cout
<< "lower bound, bound key is " << bound_key
<< " < iter key is " << iter
->key() << std::endl
;
4969 ASSERT_EQ(correct
, true);
4971 iter
->upper_bound(bound_key
);
4972 correct
= iter
->key() > bound_key
;
4974 std::cout
<< "upper bound, bound key is " << bound_key
<< " >= iter key is " << iter
->key() << std::endl
;
4976 ASSERT_EQ(correct
, true);
4978 // FileStore may deadlock an active iterator vs apply_transaction
4979 iter
= ObjectMap::ObjectMapIterator();
4981 ObjectStore::Transaction t
;
4982 t
.remove(cid
, hoid
);
4983 t
.remove_collection(cid
);
4984 r
= apply_transaction(store
, &osr
, std::move(t
));
4989 TEST_P(StoreTest
, XattrTest
) {
4990 ObjectStore::Sequencer
osr("test");
4992 ghobject_t
hoid(hobject_t("tesomap", "", CEPH_NOSNAP
, 0, 0, ""));
4994 for (unsigned i
= 0; i
< 10000; ++i
) {
4998 for (unsigned i
= 0; i
< 10; ++i
) {
5003 ObjectStore::Transaction t
;
5004 t
.create_collection(cid
, 0);
5006 r
= apply_transaction(store
, &osr
, std::move(t
));
5010 map
<string
, bufferlist
> attrs
;
5012 ObjectStore::Transaction t
;
5013 t
.setattr(cid
, hoid
, "attr1", small
);
5014 attrs
["attr1"] = small
;
5015 t
.setattr(cid
, hoid
, "attr2", big
);
5016 attrs
["attr2"] = big
;
5017 t
.setattr(cid
, hoid
, "attr3", small
);
5018 attrs
["attr3"] = small
;
5019 t
.setattr(cid
, hoid
, "attr1", small
);
5020 attrs
["attr1"] = small
;
5021 t
.setattr(cid
, hoid
, "attr4", big
);
5022 attrs
["attr4"] = big
;
5023 t
.setattr(cid
, hoid
, "attr3", big
);
5024 attrs
["attr3"] = big
;
5025 r
= apply_transaction(store
, &osr
, std::move(t
));
5029 map
<string
, bufferptr
> aset
;
5030 store
->getattrs(cid
, hoid
, aset
);
5031 ASSERT_EQ(aset
.size(), attrs
.size());
5032 for (map
<string
, bufferptr
>::iterator i
= aset
.begin();
5036 bl
.push_back(i
->second
);
5037 ASSERT_TRUE(attrs
[i
->first
] == bl
);
5041 ObjectStore::Transaction t
;
5042 t
.rmattr(cid
, hoid
, "attr2");
5043 attrs
.erase("attr2");
5044 r
= apply_transaction(store
, &osr
, std::move(t
));
5049 store
->getattrs(cid
, hoid
, aset
);
5050 ASSERT_EQ(aset
.size(), attrs
.size());
5051 for (map
<string
, bufferptr
>::iterator i
= aset
.begin();
5055 bl
.push_back(i
->second
);
5056 ASSERT_TRUE(attrs
[i
->first
] == bl
);
5060 r
= store
->getattr(cid
, hoid
, "attr2", bp
);
5061 ASSERT_EQ(r
, -ENODATA
);
5063 r
= store
->getattr(cid
, hoid
, "attr3", bp
);
5067 ASSERT_TRUE(bl2
== attrs
["attr3"]);
5069 ObjectStore::Transaction t
;
5070 t
.remove(cid
, hoid
);
5071 t
.remove_collection(cid
);
5072 r
= apply_transaction(store
, &osr
, std::move(t
));
5078 unsigned num_objects
,
5079 unsigned common_suffix_size
,
5082 ObjectStore::Sequencer
osr("test");
5083 coll_t
cid(spg_t(pg_t(0,52),shard_id_t::NO_SHARD
));
5084 coll_t
tid(spg_t(pg_t(1<<common_suffix_size
,52),shard_id_t::NO_SHARD
));
5087 ObjectStore::Transaction t
;
5088 t
.create_collection(cid
, common_suffix_size
);
5089 r
= apply_transaction(store
, &osr
, std::move(t
));
5093 small
.append("small");
5095 ObjectStore::Transaction t
;
5096 for (uint32_t i
= 0; i
< (2 - (int)clones
)*num_objects
; ++i
) {
5097 stringstream objname
;
5098 objname
<< "obj" << i
;
5099 ghobject_t
a(hobject_t(
5103 i
<<common_suffix_size
,
5105 t
.write(cid
, a
, 0, small
.length(), small
,
5106 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
5108 objname
<< "-clone";
5109 ghobject_t
b(hobject_t(
5113 i
<<common_suffix_size
,
5118 r
= apply_transaction(store
, &osr
, std::move(t
));
5120 t
= ObjectStore::Transaction();
5123 r
= apply_transaction(store
, &osr
, std::move(t
));
5127 ObjectStore::Transaction t
;
5128 t
.create_collection(tid
, common_suffix_size
+ 1);
5129 t
.split_collection(cid
, common_suffix_size
+1, 1<<common_suffix_size
, tid
);
5130 r
= apply_transaction(store
, &osr
, std::move(t
));
5134 ObjectStore::Transaction t
;
5135 vector
<ghobject_t
> objects
;
5136 r
= store
->collection_list(cid
, ghobject_t(), ghobject_t::get_max(),
5137 INT_MAX
, &objects
, 0);
5139 ASSERT_EQ(objects
.size(), num_objects
);
5141 for (vector
<ghobject_t
>::iterator i
= objects
.begin();
5144 ASSERT_EQ(!!(i
->hobj
.get_hash() & (1<<common_suffix_size
)), 0u);
5148 r
= apply_transaction(store
, &osr
, std::move(t
));
5150 t
= ObjectStore::Transaction();
5155 r
= store
->collection_list(tid
, ghobject_t(), ghobject_t::get_max(),
5156 INT_MAX
, &objects
, 0);
5158 ASSERT_EQ(objects
.size(), num_objects
);
5159 for (vector
<ghobject_t
>::iterator i
= objects
.begin();
5162 ASSERT_EQ(!(i
->hobj
.get_hash() & (1<<common_suffix_size
)), 0u);
5166 r
= apply_transaction(store
, &osr
, std::move(t
));
5168 t
= ObjectStore::Transaction();
5172 t
.remove_collection(cid
);
5173 t
.remove_collection(tid
);
5174 r
= apply_transaction(store
, &osr
, std::move(t
));
5178 TEST_P(StoreTest
, ColSplitTest1
) {
5179 colsplittest(store
.get(), 10000, 11, false);
5181 TEST_P(StoreTest
, ColSplitTest1Clones
) {
5182 colsplittest(store
.get(), 10000, 11, true);
5184 TEST_P(StoreTest
, ColSplitTest2
) {
5185 colsplittest(store
.get(), 100, 7, false);
5187 TEST_P(StoreTest
, ColSplitTest2Clones
) {
5188 colsplittest(store
.get(), 100, 7, true);
5192 TEST_P(StoreTest
, ColSplitTest3
) {
5193 colsplittest(store
.get(), 100000, 25);
5198 * This test tests adding two different groups
5199 * of objects, each with 1 common prefix and 1
5200 * different prefix. We then remove half
5201 * in order to verify that the merging correctly
5202 * stops at the common prefix subdir. See bug
5204 TEST_P(StoreTest
, TwoHash
) {
5205 ObjectStore::Sequencer
osr("test");
5209 ObjectStore::Transaction t
;
5210 t
.create_collection(cid
, 0);
5211 r
= apply_transaction(store
, &osr
, std::move(t
));
5214 std::cout
<< "Making objects" << std::endl
;
5215 for (int i
= 0; i
< 360; ++i
) {
5216 ObjectStore::Transaction t
;
5220 o
.hobj
.set_hash((i
<< 16) | 0xA1);
5223 o
.hobj
.set_hash((i
<< 16) | 0xB1);
5225 r
= apply_transaction(store
, &osr
, std::move(t
));
5228 std::cout
<< "Removing half" << std::endl
;
5229 for (int i
= 1; i
< 8; ++i
) {
5230 ObjectStore::Transaction t
;
5233 o
.hobj
.set_hash((i
<< 16) | 0xA1);
5235 r
= apply_transaction(store
, &osr
, std::move(t
));
5238 std::cout
<< "Checking" << std::endl
;
5239 for (int i
= 1; i
< 8; ++i
) {
5240 ObjectStore::Transaction t
;
5242 o
.hobj
.set_hash((i
<< 16) | 0xA1);
5244 bool exists
= store
->exists(cid
, o
);
5245 ASSERT_EQ(exists
, false);
5249 o
.hobj
.set_hash(0xA1);
5251 bool exists
= store
->exists(cid
, o
);
5252 ASSERT_EQ(exists
, true);
5254 std::cout
<< "Cleanup" << std::endl
;
5255 for (int i
= 0; i
< 360; ++i
) {
5256 ObjectStore::Transaction t
;
5258 o
.hobj
.set_hash((i
<< 16) | 0xA1);
5261 o
.hobj
.set_hash((i
<< 16) | 0xB1);
5263 r
= apply_transaction(store
, &osr
, std::move(t
));
5266 ObjectStore::Transaction t
;
5267 t
.remove_collection(cid
);
5268 r
= apply_transaction(store
, &osr
, std::move(t
));
5272 TEST_P(StoreTest
, Rename
) {
5273 ObjectStore::Sequencer
osr("test");
5274 coll_t
cid(spg_t(pg_t(0, 2122),shard_id_t::NO_SHARD
));
5275 ghobject_t
srcoid(hobject_t("src_oid", "", CEPH_NOSNAP
, 0, 0, ""));
5276 ghobject_t
dstoid(hobject_t("dest_oid", "", CEPH_NOSNAP
, 0, 0, ""));
5282 ObjectStore::Transaction t
;
5283 t
.create_collection(cid
, 0);
5284 t
.write(cid
, srcoid
, 0, a
.length(), a
);
5285 r
= apply_transaction(store
, &osr
, std::move(t
));
5288 ASSERT_TRUE(store
->exists(cid
, srcoid
));
5290 ObjectStore::Transaction t
;
5291 t
.collection_move_rename(cid
, srcoid
, cid
, dstoid
);
5292 t
.write(cid
, srcoid
, 0, b
.length(), b
);
5293 t
.setattr(cid
, srcoid
, "attr", b
);
5294 r
= apply_transaction(store
, &osr
, std::move(t
));
5297 ASSERT_TRUE(store
->exists(cid
, srcoid
));
5298 ASSERT_TRUE(store
->exists(cid
, dstoid
));
5301 store
->read(cid
, srcoid
, 0, 3, bl
);
5302 ASSERT_TRUE(bl_eq(b
, bl
));
5303 store
->read(cid
, dstoid
, 0, 3, bl
);
5304 ASSERT_TRUE(bl_eq(a
, bl
));
5307 ObjectStore::Transaction t
;
5308 t
.remove(cid
, dstoid
);
5309 t
.collection_move_rename(cid
, srcoid
, cid
, dstoid
);
5310 r
= apply_transaction(store
, &osr
, std::move(t
));
5313 ASSERT_TRUE(store
->exists(cid
, dstoid
));
5314 ASSERT_FALSE(store
->exists(cid
, srcoid
));
5317 store
->read(cid
, dstoid
, 0, 3, bl
);
5318 ASSERT_TRUE(bl_eq(b
, bl
));
5321 ObjectStore::Transaction t
;
5322 t
.remove(cid
, dstoid
);
5323 t
.remove_collection(cid
);
5324 r
= apply_transaction(store
, &osr
, std::move(t
));
5329 TEST_P(StoreTest
, MoveRename
) {
5330 ObjectStore::Sequencer
osr("test");
5331 coll_t
cid(spg_t(pg_t(0, 212),shard_id_t::NO_SHARD
));
5332 ghobject_t
temp_oid(hobject_t("tmp_oid", "", CEPH_NOSNAP
, 0, 0, ""));
5333 ghobject_t
oid(hobject_t("dest_oid", "", CEPH_NOSNAP
, 0, 0, ""));
5336 ObjectStore::Transaction t
;
5337 t
.create_collection(cid
, 0);
5339 r
= apply_transaction(store
, &osr
, std::move(t
));
5342 ASSERT_TRUE(store
->exists(cid
, oid
));
5343 bufferlist data
, attr
;
5344 map
<string
, bufferlist
> omap
;
5345 data
.append("data payload");
5346 attr
.append("attr value");
5347 omap
["omap_key"].append("omap value");
5349 ObjectStore::Transaction t
;
5350 t
.touch(cid
, temp_oid
);
5351 t
.write(cid
, temp_oid
, 0, data
.length(), data
);
5352 t
.setattr(cid
, temp_oid
, "attr", attr
);
5353 t
.omap_setkeys(cid
, temp_oid
, omap
);
5354 r
= apply_transaction(store
, &osr
, std::move(t
));
5357 ASSERT_TRUE(store
->exists(cid
, temp_oid
));
5359 ObjectStore::Transaction t
;
5361 t
.collection_move_rename(cid
, temp_oid
, cid
, oid
);
5362 r
= apply_transaction(store
, &osr
, std::move(t
));
5365 ASSERT_TRUE(store
->exists(cid
, oid
));
5366 ASSERT_FALSE(store
->exists(cid
, temp_oid
));
5369 r
= store
->read(cid
, oid
, 0, 1000, newdata
);
5371 ASSERT_TRUE(bl_eq(data
, newdata
));
5373 r
= store
->getattr(cid
, oid
, "attr", newattr
);
5375 ASSERT_TRUE(bl_eq(attr
, newattr
));
5377 keys
.insert("omap_key");
5378 map
<string
, bufferlist
> newomap
;
5379 r
= store
->omap_get_values(cid
, oid
, keys
, &newomap
);
5381 ASSERT_EQ(1u, newomap
.size());
5382 ASSERT_TRUE(newomap
.count("omap_key"));
5383 ASSERT_TRUE(bl_eq(omap
["omap_key"], newomap
["omap_key"]));
5386 ObjectStore::Transaction t
;
5388 t
.remove_collection(cid
);
5389 r
= apply_transaction(store
, &osr
, std::move(t
));
5394 TEST_P(StoreTest
, BigRGWObjectName
) {
5395 ObjectStore::Sequencer
osr("test");
5396 coll_t
cid(spg_t(pg_t(0,12),shard_id_t::NO_SHARD
));
5399 "default.4106.50_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
5406 shard_id_t::NO_SHARD
);
5407 ghobject_t
oid2(oid
);
5408 oid2
.generation
= 17;
5409 ghobject_t
oidhead(oid
);
5410 oidhead
.generation
= ghobject_t::NO_GEN
;
5414 ObjectStore::Transaction t
;
5415 t
.create_collection(cid
, 0);
5416 t
.touch(cid
, oidhead
);
5417 t
.collection_move_rename(cid
, oidhead
, cid
, oid
);
5418 t
.touch(cid
, oidhead
);
5419 t
.collection_move_rename(cid
, oidhead
, cid
, oid2
);
5420 r
= apply_transaction(store
, &osr
, std::move(t
));
5425 ObjectStore::Transaction t
;
5427 r
= apply_transaction(store
, &osr
, std::move(t
));
5432 vector
<ghobject_t
> objects
;
5433 r
= store
->collection_list(cid
, ghobject_t(), ghobject_t::get_max(),
5434 INT_MAX
, &objects
, 0);
5436 ASSERT_EQ(objects
.size(), 1u);
5437 ASSERT_EQ(objects
[0], oid2
);
5440 ASSERT_FALSE(store
->exists(cid
, oid
));
5443 ObjectStore::Transaction t
;
5444 t
.remove(cid
, oid2
);
5445 t
.remove_collection(cid
);
5446 r
= apply_transaction(store
, &osr
, std::move(t
));
5452 TEST_P(StoreTest
, SetAllocHint
) {
5453 ObjectStore::Sequencer
osr("test");
5455 ghobject_t
hoid(hobject_t("test_hint", "", CEPH_NOSNAP
, 0, 0, ""));
5458 ObjectStore::Transaction t
;
5459 t
.create_collection(cid
, 0);
5461 r
= apply_transaction(store
, &osr
, std::move(t
));
5465 ObjectStore::Transaction t
;
5466 t
.set_alloc_hint(cid
, hoid
, 4*1024*1024, 1024*4, 0);
5467 r
= apply_transaction(store
, &osr
, std::move(t
));
5471 ObjectStore::Transaction t
;
5472 t
.remove(cid
, hoid
);
5473 r
= apply_transaction(store
, &osr
, std::move(t
));
5477 ObjectStore::Transaction t
;
5478 t
.set_alloc_hint(cid
, hoid
, 4*1024*1024, 1024*4, 0);
5479 r
= apply_transaction(store
, &osr
, std::move(t
));
5483 ObjectStore::Transaction t
;
5484 t
.remove_collection(cid
);
5485 r
= apply_transaction(store
, &osr
, std::move(t
));
5490 TEST_P(StoreTest
, TryMoveRename
) {
5491 ObjectStore::Sequencer
osr("test");
5493 ghobject_t
hoid(hobject_t("test_hint", "", CEPH_NOSNAP
, 0, -1, ""));
5494 ghobject_t
hoid2(hobject_t("test_hint2", "", CEPH_NOSNAP
, 0, -1, ""));
5497 ObjectStore::Transaction t
;
5498 t
.create_collection(cid
, 0);
5499 r
= apply_transaction(store
, &osr
, std::move(t
));
5503 ObjectStore::Transaction t
;
5504 t
.try_rename(cid
, hoid
, hoid2
);
5505 r
= apply_transaction(store
, &osr
, std::move(t
));
5509 ObjectStore::Transaction t
;
5511 r
= apply_transaction(store
, &osr
, std::move(t
));
5515 ObjectStore::Transaction t
;
5516 t
.try_rename(cid
, hoid
, hoid2
);
5517 r
= apply_transaction(store
, &osr
, std::move(t
));
5521 ASSERT_EQ(store
->stat(cid
, hoid
, &st
), -ENOENT
);
5522 ASSERT_EQ(store
->stat(cid
, hoid2
, &st
), 0);
5525 #if defined(HAVE_LIBAIO)
5526 TEST_P(StoreTest
, BluestoreOnOffCSumTest
) {
5527 if (string(GetParam()) != "bluestore")
5529 g_conf
->set_val("bluestore_csum_type", "crc32c");
5530 g_conf
->apply_changes(NULL
);
5532 ObjectStore::Sequencer
osr("test");
5535 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
5538 r
= store
->read(cid
, hoid
, 0, 5, in
);
5539 ASSERT_EQ(-ENOENT
, r
);
5542 ObjectStore::Transaction t
;
5543 t
.create_collection(cid
, 0);
5544 cerr
<< "Creating collection " << cid
<< std::endl
;
5545 r
= apply_transaction(store
, &osr
, std::move(t
));
5549 //write with csum enabled followed by read with csum disabled
5550 size_t block_size
= 64*1024;
5551 ObjectStore::Transaction t
;
5552 bufferlist bl
, orig
;
5553 bl
.append(std::string(block_size
, 'a'));
5555 t
.remove(cid
, hoid
);
5556 t
.set_alloc_hint(cid
, hoid
, 4*1024*1024, 1024*8, 0);
5557 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
5558 cerr
<< "Remove then create" << std::endl
;
5559 r
= apply_transaction(store
, &osr
, std::move(t
));
5562 g_conf
->set_val("bluestore_csum_type", "none");
5563 g_conf
->apply_changes(NULL
);
5566 r
= store
->read(cid
, hoid
, 0, block_size
, in
);
5567 ASSERT_EQ((int)block_size
, r
);
5568 ASSERT_TRUE(bl_eq(orig
, in
));
5572 //write with csum disabled followed by read with csum enabled
5574 size_t block_size
= 64*1024;
5575 ObjectStore::Transaction t
;
5576 bufferlist bl
, orig
;
5577 bl
.append(std::string(block_size
, 'a'));
5579 t
.remove(cid
, hoid
);
5580 t
.set_alloc_hint(cid
, hoid
, 4*1024*1024, 1024*8, 0);
5581 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
5582 cerr
<< "Remove then create" << std::endl
;
5583 r
= apply_transaction(store
, &osr
, std::move(t
));
5586 g_conf
->set_val("bluestore_csum_type", "crc32c");
5587 g_conf
->apply_changes(NULL
);
5590 r
= store
->read(cid
, hoid
, 0, block_size
, in
);
5591 ASSERT_EQ((int)block_size
, r
);
5592 ASSERT_TRUE(bl_eq(orig
, in
));
5595 //'mixed' non-overlapping writes to the same blob
5597 ObjectStore::Transaction t
;
5598 bufferlist bl
, orig
;
5599 size_t block_size
= 8000;
5600 bl
.append(std::string(block_size
, 'a'));
5602 t
.remove(cid
, hoid
);
5603 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
5604 cerr
<< "Remove then create" << std::endl
;
5605 r
= apply_transaction(store
, &osr
, std::move(t
));
5608 g_conf
->set_val("bluestore_csum_type", "none");
5609 g_conf
->apply_changes(NULL
);
5611 ObjectStore::Transaction t2
;
5612 t2
.write(cid
, hoid
, block_size
*2, bl
.length(), bl
);
5613 cerr
<< "Append 'unprotected'" << std::endl
;
5614 r
= apply_transaction(store
, &osr
, std::move(t2
));
5618 r
= store
->read(cid
, hoid
, 0, block_size
, in
);
5619 ASSERT_EQ((int)block_size
, r
);
5620 ASSERT_TRUE(bl_eq(orig
, in
));
5622 r
= store
->read(cid
, hoid
, block_size
*2, block_size
, in
);
5623 ASSERT_EQ((int)block_size
, r
);
5624 ASSERT_TRUE(bl_eq(orig
, in
));
5626 g_conf
->set_val("bluestore_csum_type", "crc32c");
5627 g_conf
->apply_changes(NULL
);
5629 r
= store
->read(cid
, hoid
, 0, block_size
, in
);
5630 ASSERT_EQ((int)block_size
, r
);
5631 ASSERT_TRUE(bl_eq(orig
, in
));
5633 r
= store
->read(cid
, hoid
, block_size
*2, block_size
, in
);
5634 ASSERT_EQ((int)block_size
, r
);
5635 ASSERT_TRUE(bl_eq(orig
, in
));
5638 //partially blob overwrite under a different csum enablement mode
5640 ObjectStore::Transaction t
;
5641 bufferlist bl
, orig
, orig2
;
5642 size_t block_size0
= 0x10000;
5643 size_t block_size
= 9000;
5644 size_t block_size2
= 5000;
5645 bl
.append(std::string(block_size0
, 'a'));
5646 t
.remove(cid
, hoid
);
5647 t
.set_alloc_hint(cid
, hoid
, 4*1024*1024, 1024*8, 0);
5648 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
5649 cerr
<< "Remove then create" << std::endl
;
5650 r
= apply_transaction(store
, &osr
, std::move(t
));
5653 g_conf
->set_val("bluestore_csum_type", "none");
5654 g_conf
->apply_changes(NULL
);
5656 ObjectStore::Transaction t2
;
5658 bl
.append(std::string(block_size
, 'b'));
5659 t2
.write(cid
, hoid
, 0, bl
.length(), bl
);
5660 t2
.write(cid
, hoid
, block_size0
, bl
.length(), bl
);
5661 cerr
<< "Overwrite with unprotected data" << std::endl
;
5662 r
= apply_transaction(store
, &osr
, std::move(t2
));
5667 orig
.append( std::string(block_size0
- block_size
, 'a'));
5670 r
= store
->read(cid
, hoid
, 0, block_size0
, in
);
5671 ASSERT_EQ((int)block_size0
, r
);
5672 ASSERT_TRUE(bl_eq(orig
, in
));
5674 r
= store
->read(cid
, hoid
, block_size0
, block_size
, in
);
5675 ASSERT_EQ((int)block_size
, r
);
5676 ASSERT_TRUE(bl_eq(orig2
, in
));
5678 g_conf
->set_val("bluestore_csum_type", "crc32c");
5679 g_conf
->apply_changes(NULL
);
5681 ObjectStore::Transaction t3
;
5683 bl
.append(std::string(block_size2
, 'c'));
5684 t3
.write(cid
, hoid
, block_size0
, bl
.length(), bl
);
5685 cerr
<< "Overwrite with protected data" << std::endl
;
5686 r
= apply_transaction(store
, &osr
, std::move(t3
));
5691 orig
.append( std::string(block_size
- block_size2
, 'b'));
5692 r
= store
->read(cid
, hoid
, block_size0
, block_size
, in
);
5693 ASSERT_EQ((int)block_size
, r
);
5694 ASSERT_TRUE(bl_eq(orig
, in
));
5698 ObjectStore::Transaction t
;
5699 t
.remove(cid
, hoid
);
5700 t
.remove_collection(cid
);
5701 cerr
<< "Cleaning" << std::endl
;
5702 r
= apply_transaction(store
, &osr
, std::move(t
));
5708 INSTANTIATE_TEST_CASE_P(
5714 #if defined(HAVE_LIBAIO)
5719 // Note: instantiate all stores to preserve store numbering order only
5720 INSTANTIATE_TEST_CASE_P(
5722 StoreTestSpecificAUSize
,
5726 #if defined(HAVE_LIBAIO)
5733 // Google Test may not support value-parameterized tests with some
5734 // compilers. If we use conditional compilation to compile out all
5735 // code referring to the gtest_main library, MSVC linker will not link
5736 // that library at all and consequently complain about missing entry
5737 // point defined in that library (fatal error LNK1561: entry point
5738 // must be defined). This dummy test keeps gtest_main linked in.
5739 TEST(DummyTest
, ValueParameterizedTestsAreNotSupportedOnThisPlatform
) {}
5743 void doMany4KWritesTest(boost::scoped_ptr
<ObjectStore
>& store
,
5744 unsigned max_objects
,
5746 unsigned max_object_size
,
5747 unsigned max_write_size
,
5748 unsigned write_alignment
,
5749 store_statfs_t
* res_stat
)
5751 ObjectStore::Sequencer
osr("test");
5752 MixedGenerator
gen(555);
5753 gen_type
rng(time(NULL
));
5754 coll_t
cid(spg_t(pg_t(0,555), shard_id_t::NO_SHARD
));
5756 SyntheticWorkloadState
test_obj(store
.get(),
5765 for (unsigned i
= 0; i
< max_objects
; ++i
) {
5766 if (!(i
% 500)) cerr
<< "seeding object " << i
<< std::endl
;
5769 for (unsigned i
= 0; i
< max_ops
; ++i
) {
5771 cerr
<< "Op " << i
<< std::endl
;
5772 test_obj
.print_internal_state();
5776 test_obj
.wait_for_done();
5778 test_obj
.statfs(*res_stat
);
5780 test_obj
.shutdown();
5783 TEST_P(StoreTestSpecificAUSize
, Many4KWritesTest
) {
5784 if (string(GetParam()) != "bluestore")
5787 StartDeferred(0x10000);
5789 store_statfs_t res_stat
;
5790 unsigned max_object
= 4*1024*1024;
5792 doMany4KWritesTest(store
, 1, 1000, 4*1024*1024, 4*1024, 0, &res_stat
);
5794 ASSERT_LE(res_stat
.stored
, max_object
);
5795 ASSERT_EQ(res_stat
.allocated
, max_object
);
5798 TEST_P(StoreTestSpecificAUSize
, Many4KWritesNoCSumTest
) {
5799 if (string(GetParam()) != "bluestore")
5801 StartDeferred(0x10000);
5802 g_conf
->set_val("bluestore_csum_type", "none");
5803 g_ceph_context
->_conf
->apply_changes(NULL
);
5804 store_statfs_t res_stat
;
5805 unsigned max_object
= 4*1024*1024;
5807 doMany4KWritesTest(store
, 1, 1000, max_object
, 4*1024, 0, &res_stat
);
5809 ASSERT_LE(res_stat
.stored
, max_object
);
5810 ASSERT_EQ(res_stat
.allocated
, max_object
);
5811 g_conf
->set_val("bluestore_csum_type", "crc32c");
5814 TEST_P(StoreTestSpecificAUSize
, TooManyBlobsTest
) {
5815 if (string(GetParam()) != "bluestore")
5817 StartDeferred(0x10000);
5818 store_statfs_t res_stat
;
5819 unsigned max_object
= 4*1024*1024;
5820 doMany4KWritesTest(store
, 1, 1000, max_object
, 4*1024, 0, &res_stat
);
5821 ASSERT_LE(res_stat
.stored
, max_object
);
5822 ASSERT_EQ(res_stat
.allocated
, max_object
);
5825 #if defined(HAVE_LIBAIO)
5826 void get_mempool_stats(uint64_t* total_bytes
, uint64_t* total_items
)
5828 uint64_t onode_allocated
= mempool::bluestore_cache_onode::allocated_bytes();
5829 uint64_t other_allocated
= mempool::bluestore_cache_other::allocated_bytes();
5831 uint64_t onode_items
= mempool::bluestore_cache_onode::allocated_items();
5832 uint64_t other_items
= mempool::bluestore_cache_other::allocated_items();
5833 cout
<< "onode(" << onode_allocated
<< "/" << onode_items
5834 << ") other(" << other_allocated
<< "/" << other_items
5835 << ")" << std::endl
;
5836 *total_bytes
= onode_allocated
+ other_allocated
;
5837 *total_items
= onode_items
;
5840 TEST_P(StoreTestSpecificAUSize
, OnodeSizeTracking
) {
5842 if (string(GetParam()) != "bluestore")
5845 size_t block_size
= 4096;
5846 StartDeferred(block_size
);
5847 g_conf
->set_val("bluestore_compression_mode", "none");
5848 g_conf
->set_val("bluestore_csum_type", "none");
5849 g_conf
->set_val("bluestore_cache_size_hdd", "400000000");
5850 g_conf
->set_val("bluestore_cache_size_ssd", "400000000");
5851 g_conf
->apply_changes(NULL
);
5853 ObjectStore::Sequencer
osr("test");
5856 ghobject_t
hoid(hobject_t("test_hint", "", CEPH_NOSNAP
, 0, -1, ""));
5857 size_t obj_size
= 4 * 1024 * 1024;
5858 uint64_t total_bytes
, total_bytes2
;
5859 uint64_t total_onodes
;
5860 get_mempool_stats(&total_bytes
, &total_onodes
);
5861 ASSERT_EQ(total_onodes
, 0u);
5864 ObjectStore::Transaction t
;
5865 t
.create_collection(cid
, 0);
5866 r
= apply_transaction(store
, &osr
, std::move(t
));
5870 ObjectStore::Transaction t
;
5871 bufferlist bl
, orig
, orig2
;
5873 bl
.append(std::string(obj_size
, 'a'));
5874 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
5875 r
= apply_transaction(store
, &osr
, std::move(t
));
5878 get_mempool_stats(&total_bytes
, &total_onodes
);
5879 ASSERT_NE(total_bytes
, 0u);
5880 ASSERT_EQ(total_onodes
, 1u);
5883 ObjectStore::Transaction t
;
5884 t
.truncate(cid
, hoid
, 0);
5885 r
= apply_transaction(store
, &osr
, std::move(t
));
5889 for(size_t i
= 0; i
< 1; ++i
) {
5891 bl
.append(std::string(block_size
* (i
+1), 'a'));
5892 for( size_t j
= 0; j
< obj_size
; j
+= bl
.length()) {
5893 ObjectStore::Transaction t
;
5894 t
.write(cid
, hoid
, j
, bl
.length(), bl
);
5895 r
= apply_transaction(store
, &osr
, std::move(t
));
5898 get_mempool_stats(&total_bytes2
, &total_onodes
);
5899 ASSERT_NE(total_bytes2
, 0u);
5900 ASSERT_EQ(total_onodes
, 1u);
5903 cout
<<" mempool dump:\n";
5904 JSONFormatter
f(true);
5905 f
.open_object_section("transaction");
5913 for (size_t i
= 0; i
< obj_size
; i
+= 0x1000) {
5914 store
->read(cid
, hoid
, i
, 0x1000, bl
);
5917 get_mempool_stats(&total_bytes
, &total_onodes
);
5918 ASSERT_NE(total_bytes
, 0u);
5919 ASSERT_EQ(total_onodes
, 1u);
5922 cout
<<" mempool dump:\n";
5923 JSONFormatter
f(true);
5924 f
.open_object_section("transaction");
5931 ObjectStore::Transaction t
;
5932 t
.remove(cid
, hoid
);
5933 t
.remove_collection(cid
);
5934 cerr
<< "Cleaning" << std::endl
;
5935 r
= apply_transaction(store
, &osr
, std::move(t
));
5938 g_ceph_context
->_conf
->set_val("bluestore_cache_size_hdd", "4000000");
5939 g_ceph_context
->_conf
->set_val("bluestore_cache_size_ssd", "4000000");
5940 g_conf
->set_val("bluestore_compression_mode", "none");
5941 g_conf
->set_val("bluestore_csum_type", "crc32c");
5945 TEST_P(StoreTestSpecificAUSize
, BlobReuseOnOverwrite
) {
5947 if (string(GetParam()) != "bluestore")
5950 size_t block_size
= 4096;
5951 StartDeferred(block_size
);
5952 g_conf
->set_val("bluestore_max_blob_size", "65536");
5953 g_conf
->apply_changes(NULL
);
5955 ObjectStore::Sequencer
osr("test");
5958 ghobject_t
hoid(hobject_t("test_hint", "", CEPH_NOSNAP
, 0, -1, ""));
5960 const PerfCounters
* logger
= store
->get_perf_counters();
5963 ObjectStore::Transaction t
;
5964 t
.create_collection(cid
, 0);
5965 r
= apply_transaction(store
, &osr
, std::move(t
));
5969 ObjectStore::Transaction t
;
5972 bl
.append(std::string(block_size
* 2, 'a'));
5973 t
.write(cid
, hoid
, 0, bl
.length(), bl
, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
5974 r
= apply_transaction(store
, &osr
, std::move(t
));
5978 // overwrite at the beginning
5979 ObjectStore::Transaction t
;
5982 bl
.append(std::string(block_size
, 'b'));
5983 t
.write(cid
, hoid
, 0, bl
.length(), bl
, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
5984 r
= apply_transaction(store
, &osr
, std::move(t
));
5989 ObjectStore::Transaction t
;
5992 bl
.append(std::string(block_size
* 2, 'c'));
5993 t
.write(cid
, hoid
, block_size
* 2, bl
.length(), bl
, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
5994 r
= apply_transaction(store
, &osr
, std::move(t
));
5998 // append with a gap
5999 ObjectStore::Transaction t
;
6002 bl
.append(std::string(block_size
* 2, 'd'));
6003 t
.write(cid
, hoid
, block_size
* 5, bl
.length(), bl
, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6004 r
= apply_transaction(store
, &osr
, std::move(t
));
6008 // We need to issue a read to trigger cache stat update that refresh
6009 // perf counters. additionally we need to wait some time for mempool
6010 // thread to update stats.
6012 bufferlist bl
, expected
;
6013 r
= store
->read(cid
, hoid
, 0, block_size
, bl
);
6014 ASSERT_EQ(r
, (int)block_size
);
6015 expected
.append(string(block_size
, 'b'));
6016 ASSERT_TRUE(bl_eq(expected
, bl
));
6017 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 1u);
6018 ASSERT_EQ(logger
->get(l_bluestore_extents
), 2u);
6022 ObjectStore::Transaction t
;
6025 bl
.append(std::string(block_size
* 2, 'e'));
6027 // Currently we are unable to reuse blob when overwriting in a single step
6028 t
.write(cid
, hoid
, block_size
* 6, bl
.length(), bl
, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6029 r
= apply_transaction(store
, &osr
, std::move(t
));
6033 // We need to issue a read to trigger cache stat update that refresh
6034 // perf counters. additionally we need to wait some time for mempool
6035 // thread to update stats.
6037 bufferlist bl
, expected
;
6038 r
= store
->read(cid
, hoid
, 0, block_size
, bl
);
6039 ASSERT_EQ(r
, (int)block_size
);
6040 expected
.append(string(block_size
, 'b'));
6041 ASSERT_TRUE(bl_eq(expected
, bl
));
6042 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 1u);
6043 ASSERT_EQ(logger
->get(l_bluestore_extents
), 2u);
6047 ObjectStore::Transaction t
;
6050 bl
.append(std::string(block_size
, 'f'));
6052 t
.write(cid
, hoid
, block_size
* 4, bl
.length(), bl
, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6053 r
= apply_transaction(store
, &osr
, std::move(t
));
6057 // we need to wait some time for mempool
6058 // thread to update stats to be able to check blob/extent numbers from
6062 bufferlist bl
, expected
;
6063 r
= store
->read(cid
, hoid
, 0, block_size
, bl
);
6064 ASSERT_EQ(r
, (int)block_size
);
6065 expected
.append(string(block_size
, 'b'));
6066 ASSERT_TRUE(bl_eq(expected
, bl
));
6070 r
= store
->read(cid
, hoid
, block_size
, block_size
, bl
);
6071 ASSERT_EQ(r
, (int)block_size
);
6072 expected
.append(string(block_size
, 'a'));
6073 ASSERT_TRUE(bl_eq(expected
, bl
));
6077 r
= store
->read(cid
, hoid
, block_size
* 2, block_size
* 2, bl
);
6078 ASSERT_EQ(r
, (int)block_size
* 2);
6079 expected
.append(string(block_size
* 2, 'c'));
6080 ASSERT_TRUE(bl_eq(expected
, bl
));
6084 r
= store
->read(cid
, hoid
, block_size
* 4, block_size
, bl
);
6085 ASSERT_EQ(r
, (int)block_size
);
6086 expected
.append(string(block_size
, 'f'));
6087 ASSERT_TRUE(bl_eq(expected
, bl
));
6091 r
= store
->read(cid
, hoid
, block_size
* 5, block_size
, bl
);
6092 ASSERT_EQ(r
, (int)block_size
);
6093 expected
.append(string(block_size
, 'd'));
6094 ASSERT_TRUE(bl_eq(expected
, bl
));
6098 r
= store
->read(cid
, hoid
, block_size
* 5, block_size
* 3, bl
);
6099 ASSERT_EQ(r
, (int)block_size
* 3);
6100 expected
.append(string(block_size
, 'd'));
6101 expected
.append(string(block_size
* 2, 'e'));
6102 ASSERT_TRUE(bl_eq(expected
, bl
));
6104 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 1u);
6105 ASSERT_EQ(logger
->get(l_bluestore_extents
), 1u);
6109 ObjectStore::Transaction t
;
6110 t
.remove(cid
, hoid
);
6111 t
.remove_collection(cid
);
6112 cerr
<< "Cleaning" << std::endl
;
6113 r
= apply_transaction(store
, &osr
, std::move(t
));
6116 g_conf
->set_val("bluestore_max_blob_size", "0");
6120 TEST_P(StoreTestSpecificAUSize
, BlobReuseOnOverwriteReverse
) {
6122 if (string(GetParam()) != "bluestore")
6125 size_t block_size
= 4096;
6126 StartDeferred(block_size
);
6127 g_conf
->set_val("bluestore_max_blob_size", "65536");
6129 g_conf
->apply_changes(NULL
);
6131 ObjectStore::Sequencer
osr("test");
6134 ghobject_t
hoid(hobject_t("test_hint", "", CEPH_NOSNAP
, 0, -1, ""));
6136 const PerfCounters
* logger
= store
->get_perf_counters();
6138 ObjectStore::Transaction t
;
6139 t
.create_collection(cid
, 0);
6140 r
= apply_transaction(store
, &osr
, std::move(t
));
6144 ObjectStore::Transaction t
;
6147 bl
.append(std::string(block_size
* 2, 'a'));
6148 t
.write(cid
, hoid
, block_size
* 10, bl
.length(), bl
,
6149 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6150 r
= apply_transaction(store
, &osr
, std::move(t
));
6155 ObjectStore::Transaction t
;
6158 bl
.append(std::string(block_size
, 'b'));
6159 t
.write(cid
, hoid
, block_size
* 9, bl
.length(), bl
,
6160 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6161 r
= apply_transaction(store
, &osr
, std::move(t
));
6165 // We need to issue a read to trigger cache stat update that refresh
6166 // perf counters. additionally we need to wait some time for mempool
6167 // thread to update stats.
6169 bufferlist bl
, expected
;
6170 r
= store
->read(cid
, hoid
, block_size
* 9, block_size
* 2, bl
);
6171 ASSERT_EQ(r
, (int)block_size
* 2);
6172 expected
.append(string(block_size
, 'b'));
6173 expected
.append(string(block_size
, 'a'));
6174 ASSERT_TRUE(bl_eq(expected
, bl
));
6175 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 1u);
6176 ASSERT_EQ(logger
->get(l_bluestore_extents
), 1u);
6181 // prepend existing with a gap
6182 ObjectStore::Transaction t
;
6185 bl
.append(std::string(block_size
, 'c'));
6186 t
.write(cid
, hoid
, block_size
* 7, bl
.length(), bl
,
6187 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6188 r
= apply_transaction(store
, &osr
, std::move(t
));
6192 // We need to issue a read to trigger cache stat update that refresh
6193 // perf counters. additionally we need to wait some time for mempool
6194 // thread to update stats.
6196 bufferlist bl
, expected
;
6197 r
= store
->read(cid
, hoid
, block_size
* 7, block_size
* 3, bl
);
6198 ASSERT_EQ(r
, (int)block_size
* 3);
6199 expected
.append(string(block_size
, 'c'));
6200 expected
.append(string(block_size
, 0));
6201 expected
.append(string(block_size
, 'b'));
6202 ASSERT_TRUE(bl_eq(expected
, bl
));
6203 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 1u);
6204 ASSERT_EQ(logger
->get(l_bluestore_extents
), 2u);
6208 // append after existing with a gap
6209 ObjectStore::Transaction t
;
6212 bl
.append(std::string(block_size
, 'd'));
6213 t
.write(cid
, hoid
, block_size
* 13, bl
.length(), bl
,
6214 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6215 r
= apply_transaction(store
, &osr
, std::move(t
));
6219 // We need to issue a read to trigger cache stat update that refresh
6220 // perf counters. additionally we need to wait some time for mempool
6221 // thread to update stats.
6223 bufferlist bl
, expected
;
6224 r
= store
->read(cid
, hoid
, block_size
* 11, block_size
* 3, bl
);
6225 ASSERT_EQ(r
, (int)block_size
* 3);
6226 expected
.append(string(block_size
, 'a'));
6227 expected
.append(string(block_size
, 0));
6228 expected
.append(string(block_size
, 'd'));
6229 ASSERT_TRUE(bl_eq(expected
, bl
));
6230 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 1u);
6231 ASSERT_EQ(logger
->get(l_bluestore_extents
), 3u);
6235 // append twice to the next max_blob slot
6236 ObjectStore::Transaction t
;
6239 bl
.append(std::string(block_size
, 'e'));
6240 t
.write(cid
, hoid
, block_size
* 17, bl
.length(), bl
,
6241 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6242 t
.write(cid
, hoid
, block_size
* 19, bl
.length(), bl
,
6243 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6244 r
= apply_transaction(store
, &osr
, std::move(t
));
6248 // We need to issue a read to trigger cache stat update that refresh
6249 // perf counters. additionally we need to wait some time for mempool
6250 // thread to update stats.
6252 bufferlist bl
, expected
;
6253 r
= store
->read(cid
, hoid
, block_size
* 17, block_size
* 3, bl
);
6254 ASSERT_EQ(r
, (int)block_size
* 3);
6255 expected
.append(string(block_size
, 'e'));
6256 expected
.append(string(block_size
, 0));
6257 expected
.append(string(block_size
, 'e'));
6258 ASSERT_TRUE(bl_eq(expected
, bl
));
6259 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 2u);
6260 ASSERT_EQ(logger
->get(l_bluestore_extents
), 5u);
6263 // fill gaps at the second slot
6264 ObjectStore::Transaction t
;
6267 bl
.append(std::string(block_size
, 'f'));
6268 t
.write(cid
, hoid
, block_size
* 16, bl
.length(), bl
,
6269 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6270 t
.write(cid
, hoid
, block_size
* 18, bl
.length(), bl
,
6271 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6272 r
= apply_transaction(store
, &osr
, std::move(t
));
6276 // We need to issue a read to trigger cache stat update that refresh
6277 // perf counters. additionally we need to wait some time for mempool
6278 // thread to update stats.
6280 bufferlist bl
, expected
;
6281 r
= store
->read(cid
, hoid
, block_size
* 16, block_size
* 4, bl
);
6282 ASSERT_EQ(r
, (int)block_size
* 4);
6283 expected
.append(string(block_size
, 'f'));
6284 expected
.append(string(block_size
, 'e'));
6285 expected
.append(string(block_size
, 'f'));
6286 expected
.append(string(block_size
, 'e'));
6287 ASSERT_TRUE(bl_eq(expected
, bl
));
6288 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 2u);
6289 ASSERT_EQ(logger
->get(l_bluestore_extents
), 4u);
6292 ObjectStore::Transaction t
;
6293 t
.remove(cid
, hoid
);
6294 t
.remove_collection(cid
);
6295 cerr
<< "Cleaning" << std::endl
;
6296 r
= apply_transaction(store
, &osr
, std::move(t
));
6299 g_conf
->set_val("bluestore_max_blob_size", "0");
6302 TEST_P(StoreTestSpecificAUSize
, BlobReuseOnSmallOverwrite
) {
6304 if (string(GetParam()) != "bluestore")
6307 size_t block_size
= 4096;
6308 StartDeferred(block_size
);
6309 g_conf
->set_val("bluestore_max_blob_size", "65536");
6310 g_conf
->apply_changes(NULL
);
6312 ObjectStore::Sequencer
osr("test");
6315 ghobject_t
hoid(hobject_t("test_hint", "", CEPH_NOSNAP
, 0, -1, ""));
6317 const PerfCounters
* logger
= store
->get_perf_counters();
6320 ObjectStore::Transaction t
;
6321 t
.create_collection(cid
, 0);
6322 r
= apply_transaction(store
, &osr
, std::move(t
));
6326 ObjectStore::Transaction t
;
6329 bl
.append(std::string(block_size
, 'a'));
6330 t
.write(cid
, hoid
, 0, bl
.length(), bl
, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6331 t
.write(cid
, hoid
, block_size
* 2, bl
.length(), bl
,
6332 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6333 r
= apply_transaction(store
, &osr
, std::move(t
));
6337 // write small into the gap
6338 ObjectStore::Transaction t
;
6341 bl
.append(std::string(3, 'b'));
6342 t
.write(cid
, hoid
, block_size
+ 1, bl
.length(), bl
,
6343 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6344 r
= apply_transaction(store
, &osr
, std::move(t
));
6348 // We need to issue a read to trigger cache stat update that refresh
6349 // perf counters. additionally we need to wait some time for mempool
6350 // thread to update stats.
6352 bufferlist bl
, expected
;
6353 r
= store
->read(cid
, hoid
, 0, block_size
* 3, bl
);
6354 ASSERT_EQ(r
, (int)block_size
* 3);
6355 expected
.append(string(block_size
, 'a'));
6356 expected
.append(string(1, 0));
6357 expected
.append(string(3, 'b'));
6358 expected
.append(string(block_size
- 4, 0));
6359 expected
.append(string(block_size
, 'a'));
6360 ASSERT_TRUE(bl_eq(expected
, bl
));
6362 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 1u);
6363 ASSERT_EQ(logger
->get(l_bluestore_extents
), 3u);
6366 ObjectStore::Transaction t
;
6367 t
.remove(cid
, hoid
);
6368 t
.remove_collection(cid
);
6369 cerr
<< "Cleaning" << std::endl
;
6370 r
= apply_transaction(store
, &osr
, std::move(t
));
6373 g_conf
->set_val("bluestore_max_blob_size", "0");
6376 // The test case to reproduce an issue when write happens
6377 // to a zero space between the extents sharing the same spanning blob
6378 // with unloaded shard map.
6379 // Second extent might be filled with zeros this way due to wrong result
6380 // returned by has_any_extents() call in do_write_small. The latter is caused
6381 // by incompletly loaded extent map.
6382 TEST_P(StoreTestSpecificAUSize
, SmallWriteOnShardedExtents
) {
6383 if (string(GetParam()) != "bluestore")
6386 size_t block_size
= 0x10000;
6387 StartDeferred(block_size
);
6389 g_conf
->set_val("bluestore_csum_type", "xxhash64");
6390 g_conf
->set_val("bluestore_max_target_blob", "524288"); // for sure
6392 g_conf
->apply_changes(NULL
);
6394 ObjectStore::Sequencer
osr("test");
6397 ghobject_t
hoid1(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
6400 ObjectStore::Transaction t
;
6401 t
.create_collection(cid
, 0);
6402 r
= apply_transaction(store
, &osr
, std::move(t
));
6406 //doing some tricks to have sharded extents/spanning objects
6407 ObjectStore::Transaction t
;
6410 bl
.append(std::string(0x80000, 'a'));
6411 t
.write(cid
, hoid1
, 0, bl
.length(), bl
, 0);
6412 t
.zero(cid
, hoid1
, 0x719e0, 0x75b0 );
6413 r
= apply_transaction(store
, &osr
, std::move(t
));
6416 bl2
.append(std::string(0x70000, 'b'));
6417 t
.write(cid
, hoid1
, 0, bl2
.length(), bl2
, 0);
6418 t
.zero(cid
, hoid1
, 0, 0x50000);
6419 r
= apply_transaction(store
, &osr
, std::move(t
));
6427 // do a write to zero space in between some extents sharing the same blob
6428 ObjectStore::Transaction t
;
6431 bl
.append(std::string(0x6520, 'c'));
6432 t
.write(cid
, hoid1
, 0x71c00, bl
.length(), bl
, 0);
6434 r
= apply_transaction(store
, &osr
, std::move(t
));
6439 ObjectStore::Transaction t
;
6440 bufferlist bl
, expected
;
6442 r
= store
->read(cid
, hoid1
, 0x70000, 0x9c00, bl
);
6443 ASSERT_EQ(r
, (int)0x9c00);
6444 expected
.append(string(0x19e0, 'a'));
6445 expected
.append(string(0x220, 0));
6446 expected
.append(string(0x6520, 'c'));
6447 expected
.append(string(0xe70, 0));
6448 expected
.append(string(0xc70, 'a'));
6449 ASSERT_TRUE(bl_eq(expected
, bl
));
6455 ObjectStore::Transaction t
;
6456 t
.remove(cid
, hoid1
);
6457 t
.remove_collection(cid
);
6458 cerr
<< "Cleaning" << std::endl
;
6459 r
= apply_transaction(store
, &osr
, std::move(t
));
6462 g_conf
->set_val("bluestore_max_target_blob", "524288");
6463 g_conf
->set_val("bluestore_csum_type", "crc32c");
6466 #endif //#if defined(HAVE_LIBAIO)
6468 TEST_P(StoreTest
, KVDBHistogramTest
) {
6469 if (string(GetParam()) != "bluestore")
6472 ObjectStore::Sequencer
osr("test");
6476 string
base("testobj.");
6478 bufferptr
ap(0x1000);
6479 memset(ap
.c_str(), 'a', 0x1000);
6482 ObjectStore::Transaction t
;
6483 t
.create_collection(cid
, 0);
6484 r
= apply_transaction(store
, &osr
, std::move(t
));
6487 for (int i
= 0; i
< NUM_OBJS
; ++i
) {
6488 ObjectStore::Transaction t
;
6490 snprintf(buf
, sizeof(buf
), "%d", i
);
6491 ghobject_t
hoid(hobject_t(sobject_t(base
+ string(buf
), CEPH_NOSNAP
)));
6492 t
.write(cid
, hoid
, 0, 0x1000, a
);
6493 r
= apply_transaction(store
, &osr
, std::move(t
));
6497 Formatter
*f
= Formatter::create("store_test", "json-pretty", "json-pretty");
6498 store
->generate_db_histogram(f
);
6503 TEST_P(StoreTest
, KVDBStatsTest
) {
6504 if (string(GetParam()) != "bluestore")
6507 g_conf
->set_val("rocksdb_perf", "true");
6508 g_conf
->set_val("rocksdb_collect_compaction_stats", "true");
6509 g_conf
->set_val("rocksdb_collect_extended_stats","true");
6510 g_conf
->set_val("rocksdb_collect_memory_stats","true");
6511 g_ceph_context
->_conf
->apply_changes(NULL
);
6512 int r
= store
->umount();
6514 r
= store
->mount(); //to force rocksdb stats
6517 ObjectStore::Sequencer
osr("test");
6520 string
base("testobj.");
6522 bufferptr
ap(0x1000);
6523 memset(ap
.c_str(), 'a', 0x1000);
6526 ObjectStore::Transaction t
;
6527 t
.create_collection(cid
, 0);
6528 r
= apply_transaction(store
, &osr
, std::move(t
));
6531 for (int i
= 0; i
< NUM_OBJS
; ++i
) {
6532 ObjectStore::Transaction t
;
6534 snprintf(buf
, sizeof(buf
), "%d", i
);
6535 ghobject_t
hoid(hobject_t(sobject_t(base
+ string(buf
), CEPH_NOSNAP
)));
6536 t
.write(cid
, hoid
, 0, 0x1000, a
);
6537 r
= apply_transaction(store
, &osr
, std::move(t
));
6541 Formatter
*f
= Formatter::create("store_test", "json-pretty", "json-pretty");
6542 store
->get_db_statistics(f
);
6545 g_conf
->set_val("rocksdb_perf", "false");
6546 g_conf
->set_val("rocksdb_collect_compaction_stats", "false");
6547 g_conf
->set_val("rocksdb_collect_extended_stats","false");
6548 g_conf
->set_val("rocksdb_collect_memory_stats","false");
6551 #if defined(HAVE_LIBAIO)
6552 TEST_P(StoreTestSpecificAUSize
, garbageCollection
) {
6553 ObjectStore::Sequencer
osr("test");
6556 int buf_len
= 256 * 1024;
6557 int overlap_offset
= 64 * 1024;
6558 int write_offset
= buf_len
;
6559 if (string(GetParam()) != "bluestore")
6562 #define WRITE_AT(offset, _length) {\
6563 ObjectStore::Transaction t;\
6564 if ((uint64_t)_length != bl.length()) { \
6565 buffer::ptr p(bl.c_str(), _length);\
6567 bl_tmp.push_back(p);\
6568 t.write(cid, hoid, offset, bl_tmp.length(), bl_tmp);\
6570 t.write(cid, hoid, offset, bl.length(), bl);\
6572 r = apply_transaction(store, &osr, std::move(t));\
6576 StartDeferred(65536);
6578 g_conf
->set_val("bluestore_compression_max_blob_size", "524288");
6579 g_conf
->set_val("bluestore_compression_min_blob_size", "262144");
6580 g_conf
->set_val("bluestore_compression_mode", "force");
6581 g_conf
->apply_changes(NULL
);
6583 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
6586 r
= store
->read(cid
, hoid
, 0, 5, in
);
6587 ASSERT_EQ(-ENOENT
, r
);
6590 ObjectStore::Transaction t
;
6591 t
.create_collection(cid
, 0);
6592 cerr
<< "Creating collection " << cid
<< std::endl
;
6593 r
= apply_transaction(store
, &osr
, std::move(t
));
6598 data
.resize(buf_len
);
6602 bool exists
= store
->exists(cid
, hoid
);
6603 ASSERT_TRUE(!exists
);
6605 ObjectStore::Transaction t
;
6607 cerr
<< "Creating object " << hoid
<< std::endl
;
6608 r
= apply_transaction(store
, &osr
, std::move(t
));
6611 exists
= store
->exists(cid
, hoid
);
6612 ASSERT_EQ(true, exists
);
6616 for(size_t i
= 0; i
< data
.size(); i
++)
6622 struct store_statfs_t statfs
;
6623 WRITE_AT(0, buf_len
);
6624 int r
= store
->statfs(&statfs
);
6626 ASSERT_EQ(statfs
.compressed_allocated
, 0x10000);
6629 struct store_statfs_t statfs
;
6630 WRITE_AT(write_offset
- 2 * overlap_offset
, buf_len
);
6631 int r
= store
->statfs(&statfs
);
6633 ASSERT_EQ(statfs
.compressed_allocated
, 0x20000);
6634 const PerfCounters
* counters
= store
->get_perf_counters();
6635 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0u);
6639 struct store_statfs_t statfs
;
6640 WRITE_AT(write_offset
- overlap_offset
, buf_len
);
6641 int r
= store
->statfs(&statfs
);
6643 ASSERT_EQ(statfs
.compressed_allocated
, 0x20000);
6644 const PerfCounters
* counters
= store
->get_perf_counters();
6645 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0x10000u
);
6648 struct store_statfs_t statfs
;
6649 WRITE_AT(write_offset
- 3 * overlap_offset
, buf_len
);
6650 int r
= store
->statfs(&statfs
);
6652 ASSERT_EQ(statfs
.compressed_allocated
, 0x20000);
6653 const PerfCounters
* counters
= store
->get_perf_counters();
6654 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0x20000u
);
6657 struct store_statfs_t statfs
;
6658 WRITE_AT(write_offset
+ 1, overlap_offset
-1);
6659 int r
= store
->statfs(&statfs
);
6661 ASSERT_EQ(statfs
.compressed_allocated
, 0x20000);
6662 const PerfCounters
* counters
= store
->get_perf_counters();
6663 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0x20000u
);
6666 struct store_statfs_t statfs
;
6667 WRITE_AT(write_offset
+ 1, overlap_offset
);
6668 int r
= store
->statfs(&statfs
);
6670 ASSERT_EQ(statfs
.compressed_allocated
, 0x10000);
6671 const PerfCounters
* counters
= store
->get_perf_counters();
6672 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0x3ffffu
);
6675 struct store_statfs_t statfs
;
6676 WRITE_AT(0, buf_len
-1);
6677 int r
= store
->statfs(&statfs
);
6679 ASSERT_EQ(statfs
.compressed_allocated
, 0x10000);
6680 const PerfCounters
* counters
= store
->get_perf_counters();
6681 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0x40001u
);
6683 g_conf
->set_val("bluestore_gc_enable_total_threshold", "1"); //forbid GC when saving = 0
6685 struct store_statfs_t statfs
;
6686 WRITE_AT(1, overlap_offset
-2);
6687 WRITE_AT(overlap_offset
* 2 + 1, overlap_offset
-2);
6688 int r
= store
->statfs(&statfs
);
6690 ASSERT_EQ(statfs
.compressed_allocated
, 0x10000);
6691 const PerfCounters
* counters
= store
->get_perf_counters();
6692 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0x40001u
);
6695 struct store_statfs_t statfs
;
6696 WRITE_AT(overlap_offset
+ 1, overlap_offset
-2);
6697 int r
= store
->statfs(&statfs
);
6699 ASSERT_EQ(statfs
.compressed_allocated
, 0x0);
6700 const PerfCounters
* counters
= store
->get_perf_counters();
6701 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0x40007u
);
6704 ObjectStore::Transaction t
;
6705 t
.remove(cid
, hoid
);
6706 cerr
<< "Cleaning" << std::endl
;
6707 r
= apply_transaction(store
, &osr
, std::move(t
));
6711 g_conf
->set_val("bluestore_gc_enable_total_threshold", "0");
6712 g_conf
->set_val("bluestore_compression_min_blob_size", "0");
6713 g_conf
->set_val("bluestore_compression_max_blob_size", "0");
6714 g_conf
->set_val("bluestore_compression_mode", "none");
6715 g_conf
->apply_changes(NULL
);
6719 TEST_P(StoreTestSpecificAUSize
, fsckOnUnalignedDevice
) {
6720 if (string(GetParam()) != "bluestore")
6723 g_conf
->set_val("bluestore_block_size", stringify(0x280005000)); //10 Gb + 4K
6724 g_conf
->set_val("bluestore_fsck_on_mount", "false");
6725 g_conf
->set_val("bluestore_fsck_on_umount", "false");
6726 StartDeferred(0x4000);
6728 ASSERT_EQ(store
->fsck(false), 0); // do fsck explicitly
6731 g_conf
->set_val("bluestore_fsck_on_mount", "true");
6732 g_conf
->set_val("bluestore_fsck_on_umount", "true");
6733 g_conf
->set_val("bluestore_block_size", stringify(0x280000000)); // 10 Gb
6734 g_conf
->apply_changes(NULL
);
6737 TEST_P(StoreTestSpecificAUSize
, fsckOnUnalignedDevice2
) {
6738 if (string(GetParam()) != "bluestore")
6741 g_conf
->set_val("bluestore_block_size", stringify(0x280005000)); //10 Gb + 20K
6742 g_conf
->set_val("bluestore_fsck_on_mount", "false");
6743 g_conf
->set_val("bluestore_fsck_on_umount", "false");
6744 StartDeferred(0x1000);
6746 ASSERT_EQ(store
->fsck(false), 0); // do fsck explicitly
6749 g_conf
->set_val("bluestore_block_size", stringify(0x280000000)); // 10 Gb
6750 g_conf
->set_val("bluestore_fsck_on_mount", "true");
6751 g_conf
->set_val("bluestore_fsck_on_umount", "true");
6752 g_conf
->apply_changes(NULL
);
6755 int main(int argc
, char **argv
) {
6756 vector
<const char*> args
;
6757 argv_to_vec(argc
, (const char **)argv
, args
);
6760 auto cct
= global_init(NULL
, args
, CEPH_ENTITY_TYPE_CLIENT
,
6761 CODE_ENVIRONMENT_UTILITY
, 0);
6762 common_init_finish(g_ceph_context
);
6764 g_ceph_context
->_conf
->set_val("osd_journal_size", "400");
6765 g_ceph_context
->_conf
->set_val("filestore_index_retry_probability", "0.5");
6766 g_ceph_context
->_conf
->set_val("filestore_op_thread_timeout", "1000");
6767 g_ceph_context
->_conf
->set_val("filestore_op_thread_suicide_timeout", "10000");
6768 //g_ceph_context->_conf->set_val("filestore_fiemap", "true");
6769 g_ceph_context
->_conf
->set_val("bluestore_fsck_on_mount", "true");
6770 g_ceph_context
->_conf
->set_val("bluestore_fsck_on_umount", "true");
6771 g_ceph_context
->_conf
->set_val("bluestore_debug_misc", "true");
6772 g_ceph_context
->_conf
->set_val("bluestore_debug_small_allocations", "4");
6773 g_ceph_context
->_conf
->set_val("bluestore_debug_freelist", "true");
6774 g_ceph_context
->_conf
->set_val("bluestore_clone_cow", "true");
6775 g_ceph_context
->_conf
->set_val("bluestore_max_alloc_size", "196608");
6777 // set small cache sizes so we see trimming during Synthetic tests
6778 g_ceph_context
->_conf
->set_val("bluestore_cache_size_hdd", "4000000");
6779 g_ceph_context
->_conf
->set_val("bluestore_cache_size_ssd", "4000000");
6781 // very short *_max prealloc so that we fall back to async submits
6782 g_ceph_context
->_conf
->set_val("bluestore_blobid_prealloc", "10");
6783 g_ceph_context
->_conf
->set_val("bluestore_nid_prealloc", "10");
6784 g_ceph_context
->_conf
->set_val("bluestore_debug_randomize_serial_transaction",
6787 g_ceph_context
->_conf
->set_val("bdev_debug_aio", "true");
6789 // specify device size
6790 g_ceph_context
->_conf
->set_val("bluestore_block_size", "10240000000");
6792 g_ceph_context
->_conf
->set_val(
6793 "enable_experimental_unrecoverable_data_corrupting_features", "*");
6794 g_ceph_context
->_conf
->apply_changes(NULL
);
6796 ::testing::InitGoogleTest(&argc
, argv
);
6797 return RUN_ALL_TESTS();
6802 * compile-command: "cd ../.. ; make ceph_test_objectstore &&
6803 * ./ceph_test_objectstore \
6804 * --gtest_filter=*.collect_metadata* --log-to-stderr=true --debug-filestore=20