1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
4 * Ceph - scalable distributed file system
6 * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
8 * This is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License version 2.1, as published by the Free Software
11 * Foundation. See file COPYING.
20 #include <sys/mount.h>
21 #include <boost/scoped_ptr.hpp>
22 #include <boost/random/mersenne_twister.hpp>
23 #include <boost/random/uniform_int.hpp>
24 #include <boost/random/binomial_distribution.hpp>
25 #include <gtest/gtest.h>
27 #include "os/ObjectStore.h"
28 #include "os/filestore/FileStore.h"
29 #if defined(HAVE_LIBAIO)
30 #include "os/bluestore/BlueStore.h"
32 #include "include/Context.h"
33 #include "common/ceph_argparse.h"
34 #include "global/global_init.h"
35 #include "common/Mutex.h"
36 #include "common/Cond.h"
37 #include "common/errno.h"
38 #include "include/stringify.h"
39 #include "include/coredumpctl.h"
41 #include "include/unordered_map.h"
42 #include "store_test_fixture.h"
45 typedef boost::mt11213b gen_type
;
47 const uint64_t DEF_STORE_TEST_BLOCKDEV_SIZE
= 10240000000;
48 #define dout_context g_ceph_context
50 #if GTEST_HAS_PARAM_TEST
52 static bool bl_eq(bufferlist
& expected
, bufferlist
& actual
)
54 if (expected
.contents_equal(actual
))
58 if(expected
.length() != actual
.length()) {
59 cout
<< "--- buffer lengths mismatch " << std::hex
60 << "expected 0x" << expected
.length() << " != actual 0x"
61 << actual
.length() << std::dec
<< std::endl
;
62 derr
<< "--- buffer lengths mismatch " << std::hex
63 << "expected 0x" << expected
.length() << " != actual 0x"
64 << actual
.length() << std::dec
<< dendl
;
66 auto len
= MIN(expected
.length(), actual
.length());
67 while ( first
<len
&& expected
[first
] == actual
[first
])
70 while (last
> 0 && expected
[last
-1] == actual
[last
-1])
73 cout
<< "--- buffer mismatch between offset 0x" << std::hex
<< first
74 << " and 0x" << last
<< ", total 0x" << len
<< std::dec
76 derr
<< "--- buffer mismatch between offset 0x" << std::hex
<< first
77 << " and 0x" << last
<< ", total 0x" << len
<< std::dec
79 cout
<< "--- expected:\n";
80 expected
.hexdump(cout
);
81 cout
<< "--- actual:\n";
90 int apply_transaction(
92 ObjectStore::Sequencer
*osr
,
93 ObjectStore::Transaction
&&t
) {
95 ObjectStore::Transaction t2
;
97 return store
->apply_transaction(osr
, std::move(t2
));
99 return store
->apply_transaction(osr
, std::move(t
));
104 bool sorted(const vector
<ghobject_t
> &in
) {
106 for (vector
<ghobject_t
>::const_iterator i
= in
.begin();
110 cout
<< start
<< " should follow " << *i
<< std::endl
;
118 class StoreTest
: public StoreTestFixture
,
119 public ::testing::WithParamInterface
<const char*> {
122 : StoreTestFixture(GetParam())
126 class StoreTestDeferredSetup
: public StoreTest
{
127 void SetUp() override
{
132 void DeferredSetup() {
139 class StoreTestSpecificAUSize
: public StoreTestDeferredSetup
{
144 boost::scoped_ptr
<ObjectStore
>& store
,
148 uint64_t align
)> MatrixTest
;
150 void StartDeferred(size_t min_alloc_size
) {
151 g_conf
->set_val("bluestore_min_alloc_size", stringify(min_alloc_size
));
155 void TearDown() override
{
156 g_conf
->set_val("bluestore_min_alloc_size", "0");
157 StoreTestDeferredSetup::TearDown();
161 // bluestore matrix testing
162 uint64_t max_write
= 40 * 1024;
163 uint64_t max_size
= 400 * 1024;
164 uint64_t alignment
= 0;
165 uint64_t num_ops
= 10000;
168 string
matrix_get(const char *k
) {
169 if (string(k
) == "max_write") {
170 return stringify(max_write
);
171 } else if (string(k
) == "max_size") {
172 return stringify(max_size
);
173 } else if (string(k
) == "alignment") {
174 return stringify(alignment
);
175 } else if (string(k
) == "num_ops") {
176 return stringify(num_ops
);
179 g_conf
->get_val(k
, &buf
, -1);
186 void matrix_set(const char *k
, const char *v
) {
187 if (string(k
) == "max_write") {
188 max_write
= atoll(v
);
189 } else if (string(k
) == "max_size") {
191 } else if (string(k
) == "alignment") {
192 alignment
= atoll(v
);
193 } else if (string(k
) == "num_ops") {
196 g_conf
->set_val(k
, v
);
200 void do_matrix_choose(const char *matrix
[][10],
201 int i
, int pos
, int num
,
202 boost::scoped_ptr
<ObjectStore
>& store
,
206 for (count
= 0; matrix
[i
][count
+1]; ++count
) ;
207 for (int j
= 1; matrix
[i
][j
]; ++j
) {
208 matrix_set(matrix
[i
][0], matrix
[i
][j
]);
209 do_matrix_choose(matrix
,
217 cout
<< "---------------------- " << (pos
+ 1) << " / " << num
218 << " ----------------------" << std::endl
;
219 for (unsigned k
=0; matrix
[k
][0]; ++k
) {
220 cout
<< " " << matrix
[k
][0] << " = " << matrix_get(matrix
[k
][0])
223 g_ceph_context
->_conf
->apply_changes(NULL
);
224 fn(store
, num_ops
, max_size
, max_write
, alignment
);
228 void do_matrix(const char *matrix
[][10],
229 boost::scoped_ptr
<ObjectStore
>& store
,
231 map
<string
,string
> old
;
232 for (unsigned i
=0; matrix
[i
][0]; ++i
) {
233 old
[matrix
[i
][0]] = matrix_get(matrix
[i
][0]);
235 cout
<< "saved config options " << old
<< std::endl
;
237 if (strcmp(matrix
[0][0], "bluestore_min_alloc_size") == 0) {
239 for (count
= 0; matrix
[0][count
+1]; ++count
) ;
240 for (size_t j
= 1; matrix
[0][j
]; ++j
) {
244 StartDeferred(strtoll(matrix
[0][j
], NULL
, 10));
245 do_matrix_choose(matrix
, 1, j
- 1, count
, store
, fn
);
249 do_matrix_choose(matrix
, 0, 0, 1, store
, fn
);
252 cout
<< "restoring config options " << old
<< std::endl
;
254 cout
<< " " << p
.first
<< " = " << p
.second
<< std::endl
;
255 matrix_set(p
.first
.c_str(), p
.second
.c_str());
257 g_ceph_context
->_conf
->apply_changes(NULL
);
262 TEST_P(StoreTest
, collect_metadata
) {
263 map
<string
,string
> pm
;
264 store
->collect_metadata(&pm
);
265 if (GetParam() == string("filestore")) {
266 ASSERT_NE(pm
.count("filestore_backend"), 0u);
267 ASSERT_NE(pm
.count("filestore_f_type"), 0u);
268 ASSERT_NE(pm
.count("backend_filestore_partition_path"), 0u);
269 ASSERT_NE(pm
.count("backend_filestore_dev_node"), 0u);
273 TEST_P(StoreTest
, Trivial
) {
276 TEST_P(StoreTest
, TrivialRemount
) {
277 int r
= store
->umount();
283 TEST_P(StoreTest
, SimpleRemount
) {
284 ObjectStore::Sequencer
osr("test");
286 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
287 ghobject_t
hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP
)));
289 bl
.append("1234512345");
292 cerr
<< "create collection + write" << std::endl
;
293 ObjectStore::Transaction t
;
294 t
.create_collection(cid
, 0);
295 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
296 r
= apply_transaction(store
, &osr
, std::move(t
));
304 ObjectStore::Transaction t
;
305 t
.write(cid
, hoid2
, 0, bl
.length(), bl
);
306 r
= apply_transaction(store
, &osr
, std::move(t
));
310 ObjectStore::Transaction t
;
312 t
.remove(cid
, hoid2
);
313 t
.remove_collection(cid
);
314 cerr
<< "remove collection" << std::endl
;
315 r
= apply_transaction(store
, &osr
, std::move(t
));
323 ObjectStore::Transaction t
;
324 t
.create_collection(cid
, 0);
325 r
= apply_transaction(store
, &osr
, std::move(t
));
327 bool exists
= store
->exists(cid
, hoid
);
328 ASSERT_TRUE(!exists
);
331 ObjectStore::Transaction t
;
332 t
.remove_collection(cid
);
333 cerr
<< "remove collection" << std::endl
;
334 r
= apply_transaction(store
, &osr
, std::move(t
));
339 TEST_P(StoreTest
, IORemount
) {
340 ObjectStore::Sequencer
osr("test");
343 bl
.append("1234512345");
346 cerr
<< "create collection + objects" << std::endl
;
347 ObjectStore::Transaction t
;
348 t
.create_collection(cid
, 0);
349 for (int n
=1; n
<=100; ++n
) {
350 ghobject_t
hoid(hobject_t(sobject_t("Object " + stringify(n
), CEPH_NOSNAP
)));
351 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
353 r
= apply_transaction(store
, &osr
, std::move(t
));
358 cout
<< "overwrites" << std::endl
;
359 for (int n
=1; n
<=100; ++n
) {
360 ObjectStore::Transaction t
;
361 ghobject_t
hoid(hobject_t(sobject_t("Object " + stringify(n
), CEPH_NOSNAP
)));
362 t
.write(cid
, hoid
, 1, bl
.length(), bl
);
363 r
= apply_transaction(store
, &osr
, std::move(t
));
372 ObjectStore::Transaction t
;
373 for (int n
=1; n
<=100; ++n
) {
374 ghobject_t
hoid(hobject_t(sobject_t("Object " + stringify(n
), CEPH_NOSNAP
)));
377 t
.remove_collection(cid
);
378 r
= apply_transaction(store
, &osr
, std::move(t
));
383 TEST_P(StoreTest
, UnprintableCharsName
) {
384 ObjectStore::Sequencer
osr("test");
386 string name
= "funnychars_";
387 for (unsigned i
= 0; i
< 256; ++i
) {
390 ghobject_t
oid(hobject_t(sobject_t(name
, CEPH_NOSNAP
)));
393 cerr
<< "create collection + object" << std::endl
;
394 ObjectStore::Transaction t
;
395 t
.create_collection(cid
, 0);
397 r
= apply_transaction(store
, &osr
, std::move(t
));
405 cout
<< "removing" << std::endl
;
406 ObjectStore::Transaction t
;
408 t
.remove_collection(cid
);
409 r
= apply_transaction(store
, &osr
, std::move(t
));
414 TEST_P(StoreTest
, FiemapEmpty
) {
415 ObjectStore::Sequencer
osr("test");
418 ghobject_t
oid(hobject_t(sobject_t("fiemap_object", CEPH_NOSNAP
)));
420 ObjectStore::Transaction t
;
421 t
.create_collection(cid
, 0);
423 t
.truncate(cid
, oid
, 100000);
424 r
= apply_transaction(store
, &osr
, std::move(t
));
429 store
->fiemap(cid
, oid
, 0, 100000, bl
);
430 map
<uint64_t,uint64_t> m
, e
;
431 bufferlist::iterator p
= bl
.begin();
433 cout
<< " got " << m
<< std::endl
;
435 EXPECT_TRUE(m
== e
|| m
.empty());
438 ObjectStore::Transaction t
;
440 t
.remove_collection(cid
);
441 cerr
<< "remove collection" << std::endl
;
442 r
= apply_transaction(store
, &osr
, std::move(t
));
447 TEST_P(StoreTest
, FiemapHoles
) {
448 ObjectStore::Sequencer
osr("test");
449 const uint64_t MAX_EXTENTS
= 4000;
450 const uint64_t SKIP_STEP
= 65536;
453 ghobject_t
oid(hobject_t(sobject_t("fiemap_object", CEPH_NOSNAP
)));
457 ObjectStore::Transaction t
;
458 t
.create_collection(cid
, 0);
460 for (uint64_t i
= 0; i
< MAX_EXTENTS
; i
++)
461 t
.write(cid
, oid
, SKIP_STEP
* i
, 3, bl
);
462 r
= apply_transaction(store
, &osr
, std::move(t
));
466 //fiemap test from 0 to SKIP_STEP * (MAX_EXTENTS - 1) + 3
468 store
->fiemap(cid
, oid
, 0, SKIP_STEP
* (MAX_EXTENTS
- 1) + 3, bl
);
469 map
<uint64_t,uint64_t> m
, e
;
470 bufferlist::iterator p
= bl
.begin();
472 cout
<< " got " << m
<< std::endl
;
473 ASSERT_TRUE(!m
.empty());
475 bool extents_exist
= true;
476 if (m
.size() == MAX_EXTENTS
) {
477 for (uint64_t i
= 0; i
< MAX_EXTENTS
; i
++)
478 extents_exist
= extents_exist
&& m
.count(SKIP_STEP
*i
);
480 ASSERT_TRUE((m
.size() == 1 &&
481 m
[0] > SKIP_STEP
* (MAX_EXTENTS
- 1)) ||
482 (m
.size() == MAX_EXTENTS
&& extents_exist
));
484 // fiemap test from SKIP_STEP to SKIP_STEP * (MAX_EXTENTS - 2) + 3
485 // reset bufferlist and map
489 store
->fiemap(cid
, oid
, SKIP_STEP
, SKIP_STEP
* (MAX_EXTENTS
- 2) + 3, bl
);
492 cout
<< " got " << m
<< std::endl
;
493 ASSERT_TRUE(!m
.empty());
494 ASSERT_GE(m
[SKIP_STEP
], 3u);
495 extents_exist
= true;
496 if (m
.size() == (MAX_EXTENTS
- 2)) {
497 for (uint64_t i
= 1; i
< MAX_EXTENTS
- 1; i
++)
498 extents_exist
= extents_exist
&& m
.count(SKIP_STEP
*i
);
500 ASSERT_TRUE((m
.size() == 1 &&
501 m
[SKIP_STEP
] > SKIP_STEP
* (MAX_EXTENTS
- 2)) ||
502 (m
.size() == (MAX_EXTENTS
- 1) && extents_exist
));
505 ObjectStore::Transaction t
;
507 t
.remove_collection(cid
);
508 cerr
<< "remove collection" << std::endl
;
509 r
= apply_transaction(store
, &osr
, std::move(t
));
514 TEST_P(StoreTest
, SimpleMetaColTest
) {
515 ObjectStore::Sequencer
osr("test");
519 ObjectStore::Transaction t
;
520 t
.create_collection(cid
, 0);
521 cerr
<< "create collection" << std::endl
;
522 r
= apply_transaction(store
, &osr
, std::move(t
));
526 ObjectStore::Transaction t
;
527 t
.remove_collection(cid
);
528 cerr
<< "remove collection" << std::endl
;
529 r
= apply_transaction(store
, &osr
, std::move(t
));
533 ObjectStore::Transaction t
;
534 t
.create_collection(cid
, 0);
535 cerr
<< "add collection" << std::endl
;
536 r
= apply_transaction(store
, &osr
, std::move(t
));
540 ObjectStore::Transaction t
;
541 t
.remove_collection(cid
);
542 cerr
<< "remove collection" << std::endl
;
543 r
= apply_transaction(store
, &osr
, std::move(t
));
548 TEST_P(StoreTest
, SimplePGColTest
) {
549 ObjectStore::Sequencer
osr("test");
550 coll_t
cid(spg_t(pg_t(1,2), shard_id_t::NO_SHARD
));
553 ObjectStore::Transaction t
;
554 t
.create_collection(cid
, 4);
555 cerr
<< "create collection" << std::endl
;
556 r
= apply_transaction(store
, &osr
, std::move(t
));
560 ObjectStore::Transaction t
;
561 t
.remove_collection(cid
);
562 cerr
<< "remove collection" << std::endl
;
563 r
= apply_transaction(store
, &osr
, std::move(t
));
567 ObjectStore::Transaction t
;
568 t
.create_collection(cid
, 4);
569 cerr
<< "add collection" << std::endl
;
570 r
= apply_transaction(store
, &osr
, std::move(t
));
574 ObjectStore::Transaction t
;
575 t
.remove_collection(cid
);
576 cerr
<< "remove collection" << std::endl
;
577 r
= apply_transaction(store
, &osr
, std::move(t
));
582 TEST_P(StoreTest
, SimpleColPreHashTest
) {
583 ObjectStore::Sequencer
osr("test");
584 // Firstly we will need to revert the value making sure
585 // collection hint actually works
586 int merge_threshold
= g_ceph_context
->_conf
->filestore_merge_threshold
;
587 std::ostringstream oss
;
588 if (merge_threshold
> 0) {
589 oss
<< "-" << merge_threshold
;
590 g_ceph_context
->_conf
->set_val("filestore_merge_threshold", oss
.str().c_str());
593 uint32_t pg_num
= 128;
595 boost::uniform_int
<> pg_id_range(0, pg_num
);
596 gen_type
rng(time(NULL
));
597 int pg_id
= pg_id_range(rng
);
599 int objs_per_folder
= abs(merge_threshold
) * 16 * g_ceph_context
->_conf
->filestore_split_multiple
;
600 boost::uniform_int
<> folders_range(5, 256);
601 uint64_t expected_num_objs
= (uint64_t)objs_per_folder
* (uint64_t)folders_range(rng
);
603 coll_t
cid(spg_t(pg_t(pg_id
, 15), shard_id_t::NO_SHARD
));
606 // Create a collection along with a hint
607 ObjectStore::Transaction t
;
608 t
.create_collection(cid
, 5);
609 cerr
<< "create collection" << std::endl
;
611 ::encode(pg_num
, hint
);
612 ::encode(expected_num_objs
, hint
);
613 t
.collection_hint(cid
, ObjectStore::Transaction::COLL_HINT_EXPECTED_NUM_OBJECTS
, hint
);
614 cerr
<< "collection hint" << std::endl
;
615 r
= apply_transaction(store
, &osr
, std::move(t
));
619 // Remove the collection
620 ObjectStore::Transaction t
;
621 t
.remove_collection(cid
);
622 cerr
<< "remove collection" << std::endl
;
623 r
= apply_transaction(store
, &osr
, std::move(t
));
626 // Revert the config change so that it does not affect the split/merge tests
627 if (merge_threshold
> 0) {
629 oss
<< merge_threshold
;
630 g_ceph_context
->_conf
->set_val("filestore_merge_threshold", oss
.str().c_str());
634 TEST_P(StoreTest
, SmallBlockWrites
) {
635 ObjectStore::Sequencer
osr("test");
638 ghobject_t
hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP
)));
640 ObjectStore::Transaction t
;
641 t
.create_collection(cid
, 0);
642 cerr
<< "Creating collection " << cid
<< std::endl
;
643 r
= apply_transaction(store
, &osr
, std::move(t
));
647 bufferptr
ap(0x1000);
648 memset(ap
.c_str(), 'a', 0x1000);
651 bufferptr
bp(0x1000);
652 memset(bp
.c_str(), 'b', 0x1000);
655 bufferptr
cp(0x1000);
656 memset(cp
.c_str(), 'c', 0x1000);
658 bufferptr
zp(0x1000);
663 ObjectStore::Transaction t
;
664 t
.write(cid
, hoid
, 0, 0x1000, a
);
665 r
= apply_transaction(store
, &osr
, std::move(t
));
669 r
= store
->read(cid
, hoid
, 0, 0x4000, in
);
670 ASSERT_EQ(0x1000, r
);
672 ASSERT_TRUE(bl_eq(exp
, in
));
675 ObjectStore::Transaction t
;
676 t
.write(cid
, hoid
, 0x1000, 0x1000, b
);
677 r
= apply_transaction(store
, &osr
, std::move(t
));
681 r
= store
->read(cid
, hoid
, 0, 0x4000, in
);
682 ASSERT_EQ(0x2000, r
);
685 ASSERT_TRUE(bl_eq(exp
, in
));
688 ObjectStore::Transaction t
;
689 t
.write(cid
, hoid
, 0x3000, 0x1000, c
);
690 r
= apply_transaction(store
, &osr
, std::move(t
));
694 r
= store
->read(cid
, hoid
, 0, 0x4000, in
);
695 ASSERT_EQ(0x4000, r
);
700 ASSERT_TRUE(bl_eq(exp
, in
));
703 ObjectStore::Transaction t
;
704 t
.write(cid
, hoid
, 0x2000, 0x1000, a
);
705 r
= apply_transaction(store
, &osr
, std::move(t
));
709 r
= store
->read(cid
, hoid
, 0, 0x4000, in
);
710 ASSERT_EQ(0x4000, r
);
715 ASSERT_TRUE(bl_eq(exp
, in
));
718 ObjectStore::Transaction t
;
719 t
.write(cid
, hoid
, 0, 0x1000, c
);
720 r
= apply_transaction(store
, &osr
, std::move(t
));
725 r
= store
->read(cid
, hoid
, 0, 0x4000, in
);
726 ASSERT_EQ(0x4000, r
);
731 ASSERT_TRUE(bl_eq(exp
, in
));
734 ObjectStore::Transaction t
;
736 t
.remove_collection(cid
);
737 cerr
<< "Cleaning" << std::endl
;
738 r
= apply_transaction(store
, &osr
, std::move(t
));
743 TEST_P(StoreTest
, BufferCacheReadTest
) {
744 ObjectStore::Sequencer
osr("test");
747 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
750 r
= store
->read(cid
, hoid
, 0, 5, in
);
751 ASSERT_EQ(-ENOENT
, r
);
754 ObjectStore::Transaction t
;
755 t
.create_collection(cid
, 0);
756 cerr
<< "Creating collection " << cid
<< std::endl
;
757 r
= apply_transaction(store
, &osr
, std::move(t
));
761 bool exists
= store
->exists(cid
, hoid
);
762 ASSERT_TRUE(!exists
);
764 ObjectStore::Transaction t
;
766 cerr
<< "Creating object " << hoid
<< std::endl
;
767 r
= apply_transaction(store
, &osr
, std::move(t
));
770 exists
= store
->exists(cid
, hoid
);
771 ASSERT_EQ(true, exists
);
774 ObjectStore::Transaction t
;
775 bufferlist bl
, newdata
;
777 t
.write(cid
, hoid
, 0, 5, bl
);
778 t
.write(cid
, hoid
, 10, 5, bl
);
779 cerr
<< "TwinWrite" << std::endl
;
780 r
= apply_transaction(store
, &osr
, std::move(t
));
783 r
= store
->read(cid
, hoid
, 0, 15, newdata
);
788 expected
.append_zero(5);
790 ASSERT_TRUE(bl_eq(expected
, newdata
));
793 //overwrite over the same extents
795 ObjectStore::Transaction t
;
796 bufferlist bl
, newdata
;
798 t
.write(cid
, hoid
, 0, 5, bl
);
799 t
.write(cid
, hoid
, 10, 5, bl
);
800 cerr
<< "TwinWrite" << std::endl
;
801 r
= apply_transaction(store
, &osr
, std::move(t
));
804 r
= store
->read(cid
, hoid
, 0, 15, newdata
);
809 expected
.append_zero(5);
811 ASSERT_TRUE(bl_eq(expected
, newdata
));
814 //additional write to an unused region of some blob
816 ObjectStore::Transaction t
;
817 bufferlist bl2
, newdata
;
818 bl2
.append("1234567890");
820 t
.write(cid
, hoid
, 20, bl2
.length(), bl2
);
821 cerr
<< "Append" << std::endl
;
822 r
= apply_transaction(store
, &osr
, std::move(t
));
825 r
= store
->read(cid
, hoid
, 0, 30, newdata
);
829 expected
.append("edcba");
830 expected
.append_zero(5);
831 expected
.append("edcba");
832 expected
.append_zero(5);
833 expected
.append(bl2
);
835 ASSERT_TRUE(bl_eq(expected
, newdata
));
838 //additional write to an unused region of some blob and partial owerite over existing extents
840 ObjectStore::Transaction t
;
841 bufferlist bl
, bl2
, bl3
, newdata
;
843 bl2
.append("1234567890");
846 t
.write(cid
, hoid
, 30, bl2
.length(), bl2
);
847 t
.write(cid
, hoid
, 1, bl
.length(), bl
);
848 t
.write(cid
, hoid
, 13, bl3
.length(), bl3
);
849 cerr
<< "TripleWrite" << std::endl
;
850 r
= apply_transaction(store
, &osr
, std::move(t
));
853 r
= store
->read(cid
, hoid
, 0, 40, newdata
);
857 expected
.append("eDCBa");
858 expected
.append_zero(5);
859 expected
.append("edcBA");
860 expected
.append_zero(5);
861 expected
.append(bl2
);
862 expected
.append(bl2
);
864 ASSERT_TRUE(bl_eq(expected
, newdata
));
869 void doCompressionTest( boost::scoped_ptr
<ObjectStore
>& store
)
871 ObjectStore::Sequencer
osr("test");
874 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
877 r
= store
->read(cid
, hoid
, 0, 5, in
);
878 ASSERT_EQ(-ENOENT
, r
);
881 ObjectStore::Transaction t
;
882 t
.create_collection(cid
, 0);
883 cerr
<< "Creating collection " << cid
<< std::endl
;
884 r
= apply_transaction(store
, &osr
, std::move(t
));
888 bool exists
= store
->exists(cid
, hoid
);
889 ASSERT_TRUE(!exists
);
891 ObjectStore::Transaction t
;
893 cerr
<< "Creating object " << hoid
<< std::endl
;
894 r
= apply_transaction(store
, &osr
, std::move(t
));
897 exists
= store
->exists(cid
, hoid
);
898 ASSERT_EQ(true, exists
);
901 data
.resize(0x10000 * 4);
902 for(size_t i
= 0;i
< data
.size(); i
++)
905 ObjectStore::Transaction t
;
906 bufferlist bl
, newdata
;
908 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
909 cerr
<< "CompressibleData (4xAU) Write" << std::endl
;
910 r
= apply_transaction(store
, &osr
, std::move(t
));
913 r
= store
->read(cid
, hoid
, 0, data
.size() , newdata
);
915 ASSERT_EQ(r
, (int)data
.size());
918 expected
.append(data
);
919 ASSERT_TRUE(bl_eq(expected
, newdata
));
922 r
= store
->read(cid
, hoid
, 0, 711 , newdata
);
926 expected
.append(data
.substr(0,711));
927 ASSERT_TRUE(bl_eq(expected
, newdata
));
930 r
= store
->read(cid
, hoid
, 0xf00f, data
.size(), newdata
);
931 ASSERT_EQ(r
, int(data
.size() - 0xf00f) );
934 expected
.append(data
.substr(0xf00f));
935 ASSERT_TRUE(bl_eq(expected
, newdata
));
938 struct store_statfs_t statfs
;
939 int r
= store
->statfs(&statfs
);
941 ASSERT_EQ(statfs
.stored
, (unsigned)data
.size());
942 ASSERT_LE(statfs
.compressed
, (unsigned)data
.size());
943 ASSERT_EQ(statfs
.compressed_original
, (unsigned)data
.size());
944 ASSERT_LE(statfs
.compressed_allocated
, (unsigned)data
.size());
948 data2
.resize(0x10000 * 4 - 0x9000);
949 for(size_t i
= 0;i
< data2
.size(); i
++)
950 data2
[i
] = (i
+1) / 256;
952 ObjectStore::Transaction t
;
953 bufferlist bl
, newdata
;
955 t
.write(cid
, hoid
, 0x8000, bl
.length(), bl
);
956 cerr
<< "CompressibleData partial overwrite" << std::endl
;
957 r
= apply_transaction(store
, &osr
, std::move(t
));
960 r
= store
->read(cid
, hoid
, 0, 0x10000, newdata
);
961 ASSERT_EQ(r
, (int)0x10000);
964 expected
.append(data
.substr(0, 0x8000));
965 expected
.append(data2
.substr(0, 0x8000));
966 ASSERT_TRUE(bl_eq(expected
, newdata
));
969 r
= store
->read(cid
, hoid
, 0x9000, 711 , newdata
);
973 expected
.append(data2
.substr(0x1000,711));
974 ASSERT_TRUE(bl_eq(expected
, newdata
));
977 r
= store
->read(cid
, hoid
, 0x0, 0x40000, newdata
);
978 ASSERT_EQ(r
, int(0x40000) );
981 expected
.append(data
.substr(0, 0x8000));
982 expected
.append(data2
.substr(0, 0x37000));
983 expected
.append(data
.substr(0x3f000, 0x1000));
984 ASSERT_TRUE(bl_eq(expected
, newdata
));
987 data2
.resize(0x3f000);
988 for(size_t i
= 0;i
< data2
.size(); i
++)
989 data2
[i
] = (i
+2) / 256;
991 ObjectStore::Transaction t
;
992 bufferlist bl
, newdata
;
994 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
995 cerr
<< "CompressibleData partial overwrite, two extents overlapped, single one to be removed" << std::endl
;
996 r
= apply_transaction(store
, &osr
, std::move(t
));
999 r
= store
->read(cid
, hoid
, 0, 0x3e000 - 1, newdata
);
1000 ASSERT_EQ(r
, (int)0x3e000 - 1);
1002 bufferlist expected
;
1003 expected
.append(data2
.substr(0, 0x3e000 - 1));
1004 ASSERT_TRUE(bl_eq(expected
, newdata
));
1007 r
= store
->read(cid
, hoid
, 0x3e000-1, 0x2001, newdata
);
1008 ASSERT_EQ(r
, 0x2001);
1010 bufferlist expected
;
1011 expected
.append(data2
.substr(0x3e000-1, 0x1001));
1012 expected
.append(data
.substr(0x3f000, 0x1000));
1013 ASSERT_TRUE(bl_eq(expected
, newdata
));
1016 r
= store
->read(cid
, hoid
, 0x0, 0x40000, newdata
);
1017 ASSERT_EQ(r
, int(0x40000) );
1019 bufferlist expected
;
1020 expected
.append(data2
.substr(0, 0x3f000));
1021 expected
.append(data
.substr(0x3f000, 0x1000));
1022 ASSERT_TRUE(bl_eq(expected
, newdata
));
1025 data
.resize(0x1001);
1026 for(size_t i
= 0;i
< data
.size(); i
++)
1027 data
[i
] = (i
+3) / 256;
1029 ObjectStore::Transaction t
;
1030 bufferlist bl
, newdata
;
1032 t
.write(cid
, hoid
, 0x3f000-1, bl
.length(), bl
);
1033 cerr
<< "Small chunk partial overwrite, two extents overlapped, single one to be removed" << std::endl
;
1034 r
= apply_transaction(store
, &osr
, std::move(t
));
1037 r
= store
->read(cid
, hoid
, 0x3e000, 0x2000, newdata
);
1038 ASSERT_EQ(r
, (int)0x2000);
1040 bufferlist expected
;
1041 expected
.append(data2
.substr(0x3e000, 0x1000 - 1));
1042 expected
.append(data
.substr(0, 0x1001));
1043 ASSERT_TRUE(bl_eq(expected
, newdata
));
1047 ObjectStore::Transaction t
;
1048 t
.remove(cid
, hoid
);
1049 cerr
<< "Cleaning object" << std::endl
;
1050 r
= apply_transaction(store
, &osr
, std::move(t
));
1054 EXPECT_EQ(store
->umount(), 0);
1055 EXPECT_EQ(store
->mount(), 0);
1056 auto orig_min_blob_size
= g_conf
->bluestore_compression_min_blob_size
;
1058 g_conf
->set_val("bluestore_compression_min_blob_size", "262144");
1059 g_ceph_context
->_conf
->apply_changes(NULL
);
1060 data
.resize(0x10000*6);
1062 for(size_t i
= 0;i
< data
.size(); i
++)
1064 ObjectStore::Transaction t
;
1065 bufferlist bl
, newdata
;
1067 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
1068 cerr
<< "CompressibleData large blob" << std::endl
;
1069 r
= apply_transaction(store
, &osr
, std::move(t
));
1073 EXPECT_EQ(store
->umount(), 0);
1074 EXPECT_EQ(store
->mount(), 0);
1077 ObjectStore::Transaction t
;
1078 t
.remove(cid
, hoid
);
1079 t
.remove_collection(cid
);
1080 cerr
<< "Cleaning" << std::endl
;
1081 r
= apply_transaction(store
, &osr
, std::move(t
));
1084 g_conf
->set_val("bluestore_compression_min_blob_size", stringify(orig_min_blob_size
));
1085 g_ceph_context
->_conf
->apply_changes(NULL
);
1088 TEST_P(StoreTest
, CompressionTest
) {
1089 if (string(GetParam()) != "bluestore")
1092 g_conf
->set_val("bluestore_compression_algorithm", "snappy");
1093 g_conf
->set_val("bluestore_compression_mode", "force");
1095 g_ceph_context
->_conf
->apply_changes(NULL
);
1097 doCompressionTest(store
);
1099 g_conf
->set_val("bluestore_compression_algorithm", "zlib");
1100 g_conf
->set_val("bluestore_compression_mode", "force");
1101 g_ceph_context
->_conf
->apply_changes(NULL
);
1103 doCompressionTest(store
);
1105 g_conf
->set_val("bluestore_compression_algorithm", "snappy");
1106 g_conf
->set_val("bluestore_compression_mode", "none");
1107 g_ceph_context
->_conf
->apply_changes(NULL
);
1110 TEST_P(StoreTest
, SimpleObjectTest
) {
1111 ObjectStore::Sequencer
osr("test");
1114 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
1117 r
= store
->read(cid
, hoid
, 0, 5, in
);
1118 ASSERT_EQ(-ENOENT
, r
);
1121 ObjectStore::Transaction t
;
1122 t
.create_collection(cid
, 0);
1123 cerr
<< "Creating collection " << cid
<< std::endl
;
1124 r
= apply_transaction(store
, &osr
, std::move(t
));
1128 bool exists
= store
->exists(cid
, hoid
);
1129 ASSERT_TRUE(!exists
);
1131 ObjectStore::Transaction t
;
1133 cerr
<< "Creating object " << hoid
<< std::endl
;
1134 r
= apply_transaction(store
, &osr
, std::move(t
));
1137 exists
= store
->exists(cid
, hoid
);
1138 ASSERT_EQ(true, exists
);
1141 ObjectStore::Transaction t
;
1142 t
.remove(cid
, hoid
);
1144 cerr
<< "Remove then create" << std::endl
;
1145 r
= apply_transaction(store
, &osr
, std::move(t
));
1149 ObjectStore::Transaction t
;
1150 bufferlist bl
, orig
;
1153 t
.remove(cid
, hoid
);
1154 t
.write(cid
, hoid
, 0, 5, bl
);
1155 cerr
<< "Remove then create" << std::endl
;
1156 r
= apply_transaction(store
, &osr
, std::move(t
));
1160 r
= store
->read(cid
, hoid
, 0, 5, in
);
1162 ASSERT_TRUE(bl_eq(orig
, in
));
1165 ObjectStore::Transaction t
;
1170 t
.write(cid
, hoid
, 5, 5, bl
);
1171 cerr
<< "Append" << std::endl
;
1172 r
= apply_transaction(store
, &osr
, std::move(t
));
1176 r
= store
->read(cid
, hoid
, 0, 10, in
);
1178 ASSERT_TRUE(bl_eq(exp
, in
));
1181 ObjectStore::Transaction t
;
1183 bl
.append("abcdeabcde");
1185 t
.write(cid
, hoid
, 0, 10, bl
);
1186 cerr
<< "Full overwrite" << std::endl
;
1187 r
= apply_transaction(store
, &osr
, std::move(t
));
1191 r
= store
->read(cid
, hoid
, 0, 10, in
);
1193 ASSERT_TRUE(bl_eq(exp
, in
));
1196 ObjectStore::Transaction t
;
1199 t
.write(cid
, hoid
, 3, 5, bl
);
1200 cerr
<< "Partial overwrite" << std::endl
;
1201 r
= apply_transaction(store
, &osr
, std::move(t
));
1205 exp
.append("abcabcdede");
1206 r
= store
->read(cid
, hoid
, 0, 10, in
);
1209 ASSERT_TRUE(bl_eq(exp
, in
));
1213 ObjectStore::Transaction t
;
1216 t
.truncate(cid
, hoid
, 0);
1217 t
.write(cid
, hoid
, 5, 5, bl
);
1218 cerr
<< "Truncate + hole" << std::endl
;
1219 r
= apply_transaction(store
, &osr
, std::move(t
));
1223 ObjectStore::Transaction t
;
1226 t
.write(cid
, hoid
, 0, 5, bl
);
1227 cerr
<< "Reverse fill-in" << std::endl
;
1228 r
= apply_transaction(store
, &osr
, std::move(t
));
1233 exp
.append("abcdefghij");
1234 r
= store
->read(cid
, hoid
, 0, 10, in
);
1237 ASSERT_TRUE(bl_eq(exp
, in
));
1240 ObjectStore::Transaction t
;
1242 bl
.append("abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234");
1243 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
1244 cerr
<< "larger overwrite" << std::endl
;
1245 r
= apply_transaction(store
, &osr
, std::move(t
));
1249 r
= store
->read(cid
, hoid
, 0, bl
.length(), in
);
1250 ASSERT_EQ((int)bl
.length(), r
);
1252 ASSERT_TRUE(bl_eq(bl
, in
));
1256 bl
.append("abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234abcde01234012340123401234");
1258 //test: offset=len=0 mean read all data
1260 r
= store
->read(cid
, hoid
, 0, 0, in
);
1261 ASSERT_EQ((int)bl
.length(), r
);
1263 ASSERT_TRUE(bl_eq(bl
, in
));
1266 //verifying unaligned csums
1267 std::string
s1("1"), s2(0x1000, '2'), s3("00");
1269 ObjectStore::Transaction t
;
1273 t
.truncate(cid
, hoid
, 0);
1274 t
.write(cid
, hoid
, 0x1000-1, bl
.length(), bl
);
1275 cerr
<< "Write unaligned csum, stage 1" << std::endl
;
1276 r
= apply_transaction(store
, &osr
, std::move(t
));
1280 bufferlist in
, exp1
, exp2
, exp3
;
1284 r
= store
->read(cid
, hoid
, 0x1000-1, 1, in
);
1286 ASSERT_TRUE(bl_eq(exp1
, in
));
1288 r
= store
->read(cid
, hoid
, 0x1000, 0x1000, in
);
1289 ASSERT_EQ(0x1000, r
);
1290 ASSERT_TRUE(bl_eq(exp2
, in
));
1293 ObjectStore::Transaction t
;
1296 t
.write(cid
, hoid
, 1, bl
.length(), bl
);
1297 cerr
<< "Write unaligned csum, stage 2" << std::endl
;
1298 r
= apply_transaction(store
, &osr
, std::move(t
));
1302 r
= store
->read(cid
, hoid
, 1, 2, in
);
1304 ASSERT_TRUE(bl_eq(exp3
, in
));
1306 r
= store
->read(cid
, hoid
, 0x1000-1, 1, in
);
1308 ASSERT_TRUE(bl_eq(exp1
, in
));
1310 r
= store
->read(cid
, hoid
, 0x1000, 0x1000, in
);
1311 ASSERT_EQ(0x1000, r
);
1312 ASSERT_TRUE(bl_eq(exp2
, in
));
1317 ObjectStore::Transaction t
;
1318 t
.remove(cid
, hoid
);
1319 t
.remove_collection(cid
);
1320 cerr
<< "Cleaning" << std::endl
;
1321 r
= apply_transaction(store
, &osr
, std::move(t
));
1326 #if defined(HAVE_LIBAIO)
1327 TEST_P(StoreTestSpecificAUSize
, BluestoreStatFSTest
) {
1328 if(string(GetParam()) != "bluestore")
1330 StartDeferred(65536);
1331 g_conf
->set_val("bluestore_compression_mode", "force");
1333 // just a big number to disble gc
1334 g_conf
->set_val("bluestore_gc_enable_total_threshold", "100000");
1335 g_conf
->apply_changes(NULL
);
1338 ObjectStore::Sequencer
osr("test");
1340 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
1341 ghobject_t hoid2
= hoid
;
1342 hoid2
.hobj
.snap
= 1;
1345 r
= store
->read(cid
, hoid
, 0, 5, in
);
1346 ASSERT_EQ(-ENOENT
, r
);
1349 ObjectStore::Transaction t
;
1350 t
.create_collection(cid
, 0);
1351 cerr
<< "Creating collection " << cid
<< std::endl
;
1352 r
= apply_transaction(store
, &osr
, std::move(t
));
1356 bool exists
= store
->exists(cid
, hoid
);
1357 ASSERT_TRUE(!exists
);
1359 ObjectStore::Transaction t
;
1361 cerr
<< "Creating object " << hoid
<< std::endl
;
1362 r
= apply_transaction(store
, &osr
, std::move(t
));
1365 exists
= store
->exists(cid
, hoid
);
1366 ASSERT_EQ(true, exists
);
1369 struct store_statfs_t statfs
;
1370 int r
= store
->statfs(&statfs
);
1372 ASSERT_EQ( 0u, statfs
.allocated
);
1373 ASSERT_EQ( 0u, statfs
.stored
);
1374 ASSERT_EQ(g_conf
->bluestore_block_size
, statfs
.total
);
1375 ASSERT_TRUE(statfs
.available
> 0u && statfs
.available
< g_conf
->bluestore_block_size
);
1377 EXPECT_EQ(store
->umount(), 0);
1378 EXPECT_EQ(store
->mount(), 0);
1381 ObjectStore::Transaction t
;
1384 t
.write(cid
, hoid
, 0, 5, bl
);
1385 cerr
<< "Append 5 bytes" << std::endl
;
1386 r
= apply_transaction(store
, &osr
, std::move(t
));
1389 struct store_statfs_t statfs
;
1390 int r
= store
->statfs(&statfs
);
1392 ASSERT_EQ(5, statfs
.stored
);
1393 ASSERT_EQ(0x10000, statfs
.allocated
);
1394 ASSERT_EQ(0, statfs
.compressed
);
1395 ASSERT_EQ(0, statfs
.compressed_original
);
1396 ASSERT_EQ(0, statfs
.compressed_allocated
);
1398 EXPECT_EQ(store
->umount(), 0);
1399 EXPECT_EQ(store
->mount(), 0);
1402 ObjectStore::Transaction t
;
1403 std::string
s(0x30000, 'a');
1406 t
.write(cid
, hoid
, 0x10000, bl
.length(), bl
);
1407 cerr
<< "Append 0x30000 compressible bytes" << std::endl
;
1408 r
= apply_transaction(store
, &osr
, std::move(t
));
1411 struct store_statfs_t statfs
;
1412 int r
= store
->statfs(&statfs
);
1414 ASSERT_EQ(0x30005, statfs
.stored
);
1415 ASSERT_EQ(0x30000, statfs
.allocated
);
1416 ASSERT_LE(statfs
.compressed
, 0x10000);
1417 ASSERT_EQ(0x20000, statfs
.compressed_original
);
1418 ASSERT_EQ(statfs
.compressed_allocated
, 0x10000);
1420 EXPECT_EQ(store
->umount(), 0);
1421 EXPECT_EQ(store
->mount(), 0);
1424 ObjectStore::Transaction t
;
1425 t
.zero(cid
, hoid
, 1, 3);
1426 t
.zero(cid
, hoid
, 0x20000, 9);
1427 cerr
<< "Punch hole at 1~3, 0x20000~9" << std::endl
;
1428 r
= apply_transaction(store
, &osr
, std::move(t
));
1431 struct store_statfs_t statfs
;
1432 int r
= store
->statfs(&statfs
);
1434 ASSERT_EQ(0x30005 - 3 - 9, statfs
.stored
);
1435 ASSERT_EQ(0x30000, statfs
.allocated
);
1436 ASSERT_LE(statfs
.compressed
, 0x10000);
1437 ASSERT_EQ(0x20000 - 9, statfs
.compressed_original
);
1438 ASSERT_EQ(statfs
.compressed_allocated
, 0x10000);
1440 EXPECT_EQ(store
->umount(), 0);
1441 EXPECT_EQ(store
->mount(), 0);
1444 ObjectStore::Transaction t
;
1445 std::string
s(0x1000, 'b');
1448 t
.write(cid
, hoid
, 1, bl
.length(), bl
);
1449 t
.write(cid
, hoid
, 0x10001, bl
.length(), bl
);
1450 cerr
<< "Overwrite first and second(compressible) extents" << std::endl
;
1451 r
= apply_transaction(store
, &osr
, std::move(t
));
1454 struct store_statfs_t statfs
;
1455 int r
= store
->statfs(&statfs
);
1457 ASSERT_EQ(0x30001 - 9 + 0x1000, statfs
.stored
);
1458 ASSERT_EQ(0x40000, statfs
.allocated
);
1459 ASSERT_LE(statfs
.compressed
, 0x10000);
1460 ASSERT_EQ(0x20000 - 9 - 0x1000, statfs
.compressed_original
);
1461 ASSERT_EQ(statfs
.compressed_allocated
, 0x10000);
1463 EXPECT_EQ(store
->umount(), 0);
1464 EXPECT_EQ(store
->mount(), 0);
1467 ObjectStore::Transaction t
;
1468 std::string
s(0x10000, 'c');
1471 t
.write(cid
, hoid
, 0x10000, bl
.length(), bl
);
1472 t
.write(cid
, hoid
, 0x20000, bl
.length(), bl
);
1473 t
.write(cid
, hoid
, 0x30000, bl
.length(), bl
);
1474 cerr
<< "Overwrite compressed extent with 3 uncompressible ones" << std::endl
;
1475 r
= apply_transaction(store
, &osr
, std::move(t
));
1478 struct store_statfs_t statfs
;
1479 int r
= store
->statfs(&statfs
);
1481 ASSERT_EQ(0x30000 + 0x1001, statfs
.stored
);
1482 ASSERT_EQ(0x40000, statfs
.allocated
);
1483 ASSERT_LE(statfs
.compressed
, 0);
1484 ASSERT_EQ(0, statfs
.compressed_original
);
1485 ASSERT_EQ(0, statfs
.compressed_allocated
);
1487 EXPECT_EQ(store
->umount(), 0);
1488 EXPECT_EQ(store
->mount(), 0);
1491 ObjectStore::Transaction t
;
1492 t
.zero(cid
, hoid
, 0, 0x40000);
1493 cerr
<< "Zero object" << std::endl
;
1494 r
= apply_transaction(store
, &osr
, std::move(t
));
1496 struct store_statfs_t statfs
;
1497 int r
= store
->statfs(&statfs
);
1499 ASSERT_EQ(0u, statfs
.allocated
);
1500 ASSERT_EQ(0u, statfs
.stored
);
1501 ASSERT_EQ(0u, statfs
.compressed_original
);
1502 ASSERT_EQ(0u, statfs
.compressed
);
1503 ASSERT_EQ(0u, statfs
.compressed_allocated
);
1505 EXPECT_EQ(store
->umount(), 0);
1506 EXPECT_EQ(store
->mount(), 0);
1509 ObjectStore::Transaction t
;
1510 std::string
s(0x10000, 'c');
1515 bl
.append(s
.substr(0, 0x10000-2));
1516 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
1517 cerr
<< "Yet another compressible write" << std::endl
;
1518 r
= apply_transaction(store
, &osr
, std::move(t
));
1520 struct store_statfs_t statfs
;
1521 r
= store
->statfs(&statfs
);
1523 ASSERT_EQ(0x40000 - 2, statfs
.stored
);
1524 ASSERT_EQ(0x30000, statfs
.allocated
);
1525 ASSERT_LE(statfs
.compressed
, 0x10000);
1526 ASSERT_EQ(0x20000, statfs
.compressed_original
);
1527 ASSERT_EQ(0x10000, statfs
.compressed_allocated
);
1529 EXPECT_EQ(store
->umount(), 0);
1530 EXPECT_EQ(store
->mount(), 0);
1533 struct store_statfs_t statfs
;
1534 r
= store
->statfs(&statfs
);
1537 ObjectStore::Transaction t
;
1538 t
.clone(cid
, hoid
, hoid2
);
1539 cerr
<< "Clone compressed objecte" << std::endl
;
1540 r
= apply_transaction(store
, &osr
, std::move(t
));
1542 struct store_statfs_t statfs2
;
1543 r
= store
->statfs(&statfs2
);
1545 ASSERT_GT(statfs2
.stored
, statfs
.stored
);
1546 ASSERT_EQ(statfs2
.allocated
, statfs
.allocated
);
1547 ASSERT_GT(statfs2
.compressed
, statfs
.compressed
);
1548 ASSERT_GT(statfs2
.compressed_original
, statfs
.compressed_original
);
1549 ASSERT_EQ(statfs2
.compressed_allocated
, statfs
.compressed_allocated
);
1553 ObjectStore::Transaction t
;
1554 t
.remove(cid
, hoid
);
1555 t
.remove(cid
, hoid2
);
1556 t
.remove_collection(cid
);
1557 cerr
<< "Cleaning" << std::endl
;
1558 r
= apply_transaction(store
, &osr
, std::move(t
));
1561 struct store_statfs_t statfs
;
1562 r
= store
->statfs(&statfs
);
1564 ASSERT_EQ( 0u, statfs
.allocated
);
1565 ASSERT_EQ( 0u, statfs
.stored
);
1566 ASSERT_EQ( 0u, statfs
.compressed_original
);
1567 ASSERT_EQ( 0u, statfs
.compressed
);
1568 ASSERT_EQ( 0u, statfs
.compressed_allocated
);
1570 g_conf
->set_val("bluestore_gc_enable_total_threshold", "0");
1571 g_conf
->set_val("bluestore_compression_mode", "none");
1572 g_ceph_context
->_conf
->apply_changes(NULL
);
1575 TEST_P(StoreTestSpecificAUSize
, BluestoreFragmentedBlobTest
) {
1576 if(string(GetParam()) != "bluestore")
1578 StartDeferred(0x10000);
1580 ObjectStore::Sequencer
osr("test");
1583 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
1585 ObjectStore::Transaction t
;
1586 t
.create_collection(cid
, 0);
1587 cerr
<< "Creating collection " << cid
<< std::endl
;
1588 r
= apply_transaction(store
, &osr
, std::move(t
));
1592 bool exists
= store
->exists(cid
, hoid
);
1593 ASSERT_TRUE(!exists
);
1595 ObjectStore::Transaction t
;
1597 cerr
<< "Creating object " << hoid
<< std::endl
;
1598 r
= apply_transaction(store
, &osr
, std::move(t
));
1601 exists
= store
->exists(cid
, hoid
);
1602 ASSERT_EQ(true, exists
);
1605 struct store_statfs_t statfs
;
1606 int r
= store
->statfs(&statfs
);
1608 ASSERT_EQ(g_conf
->bluestore_block_size
, statfs
.total
);
1609 ASSERT_EQ(0u, statfs
.allocated
);
1610 ASSERT_EQ(0u, statfs
.stored
);
1611 ASSERT_TRUE(statfs
.available
> 0u && statfs
.available
< g_conf
->bluestore_block_size
);
1614 data
.resize(0x10000 * 3);
1616 ObjectStore::Transaction t
;
1617 for(size_t i
= 0;i
< data
.size(); i
++)
1618 data
[i
] = i
/ 256 + 1;
1619 bufferlist bl
, newdata
;
1621 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
1622 t
.zero(cid
, hoid
, 0x10000, 0x10000);
1623 cerr
<< "Append 3*0x10000 bytes and punch a hole 0x10000~10000" << std::endl
;
1624 r
= apply_transaction(store
, &osr
, std::move(t
));
1627 struct store_statfs_t statfs
;
1628 int r
= store
->statfs(&statfs
);
1630 ASSERT_EQ(0x20000, statfs
.stored
);
1631 ASSERT_EQ(0x20000, statfs
.allocated
);
1633 r
= store
->read(cid
, hoid
, 0, data
.size(), newdata
);
1634 ASSERT_EQ(r
, (int)data
.size());
1636 bufferlist expected
;
1637 expected
.append(data
.substr(0, 0x10000));
1638 expected
.append(string(0x10000, 0));
1639 expected
.append(data
.substr(0x20000, 0x10000));
1640 ASSERT_TRUE(bl_eq(expected
, newdata
));
1644 r
= store
->read(cid
, hoid
, 1, data
.size()-2, newdata
);
1645 ASSERT_EQ(r
, (int)data
.size()-2);
1647 bufferlist expected
;
1648 expected
.append(data
.substr(1, 0x10000-1));
1649 expected
.append(string(0x10000, 0));
1650 expected
.append(data
.substr(0x20000, 0x10000 - 1));
1651 ASSERT_TRUE(bl_eq(expected
, newdata
));
1656 EXPECT_EQ(store
->umount(), 0);
1657 EXPECT_EQ(store
->mount(), 0);
1660 ObjectStore::Transaction t
;
1661 std::string
data2(3, 'b');
1662 bufferlist bl
, newdata
;
1664 t
.write(cid
, hoid
, 0x20000, bl
.length(), bl
);
1665 cerr
<< "Write 3 bytes after the hole" << std::endl
;
1666 r
= apply_transaction(store
, &osr
, std::move(t
));
1669 struct store_statfs_t statfs
;
1670 int r
= store
->statfs(&statfs
);
1672 ASSERT_EQ(0x20000, statfs
.allocated
);
1673 ASSERT_EQ(0x20000, statfs
.stored
);
1675 r
= store
->read(cid
, hoid
, 0x20000-1, 21, newdata
);
1676 ASSERT_EQ(r
, (int)21);
1678 bufferlist expected
;
1679 expected
.append(string(0x1, 0));
1680 expected
.append(string(data2
));
1681 expected
.append(data
.substr(0x20003, 21-4));
1682 ASSERT_TRUE(bl_eq(expected
, newdata
));
1687 EXPECT_EQ(store
->umount(), 0);
1688 EXPECT_EQ(store
->mount(), 0);
1691 ObjectStore::Transaction t
;
1692 std::string
data2(3, 'a');
1693 bufferlist bl
, newdata
;
1695 t
.write(cid
, hoid
, 0x10000+1, bl
.length(), bl
);
1696 cerr
<< "Write 3 bytes to the hole" << std::endl
;
1697 r
= apply_transaction(store
, &osr
, std::move(t
));
1700 struct store_statfs_t statfs
;
1701 int r
= store
->statfs(&statfs
);
1703 ASSERT_EQ(0x30000, statfs
.allocated
);
1704 ASSERT_EQ(0x20003, statfs
.stored
);
1706 r
= store
->read(cid
, hoid
, 0x10000-1, 0x10000+22, newdata
);
1707 ASSERT_EQ(r
, (int)0x10000+22);
1709 bufferlist expected
;
1710 expected
.append(data
.substr(0x10000-1, 1));
1711 expected
.append(string(0x1, 0));
1712 expected
.append(data2
);
1713 expected
.append(string(0x10000-4, 0));
1714 expected
.append(string(0x3, 'b'));
1715 expected
.append(data
.substr(0x20004, 21-3));
1716 ASSERT_TRUE(bl_eq(expected
, newdata
));
1721 ObjectStore::Transaction t
;
1722 bufferlist bl
, newdata
;
1723 bl
.append(string(0x30000, 'c'));
1724 t
.write(cid
, hoid
, 0, 0x30000, bl
);
1725 t
.zero(cid
, hoid
, 0, 0x10000);
1726 t
.zero(cid
, hoid
, 0x20000, 0x10000);
1727 cerr
<< "Rewrite an object and create two holes at the begining and the end" << std::endl
;
1728 r
= apply_transaction(store
, &osr
, std::move(t
));
1731 struct store_statfs_t statfs
;
1732 int r
= store
->statfs(&statfs
);
1734 ASSERT_EQ(0x10000, statfs
.allocated
);
1735 ASSERT_EQ(0x10000, statfs
.stored
);
1737 r
= store
->read(cid
, hoid
, 0, 0x30000, newdata
);
1738 ASSERT_EQ(r
, (int)0x30000);
1740 bufferlist expected
;
1741 expected
.append(string(0x10000, 0));
1742 expected
.append(string(0x10000, 'c'));
1743 expected
.append(string(0x10000, 0));
1744 ASSERT_TRUE(bl_eq(expected
, newdata
));
1750 EXPECT_EQ(store
->umount(), 0);
1751 EXPECT_EQ(store
->mount(), 0);
1754 ObjectStore::Transaction t
;
1755 t
.remove(cid
, hoid
);
1756 t
.remove_collection(cid
);
1757 cerr
<< "Cleaning" << std::endl
;
1758 r
= apply_transaction(store
, &osr
, std::move(t
));
1761 struct store_statfs_t statfs
;
1762 r
= store
->statfs(&statfs
);
1764 ASSERT_EQ( 0u, statfs
.allocated
);
1765 ASSERT_EQ( 0u, statfs
.stored
);
1766 ASSERT_EQ( 0u, statfs
.compressed_original
);
1767 ASSERT_EQ( 0u, statfs
.compressed
);
1768 ASSERT_EQ( 0u, statfs
.compressed_allocated
);
1773 TEST_P(StoreTest
, ManySmallWrite
) {
1774 ObjectStore::Sequencer
osr("test");
1777 ghobject_t
a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
1778 ghobject_t
b(hobject_t(sobject_t("Object 2", CEPH_NOSNAP
)));
1780 ObjectStore::Transaction t
;
1781 t
.create_collection(cid
, 0);
1782 cerr
<< "Creating collection " << cid
<< std::endl
;
1783 r
= apply_transaction(store
, &osr
, std::move(t
));
1790 for (int i
=0; i
<100; ++i
) {
1791 ObjectStore::Transaction t
;
1792 t
.write(cid
, a
, i
*4096, 4096, bl
, 0);
1793 r
= apply_transaction(store
, &osr
, std::move(t
));
1796 for (int i
=0; i
<100; ++i
) {
1797 ObjectStore::Transaction t
;
1798 t
.write(cid
, b
, (rand() % 1024)*4096, 4096, bl
, 0);
1799 r
= apply_transaction(store
, &osr
, std::move(t
));
1803 ObjectStore::Transaction t
;
1806 t
.remove_collection(cid
);
1807 cerr
<< "Cleaning" << std::endl
;
1808 r
= apply_transaction(store
, &osr
, std::move(t
));
1813 TEST_P(StoreTest
, MultiSmallWriteSameBlock
) {
1814 ObjectStore::Sequencer
osr("test");
1817 ghobject_t
a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
1819 ObjectStore::Transaction t
;
1820 t
.create_collection(cid
, 0);
1821 cerr
<< "Creating collection " << cid
<< std::endl
;
1822 r
= apply_transaction(store
, &osr
, std::move(t
));
1828 // touch same block in both same transaction, tls, and pipelined txns
1830 ObjectStore::Transaction t
, u
;
1831 t
.write(cid
, a
, 0, 5, bl
, 0);
1832 t
.write(cid
, a
, 5, 5, bl
, 0);
1833 t
.write(cid
, a
, 4094, 5, bl
, 0);
1834 t
.write(cid
, a
, 9000, 5, bl
, 0);
1835 u
.write(cid
, a
, 10, 5, bl
, 0);
1836 u
.write(cid
, a
, 7000, 5, bl
, 0);
1837 vector
<ObjectStore::Transaction
> v
= {t
, u
};
1838 store
->queue_transactions(&osr
, v
, nullptr, &c
);
1841 ObjectStore::Transaction t
, u
;
1842 t
.write(cid
, a
, 40, 5, bl
, 0);
1843 t
.write(cid
, a
, 45, 5, bl
, 0);
1844 t
.write(cid
, a
, 4094, 5, bl
, 0);
1845 t
.write(cid
, a
, 6000, 5, bl
, 0);
1846 u
.write(cid
, a
, 610, 5, bl
, 0);
1847 u
.write(cid
, a
, 11000, 5, bl
, 0);
1848 vector
<ObjectStore::Transaction
> v
= {t
, u
};
1849 store
->queue_transactions(&osr
, v
, nullptr, &d
);
1855 r
= store
->read(cid
, a
, 0, 16000, bl2
);
1859 ObjectStore::Transaction t
;
1861 t
.remove_collection(cid
);
1862 cerr
<< "Cleaning" << std::endl
;
1863 r
= apply_transaction(store
, &osr
, std::move(t
));
1868 TEST_P(StoreTest
, SmallSkipFront
) {
1869 ObjectStore::Sequencer
osr("test");
1872 ghobject_t
a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
1874 ObjectStore::Transaction t
;
1875 t
.create_collection(cid
, 0);
1876 cerr
<< "Creating collection " << cid
<< std::endl
;
1877 r
= apply_transaction(store
, &osr
, std::move(t
));
1881 ObjectStore::Transaction t
;
1883 t
.truncate(cid
, a
, 3000);
1884 r
= apply_transaction(store
, &osr
, std::move(t
));
1890 memset(bp
.c_str(), 1, 4096);
1892 ObjectStore::Transaction t
;
1893 t
.write(cid
, a
, 4096, 4096, bl
);
1894 r
= apply_transaction(store
, &osr
, std::move(t
));
1899 ASSERT_EQ(8192, store
->read(cid
, a
, 0, 8192, bl
));
1900 for (unsigned i
=0; i
<4096; ++i
)
1901 ASSERT_EQ(0, bl
[i
]);
1902 for (unsigned i
=4096; i
<8192; ++i
)
1903 ASSERT_EQ(1, bl
[i
]);
1906 ObjectStore::Transaction t
;
1908 t
.remove_collection(cid
);
1909 cerr
<< "Cleaning" << std::endl
;
1910 r
= apply_transaction(store
, &osr
, std::move(t
));
1915 TEST_P(StoreTest
, AppendDeferredVsTailCache
) {
1916 ObjectStore::Sequencer
osr("test");
1919 ghobject_t
a(hobject_t(sobject_t("fooo", CEPH_NOSNAP
)));
1921 ObjectStore::Transaction t
;
1922 t
.create_collection(cid
, 0);
1923 cerr
<< "Creating collection " << cid
<< std::endl
;
1924 r
= store
->apply_transaction(&osr
, std::move(t
));
1927 unsigned min_alloc
= g_conf
->bluestore_min_alloc_size
;
1928 g_conf
->set_val("bluestore_inject_deferred_apply_delay", "1.0");
1929 g_ceph_context
->_conf
->apply_changes(NULL
);
1930 unsigned size
= min_alloc
/ 3;
1931 bufferptr
bpa(size
);
1932 memset(bpa
.c_str(), 1, bpa
.length());
1936 ObjectStore::Transaction t
;
1937 t
.write(cid
, a
, 0, bla
.length(), bla
, 0);
1938 r
= store
->apply_transaction(&osr
, std::move(t
));
1942 // force cached tail to clear ...
1944 int r
= store
->umount();
1950 bufferptr
bpb(size
);
1951 memset(bpb
.c_str(), 2, bpb
.length());
1955 ObjectStore::Transaction t
;
1956 t
.write(cid
, a
, bla
.length(), blb
.length(), blb
, 0);
1957 r
= store
->apply_transaction(&osr
, std::move(t
));
1960 bufferptr
bpc(size
);
1961 memset(bpc
.c_str(), 3, bpc
.length());
1965 ObjectStore::Transaction t
;
1966 t
.write(cid
, a
, bla
.length() + blb
.length(), blc
.length(), blc
, 0);
1967 r
= store
->apply_transaction(&osr
, std::move(t
));
1976 ASSERT_EQ((int)final
.length(),
1977 store
->read(cid
, a
, 0, final
.length(), actual
));
1978 ASSERT_TRUE(bl_eq(final
, actual
));
1981 ObjectStore::Transaction t
;
1983 t
.remove_collection(cid
);
1984 cerr
<< "Cleaning" << std::endl
;
1985 r
= store
->apply_transaction(&osr
, std::move(t
));
1988 g_conf
->set_val("bluestore_inject_deferred_apply_delay", "0");
1989 g_ceph_context
->_conf
->apply_changes(NULL
);
1992 TEST_P(StoreTest
, AppendZeroTrailingSharedBlock
) {
1993 ObjectStore::Sequencer
osr("test");
1996 ghobject_t
a(hobject_t(sobject_t("fooo", CEPH_NOSNAP
)));
2000 ObjectStore::Transaction t
;
2001 t
.create_collection(cid
, 0);
2002 cerr
<< "Creating collection " << cid
<< std::endl
;
2003 r
= store
->apply_transaction(&osr
, std::move(t
));
2006 unsigned min_alloc
= g_conf
->bluestore_min_alloc_size
;
2007 unsigned size
= min_alloc
/ 3;
2008 bufferptr
bpa(size
);
2009 memset(bpa
.c_str(), 1, bpa
.length());
2012 // make sure there is some trailing gunk in the last block
2016 bt
.append("BADBADBADBAD");
2017 ObjectStore::Transaction t
;
2018 t
.write(cid
, a
, 0, bt
.length(), bt
, 0);
2019 r
= store
->apply_transaction(&osr
, std::move(t
));
2023 ObjectStore::Transaction t
;
2024 t
.truncate(cid
, a
, size
);
2025 r
= store
->apply_transaction(&osr
, std::move(t
));
2031 ObjectStore::Transaction t
;
2033 r
= store
->apply_transaction(&osr
, std::move(t
));
2037 // append with implicit zeroing
2038 bufferptr
bpb(size
);
2039 memset(bpb
.c_str(), 2, bpb
.length());
2043 ObjectStore::Transaction t
;
2044 t
.write(cid
, a
, min_alloc
* 3, blb
.length(), blb
, 0);
2045 r
= store
->apply_transaction(&osr
, std::move(t
));
2051 zeros
.append_zero(min_alloc
* 3 - size
);
2052 final
.append(zeros
);
2056 ASSERT_EQ((int)final
.length(),
2057 store
->read(cid
, a
, 0, final
.length(), actual
));
2058 final
.hexdump(cout
);
2059 actual
.hexdump(cout
);
2060 ASSERT_TRUE(bl_eq(final
, actual
));
2063 ObjectStore::Transaction t
;
2066 t
.remove_collection(cid
);
2067 cerr
<< "Cleaning" << std::endl
;
2068 r
= store
->apply_transaction(&osr
, std::move(t
));
2073 TEST_P(StoreTest
, SmallSequentialUnaligned
) {
2074 ObjectStore::Sequencer
osr("test");
2077 ghobject_t
a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
2079 ObjectStore::Transaction t
;
2080 t
.create_collection(cid
, 0);
2081 cerr
<< "Creating collection " << cid
<< std::endl
;
2082 r
= apply_transaction(store
, &osr
, std::move(t
));
2090 for (int i
=0; i
<1000; ++i
) {
2091 ObjectStore::Transaction t
;
2092 t
.write(cid
, a
, i
*len
, len
, bl
, 0);
2093 r
= apply_transaction(store
, &osr
, std::move(t
));
2097 ObjectStore::Transaction t
;
2099 t
.remove_collection(cid
);
2100 cerr
<< "Cleaning" << std::endl
;
2101 r
= apply_transaction(store
, &osr
, std::move(t
));
2106 TEST_P(StoreTest
, ManyBigWrite
) {
2107 ObjectStore::Sequencer
osr("test");
2110 ghobject_t
a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
2111 ghobject_t
b(hobject_t(sobject_t("Object 2", CEPH_NOSNAP
)));
2113 ObjectStore::Transaction t
;
2114 t
.create_collection(cid
, 0);
2115 cerr
<< "Creating collection " << cid
<< std::endl
;
2116 r
= apply_transaction(store
, &osr
, std::move(t
));
2120 bufferptr
bp(4 * 1048576);
2123 for (int i
=0; i
<10; ++i
) {
2124 ObjectStore::Transaction t
;
2125 t
.write(cid
, a
, i
*4*1048586, 4*1048576, bl
, 0);
2126 r
= apply_transaction(store
, &osr
, std::move(t
));
2130 for (int i
=0; i
<10; ++i
) {
2131 ObjectStore::Transaction t
;
2132 t
.write(cid
, b
, (rand() % 256)*4*1048576, 4*1048576, bl
, 0);
2133 r
= apply_transaction(store
, &osr
, std::move(t
));
2137 for (int i
=0; i
<10; ++i
) {
2138 ObjectStore::Transaction t
;
2139 t
.write(cid
, b
, (rand() % (256*4096))*1024, 4*1048576, bl
, 0);
2140 r
= apply_transaction(store
, &osr
, std::move(t
));
2144 for (int i
=0; i
<10; ++i
) {
2145 ObjectStore::Transaction t
;
2146 t
.zero(cid
, b
, (rand() % (256*4096))*1024, 16*1048576);
2147 r
= apply_transaction(store
, &osr
, std::move(t
));
2151 ObjectStore::Transaction t
;
2154 t
.remove_collection(cid
);
2155 cerr
<< "Cleaning" << std::endl
;
2156 r
= apply_transaction(store
, &osr
, std::move(t
));
2161 TEST_P(StoreTest
, BigWriteBigZero
) {
2162 ObjectStore::Sequencer
osr("test");
2165 ghobject_t
a(hobject_t(sobject_t("foo", CEPH_NOSNAP
)));
2167 ObjectStore::Transaction t
;
2168 t
.create_collection(cid
, 0);
2169 r
= apply_transaction(store
, &osr
, std::move(t
));
2173 bufferptr
bp(1048576);
2174 memset(bp
.c_str(), 'b', bp
.length());
2178 memset(sp
.c_str(), 's', sp
.length());
2181 ObjectStore::Transaction t
;
2182 t
.write(cid
, a
, 0, bl
.length(), bl
);
2183 r
= apply_transaction(store
, &osr
, std::move(t
));
2187 ObjectStore::Transaction t
;
2188 t
.zero(cid
, a
, bl
.length() / 4, bl
.length() / 2);
2189 r
= apply_transaction(store
, &osr
, std::move(t
));
2193 ObjectStore::Transaction t
;
2194 t
.write(cid
, a
, bl
.length() / 2, s
.length(), s
);
2195 r
= apply_transaction(store
, &osr
, std::move(t
));
2199 ObjectStore::Transaction t
;
2201 t
.remove_collection(cid
);
2202 r
= apply_transaction(store
, &osr
, std::move(t
));
2207 TEST_P(StoreTest
, MiscFragmentTests
) {
2208 ObjectStore::Sequencer
osr("test");
2211 ghobject_t
a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
2213 ObjectStore::Transaction t
;
2214 t
.create_collection(cid
, 0);
2215 cerr
<< "Creating collection " << cid
<< std::endl
;
2216 r
= apply_transaction(store
, &osr
, std::move(t
));
2220 bufferptr
bp(524288);
2224 ObjectStore::Transaction t
;
2225 t
.write(cid
, a
, 0, 524288, bl
, 0);
2226 r
= apply_transaction(store
, &osr
, std::move(t
));
2230 ObjectStore::Transaction t
;
2231 t
.write(cid
, a
, 1048576, 524288, bl
, 0);
2232 r
= apply_transaction(store
, &osr
, std::move(t
));
2237 int r
= store
->read(cid
, a
, 524288 + 131072, 1024, inbl
);
2239 ASSERT_EQ(inbl
.length(), 1024u);
2240 ASSERT_TRUE(inbl
.is_zero());
2243 ObjectStore::Transaction t
;
2244 t
.write(cid
, a
, 1048576 - 4096, 524288, bl
, 0);
2245 r
= apply_transaction(store
, &osr
, std::move(t
));
2249 ObjectStore::Transaction t
;
2251 t
.remove_collection(cid
);
2252 cerr
<< "Cleaning" << std::endl
;
2253 r
= apply_transaction(store
, &osr
, std::move(t
));
2259 TEST_P(StoreTest
, ZeroVsObjectSize
) {
2260 ObjectStore::Sequencer
osr("test");
2264 ghobject_t
hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP
)));
2266 ObjectStore::Transaction t
;
2267 t
.create_collection(cid
, 0);
2268 cerr
<< "Creating collection " << cid
<< std::endl
;
2269 r
= apply_transaction(store
, &osr
, std::move(t
));
2275 ObjectStore::Transaction t
;
2276 t
.write(cid
, hoid
, 0, 5, a
);
2277 r
= apply_transaction(store
, &osr
, std::move(t
));
2280 ASSERT_EQ(0, store
->stat(cid
, hoid
, &stat
));
2281 ASSERT_EQ(5, stat
.st_size
);
2283 ObjectStore::Transaction t
;
2284 t
.zero(cid
, hoid
, 1, 2);
2285 r
= apply_transaction(store
, &osr
, std::move(t
));
2288 ASSERT_EQ(0, store
->stat(cid
, hoid
, &stat
));
2289 ASSERT_EQ(5, stat
.st_size
);
2291 ObjectStore::Transaction t
;
2292 t
.zero(cid
, hoid
, 3, 200);
2293 r
= apply_transaction(store
, &osr
, std::move(t
));
2296 ASSERT_EQ(0, store
->stat(cid
, hoid
, &stat
));
2297 ASSERT_EQ(203, stat
.st_size
);
2299 ObjectStore::Transaction t
;
2300 t
.zero(cid
, hoid
, 100000, 200);
2301 r
= apply_transaction(store
, &osr
, std::move(t
));
2304 ASSERT_EQ(0, store
->stat(cid
, hoid
, &stat
));
2305 ASSERT_EQ(100200, stat
.st_size
);
2308 TEST_P(StoreTest
, ZeroLengthWrite
) {
2309 ObjectStore::Sequencer
osr("test");
2312 ghobject_t
hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP
)));
2314 ObjectStore::Transaction t
;
2315 t
.create_collection(cid
, 0);
2317 r
= apply_transaction(store
, &osr
, std::move(t
));
2321 ObjectStore::Transaction t
;
2323 t
.write(cid
, hoid
, 1048576, 0, empty
);
2324 r
= apply_transaction(store
, &osr
, std::move(t
));
2328 r
= store
->stat(cid
, hoid
, &stat
);
2330 ASSERT_EQ(0, stat
.st_size
);
2333 r
= store
->read(cid
, hoid
, 0, 1048576, newdata
);
2337 TEST_P(StoreTest
, ZeroLengthZero
) {
2338 ObjectStore::Sequencer
osr("test");
2341 ghobject_t
hoid(hobject_t(sobject_t("foo", CEPH_NOSNAP
)));
2343 ObjectStore::Transaction t
;
2344 t
.create_collection(cid
, 0);
2346 r
= apply_transaction(store
, &osr
, std::move(t
));
2350 ObjectStore::Transaction t
;
2351 t
.zero(cid
, hoid
, 1048576, 0);
2352 r
= apply_transaction(store
, &osr
, std::move(t
));
2356 r
= store
->stat(cid
, hoid
, &stat
);
2358 ASSERT_EQ(0, stat
.st_size
);
2361 r
= store
->read(cid
, hoid
, 0, 1048576, newdata
);
2365 TEST_P(StoreTest
, SimpleAttrTest
) {
2366 ObjectStore::Sequencer
osr("test");
2369 ghobject_t
hoid(hobject_t(sobject_t("attr object 1", CEPH_NOSNAP
)));
2370 bufferlist val
, val2
;
2371 val
.append("value");
2372 val
.append("value2");
2375 map
<string
,bufferptr
> aset
;
2376 r
= store
->getattr(cid
, hoid
, "nofoo", bp
);
2377 ASSERT_EQ(-ENOENT
, r
);
2378 r
= store
->getattrs(cid
, hoid
, aset
);
2379 ASSERT_EQ(-ENOENT
, r
);
2382 ObjectStore::Transaction t
;
2383 t
.create_collection(cid
, 0);
2384 r
= apply_transaction(store
, &osr
, std::move(t
));
2390 int r
= store
->collection_empty(cid
, &empty
);
2396 r
= store
->getattr(cid
, hoid
, "nofoo", bp
);
2397 ASSERT_EQ(-ENOENT
, r
);
2400 ObjectStore::Transaction t
;
2402 t
.setattr(cid
, hoid
, "foo", val
);
2403 t
.setattr(cid
, hoid
, "bar", val2
);
2404 r
= apply_transaction(store
, &osr
, std::move(t
));
2410 int r
= store
->collection_empty(cid
, &empty
);
2412 ASSERT_TRUE(!empty
);
2416 r
= store
->getattr(cid
, hoid
, "nofoo", bp
);
2417 ASSERT_EQ(-ENODATA
, r
);
2419 r
= store
->getattr(cid
, hoid
, "foo", bp
);
2423 ASSERT_TRUE(bl_eq(val
, bl
));
2425 map
<string
,bufferptr
> bm
;
2426 r
= store
->getattrs(cid
, hoid
, bm
);
2431 ObjectStore::Transaction t
;
2432 t
.remove(cid
, hoid
);
2433 t
.remove_collection(cid
);
2434 r
= apply_transaction(store
, &osr
, std::move(t
));
2439 TEST_P(StoreTest
, SimpleListTest
) {
2440 ObjectStore::Sequencer
osr("test");
2442 coll_t
cid(spg_t(pg_t(0, 1), shard_id_t(1)));
2444 ObjectStore::Transaction t
;
2445 t
.create_collection(cid
, 0);
2446 cerr
<< "Creating collection " << cid
<< std::endl
;
2447 r
= apply_transaction(store
, &osr
, std::move(t
));
2450 set
<ghobject_t
> all
;
2452 ObjectStore::Transaction t
;
2453 for (int i
=0; i
<200; ++i
) {
2454 string
name("object_");
2455 name
+= stringify(i
);
2456 ghobject_t
hoid(hobject_t(sobject_t(name
, CEPH_NOSNAP
)),
2457 ghobject_t::NO_GEN
, shard_id_t(1));
2461 cerr
<< "Creating object " << hoid
<< std::endl
;
2463 r
= apply_transaction(store
, &osr
, std::move(t
));
2467 set
<ghobject_t
> saw
;
2468 vector
<ghobject_t
> objects
;
2469 ghobject_t next
, current
;
2470 while (!next
.is_max()) {
2471 int r
= store
->collection_list(cid
, current
, ghobject_t::get_max(),
2475 ASSERT_TRUE(sorted(objects
));
2476 cout
<< " got " << objects
.size() << " next " << next
<< std::endl
;
2477 for (vector
<ghobject_t
>::iterator p
= objects
.begin(); p
!= objects
.end();
2479 if (saw
.count(*p
)) {
2480 cout
<< "got DUP " << *p
<< std::endl
;
2482 //cout << "got new " << *p << std::endl;
2489 ASSERT_EQ(saw
.size(), all
.size());
2490 ASSERT_EQ(saw
, all
);
2493 ObjectStore::Transaction t
;
2494 for (set
<ghobject_t
>::iterator p
= all
.begin(); p
!= all
.end(); ++p
)
2496 t
.remove_collection(cid
);
2497 cerr
<< "Cleaning" << std::endl
;
2498 r
= apply_transaction(store
, &osr
, std::move(t
));
2503 TEST_P(StoreTest
, ListEndTest
) {
2504 ObjectStore::Sequencer
osr("test");
2506 coll_t
cid(spg_t(pg_t(0, 1), shard_id_t(1)));
2508 ObjectStore::Transaction t
;
2509 t
.create_collection(cid
, 0);
2510 cerr
<< "Creating collection " << cid
<< std::endl
;
2511 r
= apply_transaction(store
, &osr
, std::move(t
));
2514 set
<ghobject_t
> all
;
2516 ObjectStore::Transaction t
;
2517 for (int i
=0; i
<200; ++i
) {
2518 string
name("object_");
2519 name
+= stringify(i
);
2520 ghobject_t
hoid(hobject_t(sobject_t(name
, CEPH_NOSNAP
)),
2521 ghobject_t::NO_GEN
, shard_id_t(1));
2525 cerr
<< "Creating object " << hoid
<< std::endl
;
2527 r
= apply_transaction(store
, &osr
, std::move(t
));
2531 ghobject_t
end(hobject_t(sobject_t("object_100", CEPH_NOSNAP
)),
2532 ghobject_t::NO_GEN
, shard_id_t(1));
2534 vector
<ghobject_t
> objects
;
2536 int r
= store
->collection_list(cid
, ghobject_t(), end
, 500,
2539 for (auto &p
: objects
) {
2544 ObjectStore::Transaction t
;
2545 for (set
<ghobject_t
>::iterator p
= all
.begin(); p
!= all
.end(); ++p
)
2547 t
.remove_collection(cid
);
2548 cerr
<< "Cleaning" << std::endl
;
2549 r
= apply_transaction(store
, &osr
, std::move(t
));
2554 TEST_P(StoreTest
, Sort
) {
2556 hobject_t
a(sobject_t("a", CEPH_NOSNAP
));
2569 ghobject_t
a(hobject_t(sobject_t("a", CEPH_NOSNAP
)));
2570 ghobject_t
b(hobject_t(sobject_t("b", CEPH_NOSNAP
)));
2582 TEST_P(StoreTest
, MultipoolListTest
) {
2583 ObjectStore::Sequencer
osr("test");
2586 coll_t cid
= coll_t(spg_t(pg_t(0, poolid
), shard_id_t::NO_SHARD
));
2588 ObjectStore::Transaction t
;
2589 t
.create_collection(cid
, 0);
2590 cerr
<< "Creating collection " << cid
<< std::endl
;
2591 r
= apply_transaction(store
, &osr
, std::move(t
));
2594 set
<ghobject_t
> all
, saw
;
2596 ObjectStore::Transaction t
;
2597 for (int i
=0; i
<200; ++i
) {
2598 string
name("object_");
2599 name
+= stringify(i
);
2600 ghobject_t
hoid(hobject_t(sobject_t(name
, CEPH_NOSNAP
)));
2602 hoid
.hobj
.pool
= -2 - poolid
;
2604 hoid
.hobj
.pool
= poolid
;
2607 cerr
<< "Creating object " << hoid
<< std::endl
;
2609 r
= apply_transaction(store
, &osr
, std::move(t
));
2613 vector
<ghobject_t
> objects
;
2614 ghobject_t next
, current
;
2615 while (!next
.is_max()) {
2616 int r
= store
->collection_list(cid
, current
, ghobject_t::get_max(), 50,
2619 cout
<< " got " << objects
.size() << " next " << next
<< std::endl
;
2620 for (vector
<ghobject_t
>::iterator p
= objects
.begin(); p
!= objects
.end();
2627 ASSERT_EQ(saw
, all
);
2630 ObjectStore::Transaction t
;
2631 for (set
<ghobject_t
>::iterator p
= all
.begin(); p
!= all
.end(); ++p
)
2633 t
.remove_collection(cid
);
2634 cerr
<< "Cleaning" << std::endl
;
2635 r
= apply_transaction(store
, &osr
, std::move(t
));
2640 TEST_P(StoreTest
, SimpleCloneTest
) {
2641 ObjectStore::Sequencer
osr("test");
2645 ObjectStore::Transaction t
;
2646 t
.create_collection(cid
, 0);
2647 cerr
<< "Creating collection " << cid
<< std::endl
;
2648 r
= apply_transaction(store
, &osr
, std::move(t
));
2651 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
),
2652 "key", 123, -1, ""));
2653 bufferlist small
, large
, xlarge
, newdata
, attr
;
2654 small
.append("small");
2655 large
.append("large");
2656 xlarge
.append("xlarge");
2658 ObjectStore::Transaction t
;
2660 t
.setattr(cid
, hoid
, "attr1", small
);
2661 t
.setattr(cid
, hoid
, "attr2", large
);
2662 t
.setattr(cid
, hoid
, "attr3", xlarge
);
2663 t
.write(cid
, hoid
, 0, small
.length(), small
);
2664 t
.write(cid
, hoid
, 10, small
.length(), small
);
2665 cerr
<< "Creating object and set attr " << hoid
<< std::endl
;
2666 r
= apply_transaction(store
, &osr
, std::move(t
));
2670 ghobject_t
hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP
),
2671 "key", 123, -1, ""));
2672 ghobject_t
hoid3(hobject_t(sobject_t("Object 3", CEPH_NOSNAP
)));
2674 ObjectStore::Transaction t
;
2675 t
.clone(cid
, hoid
, hoid2
);
2676 t
.setattr(cid
, hoid2
, "attr2", small
);
2677 t
.rmattr(cid
, hoid2
, "attr1");
2678 t
.write(cid
, hoid
, 10, large
.length(), large
);
2679 t
.setattr(cid
, hoid
, "attr1", large
);
2680 t
.setattr(cid
, hoid
, "attr2", small
);
2681 cerr
<< "Clone object and rm attr" << std::endl
;
2682 r
= apply_transaction(store
, &osr
, std::move(t
));
2685 r
= store
->read(cid
, hoid
, 10, 5, newdata
);
2687 ASSERT_TRUE(bl_eq(large
, newdata
));
2690 r
= store
->read(cid
, hoid
, 0, 5, newdata
);
2692 ASSERT_TRUE(bl_eq(small
, newdata
));
2695 r
= store
->read(cid
, hoid2
, 10, 5, newdata
);
2697 ASSERT_TRUE(bl_eq(small
, newdata
));
2699 r
= store
->getattr(cid
, hoid2
, "attr2", attr
);
2701 ASSERT_TRUE(bl_eq(small
, attr
));
2704 r
= store
->getattr(cid
, hoid2
, "attr3", attr
);
2706 ASSERT_TRUE(bl_eq(xlarge
, attr
));
2709 r
= store
->getattr(cid
, hoid
, "attr1", attr
);
2711 ASSERT_TRUE(bl_eq(large
, attr
));
2714 ObjectStore::Transaction t
;
2715 t
.remove(cid
, hoid
);
2716 t
.remove(cid
, hoid2
);
2717 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2722 memset(p
.c_str(), 1, p
.length());
2726 ObjectStore::Transaction t
;
2727 t
.write(cid
, hoid
, 0, pl
.length(), pl
);
2728 t
.clone(cid
, hoid
, hoid2
);
2730 memset(a
.c_str(), 2, a
.length());
2734 t
.write(cid
, hoid
, pl
.length(), a
.length(), al
);
2735 r
= apply_transaction(store
, &osr
, std::move(t
));
2738 ASSERT_EQ((int)final
.length(),
2739 store
->read(cid
, hoid
, 0, final
.length(), rl
));
2740 ASSERT_TRUE(bl_eq(rl
, final
));
2743 ObjectStore::Transaction t
;
2744 t
.remove(cid
, hoid
);
2745 t
.remove(cid
, hoid2
);
2746 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2751 memset(p
.c_str(), 111, p
.length());
2755 ObjectStore::Transaction t
;
2756 t
.write(cid
, hoid
, 0, pl
.length(), pl
);
2757 t
.clone(cid
, hoid
, hoid2
);
2762 memset(a
.c_str(), 112, a
.length());
2766 t
.write(cid
, hoid
, pl
.length() + z
.length(), a
.length(), al
);
2767 r
= apply_transaction(store
, &osr
, std::move(t
));
2770 ASSERT_EQ((int)final
.length(),
2771 store
->read(cid
, hoid
, 0, final
.length(), rl
));
2772 ASSERT_TRUE(bl_eq(rl
, final
));
2775 ObjectStore::Transaction t
;
2776 t
.remove(cid
, hoid
);
2777 t
.remove(cid
, hoid2
);
2778 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2783 memset(p
.c_str(), 5, p
.length());
2787 ObjectStore::Transaction t
;
2788 t
.write(cid
, hoid
, 0, pl
.length(), pl
);
2789 t
.clone(cid
, hoid
, hoid2
);
2794 memset(a
.c_str(), 6, a
.length());
2798 t
.write(cid
, hoid
, 17000, a
.length(), al
);
2799 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2801 ASSERT_EQ((int)final
.length(),
2802 store
->read(cid
, hoid
, 0, final
.length(), rl
));
2803 /*cout << "expected:\n";
2804 final.hexdump(cout);
2807 ASSERT_TRUE(bl_eq(rl
, final
));
2810 ObjectStore::Transaction t
;
2811 t
.remove(cid
, hoid
);
2812 t
.remove(cid
, hoid2
);
2813 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2816 bufferptr
p(1048576);
2817 memset(p
.c_str(), 3, p
.length());
2820 ObjectStore::Transaction t
;
2821 t
.write(cid
, hoid
, 0, pl
.length(), pl
);
2822 t
.clone(cid
, hoid
, hoid2
);
2824 memset(a
.c_str(), 4, a
.length());
2827 t
.write(cid
, hoid
, a
.length(), a
.length(), al
);
2828 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2831 final
.substr_of(pl
, 0, al
.length());
2834 end
.substr_of(pl
, al
.length()*2, pl
.length() - al
.length()*2);
2836 ASSERT_EQ((int)final
.length(),
2837 store
->read(cid
, hoid
, 0, final
.length(), rl
));
2838 /*cout << "expected:\n";
2839 final.hexdump(cout);
2842 ASSERT_TRUE(bl_eq(rl
, final
));
2845 ObjectStore::Transaction t
;
2846 t
.remove(cid
, hoid
);
2847 t
.remove(cid
, hoid2
);
2848 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2852 memset(p
.c_str(), 7, p
.length());
2855 ObjectStore::Transaction t
;
2856 t
.write(cid
, hoid
, 0, pl
.length(), pl
);
2857 t
.clone(cid
, hoid
, hoid2
);
2859 memset(a
.c_str(), 8, a
.length());
2862 t
.write(cid
, hoid
, 32768, a
.length(), al
);
2863 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2866 final
.substr_of(pl
, 0, 32768);
2869 end
.substr_of(pl
, final
.length(), pl
.length() - final
.length());
2871 ASSERT_EQ((int)final
.length(),
2872 store
->read(cid
, hoid
, 0, final
.length(), rl
));
2873 /*cout << "expected:\n";
2874 final.hexdump(cout);
2877 ASSERT_TRUE(bl_eq(rl
, final
));
2880 ObjectStore::Transaction t
;
2881 t
.remove(cid
, hoid
);
2882 t
.remove(cid
, hoid2
);
2883 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2887 memset(p
.c_str(), 9, p
.length());
2890 ObjectStore::Transaction t
;
2891 t
.write(cid
, hoid
, 0, pl
.length(), pl
);
2892 t
.clone(cid
, hoid
, hoid2
);
2894 memset(a
.c_str(), 10, a
.length());
2897 t
.write(cid
, hoid
, 33768, a
.length(), al
);
2898 ASSERT_EQ(0, apply_transaction(store
, &osr
, std::move(t
)));
2901 final
.substr_of(pl
, 0, 33768);
2904 end
.substr_of(pl
, final
.length(), pl
.length() - final
.length());
2906 ASSERT_EQ((int)final
.length(),
2907 store
->read(cid
, hoid
, 0, final
.length(), rl
));
2908 /*cout << "expected:\n";
2909 final.hexdump(cout);
2912 ASSERT_TRUE(bl_eq(rl
, final
));
2915 //Unfortunately we need a workaround for filestore since EXPECT_DEATH
2916 // macro has potential issues when using /in multithread environments.
2917 //It works well for all stores but filestore for now.
2918 //A fix setting gtest_death_test_style = "threadsafe" doesn't help as well -
2919 // test app clone asserts on store folder presence.
2921 if (string(GetParam()) != "filestore") {
2922 //verify if non-empty collection is properly handled after store reload
2923 r
= store
->umount();
2928 ObjectStore::Transaction t
;
2929 t
.remove_collection(cid
);
2930 cerr
<< "Invalid rm coll" << std::endl
;
2931 PrCtl unset_dumpable
;
2932 EXPECT_DEATH(apply_transaction(store
, &osr
, std::move(t
)), "");
2935 ObjectStore::Transaction t
;
2936 t
.touch(cid
, hoid3
); //new record in db
2937 r
= apply_transaction(store
, &osr
, std::move(t
));
2940 //See comment above for "filestore" check explanation.
2941 if (string(GetParam()) != "filestore") {
2942 ObjectStore::Transaction t
;
2943 //verify if non-empty collection is properly handled when there are some pending removes and live records in db
2944 cerr
<< "Invalid rm coll again" << std::endl
;
2945 r
= store
->umount();
2950 t
.remove(cid
, hoid
);
2951 t
.remove(cid
, hoid2
);
2952 t
.remove_collection(cid
);
2953 PrCtl unset_dumpable
;
2954 EXPECT_DEATH(apply_transaction(store
, &osr
, std::move(t
)), "");
2957 ObjectStore::Transaction t
;
2958 t
.remove(cid
, hoid
);
2959 t
.remove(cid
, hoid2
);
2960 t
.remove(cid
, hoid3
);
2961 t
.remove_collection(cid
);
2962 cerr
<< "Cleaning" << std::endl
;
2963 r
= apply_transaction(store
, &osr
, std::move(t
));
2968 TEST_P(StoreTest
, OmapSimple
) {
2969 ObjectStore::Sequencer
osr("test");
2973 ObjectStore::Transaction t
;
2974 t
.create_collection(cid
, 0);
2975 cerr
<< "Creating collection " << cid
<< std::endl
;
2976 r
= apply_transaction(store
, &osr
, std::move(t
));
2979 ghobject_t
hoid(hobject_t(sobject_t("omap_obj", CEPH_NOSNAP
),
2980 "key", 123, -1, ""));
2982 small
.append("small");
2983 map
<string
,bufferlist
> km
;
2985 km
["bar"].append("asdfjkasdkjdfsjkafskjsfdj");
2987 header
.append("this is a header");
2989 ObjectStore::Transaction t
;
2991 t
.omap_setkeys(cid
, hoid
, km
);
2992 t
.omap_setheader(cid
, hoid
, header
);
2993 cerr
<< "Creating object and set omap " << hoid
<< std::endl
;
2994 r
= apply_transaction(store
, &osr
, std::move(t
));
3000 map
<string
,bufferlist
> r
;
3001 store
->omap_get(cid
, hoid
, &h
, &r
);
3002 ASSERT_TRUE(bl_eq(header
, h
));
3003 ASSERT_EQ(r
.size(), km
.size());
3004 cout
<< "r: " << r
<< std::endl
;
3006 // test iterator with seek_to_first
3008 map
<string
,bufferlist
> r
;
3009 ObjectMap::ObjectMapIterator iter
= store
->get_omap_iterator(cid
, hoid
);
3010 for (iter
->seek_to_first(); iter
->valid(); iter
->next(false)) {
3011 r
[iter
->key()] = iter
->value();
3013 cout
<< "r: " << r
<< std::endl
;
3014 ASSERT_EQ(r
.size(), km
.size());
3016 // test iterator with initial lower_bound
3018 map
<string
,bufferlist
> r
;
3019 ObjectMap::ObjectMapIterator iter
= store
->get_omap_iterator(cid
, hoid
);
3020 for (iter
->lower_bound(string()); iter
->valid(); iter
->next(false)) {
3021 r
[iter
->key()] = iter
->value();
3023 cout
<< "r: " << r
<< std::endl
;
3024 ASSERT_EQ(r
.size(), km
.size());
3027 ObjectStore::Transaction t
;
3028 t
.remove(cid
, hoid
);
3029 t
.remove_collection(cid
);
3030 cerr
<< "Cleaning" << std::endl
;
3031 r
= apply_transaction(store
, &osr
, std::move(t
));
3036 TEST_P(StoreTest
, OmapCloneTest
) {
3037 ObjectStore::Sequencer
osr("test");
3041 ObjectStore::Transaction t
;
3042 t
.create_collection(cid
, 0);
3043 cerr
<< "Creating collection " << cid
<< std::endl
;
3044 r
= apply_transaction(store
, &osr
, std::move(t
));
3047 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
),
3048 "key", 123, -1, ""));
3050 small
.append("small");
3051 map
<string
,bufferlist
> km
;
3053 km
["bar"].append("asdfjkasdkjdfsjkafskjsfdj");
3055 header
.append("this is a header");
3057 ObjectStore::Transaction t
;
3059 t
.omap_setkeys(cid
, hoid
, km
);
3060 t
.omap_setheader(cid
, hoid
, header
);
3061 cerr
<< "Creating object and set omap " << hoid
<< std::endl
;
3062 r
= apply_transaction(store
, &osr
, std::move(t
));
3065 ghobject_t
hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP
),
3066 "key", 123, -1, ""));
3068 ObjectStore::Transaction t
;
3069 t
.clone(cid
, hoid
, hoid2
);
3070 cerr
<< "Clone object" << std::endl
;
3071 r
= apply_transaction(store
, &osr
, std::move(t
));
3075 map
<string
,bufferlist
> r
;
3077 store
->omap_get(cid
, hoid2
, &h
, &r
);
3078 ASSERT_TRUE(bl_eq(header
, h
));
3079 ASSERT_EQ(r
.size(), km
.size());
3082 ObjectStore::Transaction t
;
3083 t
.remove(cid
, hoid
);
3084 t
.remove(cid
, hoid2
);
3085 t
.remove_collection(cid
);
3086 cerr
<< "Cleaning" << std::endl
;
3087 r
= apply_transaction(store
, &osr
, std::move(t
));
3092 TEST_P(StoreTest
, SimpleCloneRangeTest
) {
3093 ObjectStore::Sequencer
osr("test");
3097 ObjectStore::Transaction t
;
3098 t
.create_collection(cid
, 0);
3099 cerr
<< "Creating collection " << cid
<< std::endl
;
3100 r
= apply_transaction(store
, &osr
, std::move(t
));
3103 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
3104 hoid
.hobj
.pool
= -1;
3105 bufferlist small
, newdata
;
3106 small
.append("small");
3108 ObjectStore::Transaction t
;
3109 t
.write(cid
, hoid
, 10, 5, small
);
3110 cerr
<< "Creating object and write bl " << hoid
<< std::endl
;
3111 r
= apply_transaction(store
, &osr
, std::move(t
));
3114 ghobject_t
hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP
)));
3115 hoid2
.hobj
.pool
= -1;
3117 ObjectStore::Transaction t
;
3118 t
.clone_range(cid
, hoid
, hoid2
, 10, 5, 10);
3119 cerr
<< "Clone range object" << std::endl
;
3120 r
= apply_transaction(store
, &osr
, std::move(t
));
3122 r
= store
->read(cid
, hoid2
, 10, 5, newdata
);
3124 ASSERT_TRUE(bl_eq(small
, newdata
));
3127 ObjectStore::Transaction t
;
3128 t
.truncate(cid
, hoid
, 1024*1024);
3129 t
.clone_range(cid
, hoid
, hoid2
, 0, 1024*1024, 0);
3130 cerr
<< "Clone range object" << std::endl
;
3131 r
= apply_transaction(store
, &osr
, std::move(t
));
3133 struct stat stat
, stat2
;
3134 r
= store
->stat(cid
, hoid
, &stat
);
3135 r
= store
->stat(cid
, hoid2
, &stat2
);
3136 ASSERT_EQ(stat
.st_size
, stat2
.st_size
);
3137 ASSERT_EQ(1024*1024, stat2
.st_size
);
3140 ObjectStore::Transaction t
;
3141 t
.remove(cid
, hoid
);
3142 t
.remove(cid
, hoid2
);
3143 t
.remove_collection(cid
);
3144 cerr
<< "Cleaning" << std::endl
;
3145 r
= apply_transaction(store
, &osr
, std::move(t
));
3151 TEST_P(StoreTest
, SimpleObjectLongnameTest
) {
3152 ObjectStore::Sequencer
osr("test");
3156 ObjectStore::Transaction t
;
3157 t
.create_collection(cid
, 0);
3158 cerr
<< "Creating collection " << cid
<< std::endl
;
3159 r
= apply_transaction(store
, &osr
, std::move(t
));
3162 ghobject_t
hoid(hobject_t(sobject_t("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaObjectaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 1", CEPH_NOSNAP
)));
3164 ObjectStore::Transaction t
;
3166 cerr
<< "Creating object " << hoid
<< std::endl
;
3167 r
= apply_transaction(store
, &osr
, std::move(t
));
3171 ObjectStore::Transaction t
;
3172 t
.remove(cid
, hoid
);
3173 t
.remove_collection(cid
);
3174 cerr
<< "Cleaning" << std::endl
;
3175 r
= apply_transaction(store
, &osr
, std::move(t
));
3180 ghobject_t
generate_long_name(unsigned i
)
3183 name
<< "object id " << i
<< " ";
3184 for (unsigned j
= 0; j
< 500; ++j
) name
<< 'a';
3185 ghobject_t
hoid(hobject_t(sobject_t(name
.str(), CEPH_NOSNAP
)));
3186 hoid
.hobj
.set_hash(i
% 2);
3190 TEST_P(StoreTest
, LongnameSplitTest
) {
3191 ObjectStore::Sequencer
osr("test");
3195 ObjectStore::Transaction t
;
3196 t
.create_collection(cid
, 0);
3197 cerr
<< "Creating collection " << cid
<< std::endl
;
3198 r
= apply_transaction(store
, &osr
, std::move(t
));
3201 for (unsigned i
= 0; i
< 320; ++i
) {
3202 ObjectStore::Transaction t
;
3203 ghobject_t hoid
= generate_long_name(i
);
3205 cerr
<< "Creating object " << hoid
<< std::endl
;
3206 r
= apply_transaction(store
, &osr
, std::move(t
));
3210 ghobject_t test_obj
= generate_long_name(319);
3211 ghobject_t test_obj_2
= test_obj
;
3212 test_obj_2
.generation
= 0;
3214 ObjectStore::Transaction t
;
3215 // should cause a split
3216 t
.collection_move_rename(
3219 r
= apply_transaction(store
, &osr
, std::move(t
));
3223 for (unsigned i
= 0; i
< 319; ++i
) {
3224 ObjectStore::Transaction t
;
3225 ghobject_t hoid
= generate_long_name(i
);
3226 t
.remove(cid
, hoid
);
3227 cerr
<< "Removing object " << hoid
<< std::endl
;
3228 r
= apply_transaction(store
, &osr
, std::move(t
));
3232 ObjectStore::Transaction t
;
3233 t
.remove(cid
, test_obj_2
);
3234 t
.remove_collection(cid
);
3235 cerr
<< "Cleaning" << std::endl
;
3236 r
= apply_transaction(store
, &osr
, std::move(t
));
3242 TEST_P(StoreTest
, ManyObjectTest
) {
3243 ObjectStore::Sequencer
osr("test");
3244 int NUM_OBJS
= 2000;
3248 for (int i
= 0; i
< 100; ++i
) base
.append("aaaaa");
3249 set
<ghobject_t
> created
;
3251 ObjectStore::Transaction t
;
3252 t
.create_collection(cid
, 0);
3253 r
= apply_transaction(store
, &osr
, std::move(t
));
3256 for (int i
= 0; i
< NUM_OBJS
; ++i
) {
3258 cerr
<< "Object " << i
<< std::endl
;
3260 ObjectStore::Transaction t
;
3262 snprintf(buf
, sizeof(buf
), "%d", i
);
3263 ghobject_t
hoid(hobject_t(sobject_t(string(buf
) + base
, CEPH_NOSNAP
)));
3265 created
.insert(hoid
);
3266 r
= apply_transaction(store
, &osr
, std::move(t
));
3270 for (set
<ghobject_t
>::iterator i
= created
.begin();
3274 ASSERT_TRUE(!store
->stat(cid
, *i
, &buf
));
3277 set
<ghobject_t
> listed
, listed2
;
3278 vector
<ghobject_t
> objects
;
3279 r
= store
->collection_list(cid
, ghobject_t(), ghobject_t::get_max(), INT_MAX
, &objects
, 0);
3282 cerr
<< "objects.size() is " << objects
.size() << std::endl
;
3283 for (vector
<ghobject_t
> ::iterator i
= objects
.begin();
3287 ASSERT_TRUE(created
.count(*i
));
3289 ASSERT_TRUE(listed
.size() == created
.size());
3291 ghobject_t start
, next
;
3293 r
= store
->collection_list(
3295 ghobject_t::get_max(),
3296 ghobject_t::get_max(),
3302 ASSERT_TRUE(objects
.empty());
3306 ghobject_t start2
, next2
;
3308 r
= store
->collection_list(cid
, start
, ghobject_t::get_max(),
3312 ASSERT_TRUE(sorted(objects
));
3314 listed
.insert(objects
.begin(), objects
.end());
3315 if (objects
.size() < 50) {
3316 ASSERT_TRUE(next
.is_max());
3323 cerr
<< "listed.size() is " << listed
.size() << std::endl
;
3324 ASSERT_TRUE(listed
.size() == created
.size());
3325 if (listed2
.size()) {
3326 ASSERT_EQ(listed
.size(), listed2
.size());
3328 for (set
<ghobject_t
>::iterator i
= listed
.begin();
3331 ASSERT_TRUE(created
.count(*i
));
3334 for (set
<ghobject_t
>::iterator i
= created
.begin();
3337 ObjectStore::Transaction t
;
3339 r
= apply_transaction(store
, &osr
, std::move(t
));
3342 cerr
<< "cleaning up" << std::endl
;
3344 ObjectStore::Transaction t
;
3345 t
.remove_collection(cid
);
3346 r
= apply_transaction(store
, &osr
, std::move(t
));
3352 class ObjectGenerator
{
3354 virtual ghobject_t
create_object(gen_type
*gen
) = 0;
3355 virtual ~ObjectGenerator() {}
3358 class MixedGenerator
: public ObjectGenerator
{
3362 explicit MixedGenerator(int64_t p
) : seq(0), poolid(p
) {}
3363 ghobject_t
create_object(gen_type
*gen
) override
{
3365 snprintf(buf
, sizeof(buf
), "OBJ_%u", seq
);
3368 for (unsigned i
= 0; i
< 300; ++i
) {
3369 name
.push_back('a');
3375 name
, string(), rand() & 2 ? CEPH_NOSNAP
: rand(),
3376 (((seq
/ 1024) % 2) * 0xF00 ) +
3382 class SyntheticWorkloadState
{
3385 map
<string
, bufferlist
> attrs
;
3388 static const unsigned max_in_flight
= 16;
3389 static const unsigned max_objects
= 3000;
3390 static const unsigned max_attr_size
= 5;
3391 static const unsigned max_attr_name_len
= 100;
3392 static const unsigned max_attr_value_len
= 1024 * 64;
3394 unsigned write_alignment
;
3395 unsigned max_object_len
, max_write_len
;
3397 map
<ghobject_t
, Object
> contents
;
3398 set
<ghobject_t
> available_objects
;
3399 set
<ghobject_t
> in_flight_objects
;
3400 ObjectGenerator
*object_gen
;
3403 ObjectStore::Sequencer
*osr
;
3410 explicit EnterExit(const char *m
) : msg(m
) {
3411 //cout << pthread_self() << " enter " << msg << std::endl;
3414 //cout << pthread_self() << " exit " << msg << std::endl;
3418 class C_SyntheticOnReadable
: public Context
{
3420 SyntheticWorkloadState
*state
;
3422 C_SyntheticOnReadable(SyntheticWorkloadState
*state
, ghobject_t hoid
)
3423 : state(state
), hoid(hoid
) {}
3425 void finish(int r
) override
{
3426 Mutex::Locker
locker(state
->lock
);
3427 EnterExit
ee("onreadable finish");
3428 ASSERT_TRUE(state
->in_flight_objects
.count(hoid
));
3430 state
->in_flight_objects
.erase(hoid
);
3431 if (state
->contents
.count(hoid
))
3432 state
->available_objects
.insert(hoid
);
3433 --(state
->in_flight
);
3434 state
->cond
.Signal();
3437 r
= state
->store
->read(state
->cid
, hoid
, 0, state
->contents
[hoid
].data
.length(), r2
);
3438 assert(bl_eq(state
->contents
[hoid
].data
, r2
));
3439 state
->cond
.Signal();
3443 class C_SyntheticOnStash
: public Context
{
3445 SyntheticWorkloadState
*state
;
3446 ghobject_t oid
, noid
;
3448 C_SyntheticOnStash(SyntheticWorkloadState
*state
,
3449 ghobject_t oid
, ghobject_t noid
)
3450 : state(state
), oid(oid
), noid(noid
) {}
3452 void finish(int r
) override
{
3453 Mutex::Locker
locker(state
->lock
);
3454 EnterExit
ee("stash finish");
3455 ASSERT_TRUE(state
->in_flight_objects
.count(oid
));
3457 state
->in_flight_objects
.erase(oid
);
3458 if (state
->contents
.count(noid
))
3459 state
->available_objects
.insert(noid
);
3460 --(state
->in_flight
);
3462 r
= state
->store
->read(
3463 state
->cid
, noid
, 0,
3464 state
->contents
[noid
].data
.length(), r2
);
3465 assert(bl_eq(state
->contents
[noid
].data
, r2
));
3466 state
->cond
.Signal();
3470 class C_SyntheticOnClone
: public Context
{
3472 SyntheticWorkloadState
*state
;
3473 ghobject_t oid
, noid
;
3475 C_SyntheticOnClone(SyntheticWorkloadState
*state
,
3476 ghobject_t oid
, ghobject_t noid
)
3477 : state(state
), oid(oid
), noid(noid
) {}
3479 void finish(int r
) override
{
3480 Mutex::Locker
locker(state
->lock
);
3481 EnterExit
ee("clone finish");
3482 ASSERT_TRUE(state
->in_flight_objects
.count(oid
));
3484 state
->in_flight_objects
.erase(oid
);
3485 if (state
->contents
.count(oid
))
3486 state
->available_objects
.insert(oid
);
3487 if (state
->contents
.count(noid
))
3488 state
->available_objects
.insert(noid
);
3489 --(state
->in_flight
);
3491 r
= state
->store
->read(state
->cid
, noid
, 0, state
->contents
[noid
].data
.length(), r2
);
3492 assert(bl_eq(state
->contents
[noid
].data
, r2
));
3493 state
->cond
.Signal();
3497 static void filled_byte_array(bufferlist
& bl
, size_t size
)
3499 static const char alphanum
[] = "0123456789"
3500 "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
3501 "abcdefghijklmnopqrstuvwxyz";
3506 for (unsigned int i
= 0; i
< size
- 1; i
++) {
3507 // severely limit entropy so we can compress...
3508 bp
[i
] = alphanum
[rand() % 10]; //(sizeof(alphanum) - 1)];
3510 bp
[size
- 1] = '\0';
3515 SyntheticWorkloadState(ObjectStore
*store
,
3516 ObjectGenerator
*gen
,
3518 ObjectStore::Sequencer
*osr
,
3523 : cid(cid
), write_alignment(alignment
), max_object_len(max_size
),
3524 max_write_len(max_write
), in_flight(0), object_gen(gen
),
3525 rng(rng
), store(store
), osr(osr
), lock("State lock") {}
3528 ObjectStore::Transaction t
;
3529 t
.create_collection(cid
, 0);
3530 return apply_transaction(store
, osr
, std::move(t
));
3534 vector
<ghobject_t
> objects
;
3535 int r
= store
->collection_list(cid
, ghobject_t(), ghobject_t::get_max(),
3538 if (objects
.empty())
3540 ObjectStore::Transaction t
;
3541 for (vector
<ghobject_t
>::iterator p
= objects
.begin();
3542 p
!= objects
.end(); ++p
) {
3545 apply_transaction(store
, osr
, std::move(t
));
3547 ObjectStore::Transaction t
;
3548 t
.remove_collection(cid
);
3549 apply_transaction(store
, osr
, std::move(t
));
3551 void statfs(store_statfs_t
& stat
) {
3552 store
->statfs(&stat
);
3555 ghobject_t
get_uniform_random_object() {
3556 while (in_flight
>= max_in_flight
|| available_objects
.empty())
3558 boost::uniform_int
<> choose(0, available_objects
.size() - 1);
3559 int index
= choose(*rng
);
3560 set
<ghobject_t
>::iterator i
= available_objects
.begin();
3561 for ( ; index
> 0; --index
, ++i
) ;
3562 ghobject_t ret
= *i
;
3566 void wait_for_ready() {
3567 while (in_flight
>= max_in_flight
)
3571 void wait_for_done() {
3573 Mutex::Locker
locker(lock
);
3579 return (available_objects
.size() + in_flight_objects
.size()) < max_objects
;
3583 return (available_objects
.size() + in_flight_objects
.size()) > 0;
3586 unsigned get_random_alloc_hints() {
3589 boost::uniform_int
<> u(0, 3);
3592 f
|= CEPH_OSD_ALLOC_HINT_FLAG_SEQUENTIAL_WRITE
;
3595 f
|= CEPH_OSD_ALLOC_HINT_FLAG_RANDOM_WRITE
;
3600 boost::uniform_int
<> u(0, 3);
3603 f
|= CEPH_OSD_ALLOC_HINT_FLAG_SEQUENTIAL_READ
;
3606 f
|= CEPH_OSD_ALLOC_HINT_FLAG_RANDOM_READ
;
3611 // append_only, immutable
3612 boost::uniform_int
<> u(0, 4);
3616 boost::uniform_int
<> u(0, 3);
3619 f
|= CEPH_OSD_ALLOC_HINT_FLAG_SHORTLIVED
;
3622 f
|= CEPH_OSD_ALLOC_HINT_FLAG_LONGLIVED
;
3627 boost::uniform_int
<> u(0, 3);
3630 f
|= CEPH_OSD_ALLOC_HINT_FLAG_COMPRESSIBLE
;
3633 f
|= CEPH_OSD_ALLOC_HINT_FLAG_INCOMPRESSIBLE
;
3641 Mutex::Locker
locker(lock
);
3642 EnterExit
ee("touch");
3646 ghobject_t new_obj
= object_gen
->create_object(rng
);
3647 available_objects
.erase(new_obj
);
3648 ObjectStore::Transaction t
;
3649 t
.touch(cid
, new_obj
);
3650 boost::uniform_int
<> u(17, 22);
3651 boost::uniform_int
<> v(12, 17);
3652 t
.set_alloc_hint(cid
, new_obj
,
3655 get_random_alloc_hints());
3657 in_flight_objects
.insert(new_obj
);
3658 if (!contents
.count(new_obj
))
3659 contents
[new_obj
] = Object();
3660 int status
= store
->queue_transaction(osr
, std::move(t
), new C_SyntheticOnReadable(this, new_obj
));
3665 Mutex::Locker
locker(lock
);
3666 EnterExit
ee("stash");
3676 old_obj
= get_uniform_random_object();
3677 } while (--max
&& !contents
[old_obj
].data
.length());
3678 available_objects
.erase(old_obj
);
3679 ghobject_t new_obj
= old_obj
;
3680 new_obj
.generation
++;
3681 available_objects
.erase(new_obj
);
3683 ObjectStore::Transaction t
;
3684 t
.collection_move_rename(cid
, old_obj
, cid
, new_obj
);
3686 in_flight_objects
.insert(old_obj
);
3688 contents
[new_obj
].attrs
= contents
[old_obj
].attrs
;
3689 contents
[new_obj
].data
= contents
[old_obj
].data
;
3690 contents
.erase(old_obj
);
3691 int status
= store
->queue_transaction(
3693 new C_SyntheticOnStash(this, old_obj
, new_obj
));
3698 Mutex::Locker
locker(lock
);
3699 EnterExit
ee("clone");
3709 old_obj
= get_uniform_random_object();
3710 } while (--max
&& !contents
[old_obj
].data
.length());
3711 available_objects
.erase(old_obj
);
3712 ghobject_t new_obj
= object_gen
->create_object(rng
);
3713 // make the hash match
3714 new_obj
.hobj
.set_hash(old_obj
.hobj
.get_hash());
3715 available_objects
.erase(new_obj
);
3717 ObjectStore::Transaction t
;
3718 t
.clone(cid
, old_obj
, new_obj
);
3720 in_flight_objects
.insert(old_obj
);
3722 contents
[new_obj
].attrs
= contents
[old_obj
].attrs
;
3723 contents
[new_obj
].data
= contents
[old_obj
].data
;
3725 int status
= store
->queue_transaction(
3727 new C_SyntheticOnClone(this, old_obj
, new_obj
));
3732 Mutex::Locker
locker(lock
);
3733 EnterExit
ee("clone_range");
3743 old_obj
= get_uniform_random_object();
3744 } while (--max
&& !contents
[old_obj
].data
.length());
3745 bufferlist
&srcdata
= contents
[old_obj
].data
;
3746 if (srcdata
.length() == 0) {
3749 available_objects
.erase(old_obj
);
3750 ghobject_t new_obj
= get_uniform_random_object();
3751 available_objects
.erase(new_obj
);
3753 boost::uniform_int
<> u1(0, max_object_len
- max_write_len
);
3754 boost::uniform_int
<> u2(0, max_write_len
);
3755 uint64_t srcoff
= u1(*rng
);
3756 // make src and dst offsets match, since that's what the osd does
3757 uint64_t dstoff
= srcoff
; //u1(*rng);
3758 uint64_t len
= u2(*rng
);
3759 if (write_alignment
) {
3760 srcoff
= ROUND_UP_TO(srcoff
, write_alignment
);
3761 dstoff
= ROUND_UP_TO(dstoff
, write_alignment
);
3762 len
= ROUND_UP_TO(len
, write_alignment
);
3765 if (srcoff
> srcdata
.length() - 1) {
3766 srcoff
= srcdata
.length() - 1;
3768 if (srcoff
+ len
> srcdata
.length()) {
3769 len
= srcdata
.length() - srcoff
;
3772 cout
<< __func__
<< " from " << srcoff
<< "~" << len
3773 << " (size " << srcdata
.length() << ") to "
3774 << dstoff
<< "~" << len
<< std::endl
;
3776 ObjectStore::Transaction t
;
3777 t
.clone_range(cid
, old_obj
, new_obj
, srcoff
, len
, dstoff
);
3779 in_flight_objects
.insert(old_obj
);
3782 if (srcoff
< srcdata
.length()) {
3783 if (srcoff
+ len
> srcdata
.length()) {
3784 bl
.substr_of(srcdata
, srcoff
, srcdata
.length() - srcoff
);
3786 bl
.substr_of(srcdata
, srcoff
, len
);
3790 bufferlist
& dstdata
= contents
[new_obj
].data
;
3791 if (dstdata
.length() <= dstoff
) {
3792 if (bl
.length() > 0) {
3793 dstdata
.append_zero(dstoff
- dstdata
.length());
3798 assert(dstdata
.length() > dstoff
);
3799 dstdata
.copy(0, dstoff
, value
);
3801 if (value
.length() < dstdata
.length())
3802 dstdata
.copy(value
.length(),
3803 dstdata
.length() - value
.length(), value
);
3804 value
.swap(dstdata
);
3807 int status
= store
->queue_transaction(
3808 osr
, std::move(t
), new C_SyntheticOnClone(this, old_obj
, new_obj
));
3814 Mutex::Locker
locker(lock
);
3815 EnterExit
ee("write");
3820 ghobject_t new_obj
= get_uniform_random_object();
3821 available_objects
.erase(new_obj
);
3822 ObjectStore::Transaction t
;
3824 boost::uniform_int
<> u1(0, max_object_len
- max_write_len
);
3825 boost::uniform_int
<> u2(0, max_write_len
);
3826 uint64_t offset
= u1(*rng
);
3827 uint64_t len
= u2(*rng
);
3829 if (write_alignment
) {
3830 offset
= ROUND_UP_TO(offset
, write_alignment
);
3831 len
= ROUND_UP_TO(len
, write_alignment
);
3834 filled_byte_array(bl
, len
);
3836 bufferlist
& data
= contents
[new_obj
].data
;
3837 if (data
.length() <= offset
) {
3839 data
.append_zero(offset
-data
.length());
3844 assert(data
.length() > offset
);
3845 data
.copy(0, offset
, value
);
3847 if (value
.length() < data
.length())
3848 data
.copy(value
.length(),
3849 data
.length()-value
.length(), value
);
3853 t
.write(cid
, new_obj
, offset
, len
, bl
);
3855 in_flight_objects
.insert(new_obj
);
3856 int status
= store
->queue_transaction(
3857 osr
, std::move(t
), new C_SyntheticOnReadable(this, new_obj
));
3862 Mutex::Locker
locker(lock
);
3863 EnterExit
ee("truncate");
3868 ghobject_t obj
= get_uniform_random_object();
3869 available_objects
.erase(obj
);
3870 ObjectStore::Transaction t
;
3872 boost::uniform_int
<> choose(0, max_object_len
);
3873 size_t len
= choose(*rng
);
3874 if (write_alignment
) {
3875 len
= ROUND_UP_TO(len
, write_alignment
);
3878 t
.truncate(cid
, obj
, len
);
3880 in_flight_objects
.insert(obj
);
3881 bufferlist
& data
= contents
[obj
].data
;
3882 if (data
.length() <= len
) {
3883 data
.append_zero(len
- data
.length());
3886 data
.copy(0, len
, bl
);
3890 int status
= store
->queue_transaction(
3891 osr
, std::move(t
), new C_SyntheticOnReadable(this, obj
));
3896 Mutex::Locker
locker(lock
);
3897 EnterExit
ee("zero");
3902 ghobject_t new_obj
= get_uniform_random_object();
3903 available_objects
.erase(new_obj
);
3904 ObjectStore::Transaction t
;
3906 boost::uniform_int
<> u1(0, max_object_len
- max_write_len
);
3907 boost::uniform_int
<> u2(0, max_write_len
);
3908 uint64_t offset
= u1(*rng
);
3909 uint64_t len
= u2(*rng
);
3910 if (write_alignment
) {
3911 offset
= ROUND_UP_TO(offset
, write_alignment
);
3912 len
= ROUND_UP_TO(len
, write_alignment
);
3916 auto& data
= contents
[new_obj
].data
;
3917 if (data
.length() < offset
+ len
) {
3918 data
.append_zero(offset
+len
-data
.length());
3921 n
.substr_of(data
, 0, offset
);
3923 if (data
.length() > offset
+ len
)
3924 data
.copy(offset
+ len
, data
.length() - offset
- len
, n
);
3928 t
.zero(cid
, new_obj
, offset
, len
);
3930 in_flight_objects
.insert(new_obj
);
3931 int status
= store
->queue_transaction(
3932 osr
, std::move(t
), new C_SyntheticOnReadable(this, new_obj
));
3937 EnterExit
ee("read");
3938 boost::uniform_int
<> u1(0, max_object_len
/2);
3939 boost::uniform_int
<> u2(0, max_object_len
);
3940 uint64_t offset
= u1(*rng
);
3941 uint64_t len
= u2(*rng
);
3946 bufferlist expected
;
3949 Mutex::Locker
locker(lock
);
3950 EnterExit
ee("read locked");
3955 obj
= get_uniform_random_object();
3956 expected
= contents
[obj
].data
;
3958 bufferlist bl
, result
;
3959 if (0) cout
<< " obj " << obj
3960 << " size " << expected
.length()
3961 << " offset " << offset
3962 << " len " << len
<< std::endl
;
3963 r
= store
->read(cid
, obj
, offset
, len
, result
);
3964 if (offset
>= expected
.length()) {
3967 size_t max_len
= expected
.length() - offset
;
3970 assert(len
== result
.length());
3971 ASSERT_EQ(len
, result
.length());
3972 expected
.copy(offset
, len
, bl
);
3973 ASSERT_EQ(r
, (int)len
);
3974 ASSERT_TRUE(bl_eq(bl
, result
));
3979 Mutex::Locker
locker(lock
);
3980 EnterExit
ee("setattrs");
3985 ghobject_t obj
= get_uniform_random_object();
3986 available_objects
.erase(obj
);
3987 ObjectStore::Transaction t
;
3989 boost::uniform_int
<> u0(1, max_attr_size
);
3990 boost::uniform_int
<> u1(4, max_attr_name_len
);
3991 boost::uniform_int
<> u2(4, max_attr_value_len
);
3992 boost::uniform_int
<> u3(0, 100);
3993 uint64_t size
= u0(*rng
);
3995 map
<string
, bufferlist
> attrs
;
3997 for (map
<string
, bufferlist
>::iterator it
= contents
[obj
].attrs
.begin();
3998 it
!= contents
[obj
].attrs
.end(); ++it
)
3999 keys
.insert(it
->first
);
4002 bufferlist name
, value
;
4003 uint64_t get_exist
= u3(*rng
);
4004 uint64_t value_len
= u2(*rng
);
4005 filled_byte_array(value
, value_len
);
4006 if (get_exist
< 50 && keys
.size()) {
4007 set
<string
>::iterator k
= keys
.begin();
4009 contents
[obj
].attrs
[*k
] = value
;
4012 name_len
= u1(*rng
);
4013 filled_byte_array(name
, name_len
);
4014 attrs
[name
.c_str()] = value
;
4015 contents
[obj
].attrs
[name
.c_str()] = value
;
4018 t
.setattrs(cid
, obj
, attrs
);
4020 in_flight_objects
.insert(obj
);
4021 int status
= store
->queue_transaction(
4022 osr
, std::move(t
), new C_SyntheticOnReadable(this, obj
));
4027 EnterExit
ee("getattrs");
4029 map
<string
, bufferlist
> expected
;
4031 Mutex::Locker
locker(lock
);
4032 EnterExit
ee("getattrs locked");
4039 obj
= get_uniform_random_object();
4042 } while (contents
[obj
].attrs
.empty());
4043 expected
= contents
[obj
].attrs
;
4045 map
<string
, bufferlist
> attrs
;
4046 int r
= store
->getattrs(cid
, obj
, attrs
);
4047 ASSERT_TRUE(r
== 0);
4048 ASSERT_TRUE(attrs
.size() == expected
.size());
4049 for (map
<string
, bufferlist
>::iterator it
= expected
.begin();
4050 it
!= expected
.end(); ++it
) {
4051 ASSERT_TRUE(bl_eq(attrs
[it
->first
], it
->second
));
4056 EnterExit
ee("getattr");
4060 map
<string
, bufferlist
> expected
;
4062 Mutex::Locker
locker(lock
);
4063 EnterExit
ee("getattr locked");
4070 obj
= get_uniform_random_object();
4073 } while (contents
[obj
].attrs
.empty());
4074 expected
= contents
[obj
].attrs
;
4076 boost::uniform_int
<> u(0, expected
.size()-1);
4078 map
<string
, bufferlist
>::iterator it
= expected
.begin();
4085 r
= store
->getattr(cid
, obj
, it
->first
, bl
);
4087 ASSERT_TRUE(bl_eq(it
->second
, bl
));
4091 Mutex::Locker
locker(lock
);
4092 EnterExit
ee("rmattr");
4100 obj
= get_uniform_random_object();
4103 } while (contents
[obj
].attrs
.empty());
4105 boost::uniform_int
<> u(0, contents
[obj
].attrs
.size()-1);
4107 map
<string
, bufferlist
>::iterator it
= contents
[obj
].attrs
.begin();
4113 available_objects
.erase(obj
);
4114 ObjectStore::Transaction t
;
4115 t
.rmattr(cid
, obj
, it
->first
);
4117 contents
[obj
].attrs
.erase(it
->first
);
4119 in_flight_objects
.insert(obj
);
4120 int status
= store
->queue_transaction(
4121 osr
, std::move(t
), new C_SyntheticOnReadable(this, obj
));
4125 void fsck(bool deep
) {
4126 Mutex::Locker
locker(lock
);
4127 EnterExit
ee("fsck");
4131 int r
= store
->fsck(deep
);
4132 assert(r
== 0 || r
== -EOPNOTSUPP
);
4137 Mutex::Locker
locker(lock
);
4138 EnterExit
ee("scan");
4141 vector
<ghobject_t
> objects
;
4142 set
<ghobject_t
> objects_set
, objects_set2
;
4143 ghobject_t next
, current
;
4145 //cerr << "scanning..." << std::endl;
4146 int r
= store
->collection_list(cid
, current
, ghobject_t::get_max(), 100,
4149 ASSERT_TRUE(sorted(objects
));
4150 objects_set
.insert(objects
.begin(), objects
.end());
4152 if (next
.is_max()) break;
4155 if (objects_set
.size() != available_objects
.size()) {
4156 for (set
<ghobject_t
>::iterator p
= objects_set
.begin();
4157 p
!= objects_set
.end();
4159 if (available_objects
.count(*p
) == 0) {
4160 cerr
<< "+ " << *p
<< std::endl
;
4163 for (set
<ghobject_t
>::iterator p
= available_objects
.begin();
4164 p
!= available_objects
.end();
4166 if (objects_set
.count(*p
) == 0)
4167 cerr
<< "- " << *p
<< std::endl
;
4168 //cerr << " objects_set: " << objects_set << std::endl;
4169 //cerr << " available_set: " << available_objects << std::endl;
4170 assert(0 == "badness");
4173 ASSERT_EQ(objects_set
.size(), available_objects
.size());
4174 for (set
<ghobject_t
>::iterator i
= objects_set
.begin();
4175 i
!= objects_set
.end();
4177 ASSERT_GT(available_objects
.count(*i
), (unsigned)0);
4180 int r
= store
->collection_list(cid
, ghobject_t(), ghobject_t::get_max(),
4181 INT_MAX
, &objects
, 0);
4183 objects_set2
.insert(objects
.begin(), objects
.end());
4184 ASSERT_EQ(objects_set2
.size(), available_objects
.size());
4185 for (set
<ghobject_t
>::iterator i
= objects_set2
.begin();
4186 i
!= objects_set2
.end();
4188 ASSERT_GT(available_objects
.count(*i
), (unsigned)0);
4189 if (available_objects
.count(*i
) == 0) {
4190 cerr
<< "+ " << *i
<< std::endl
;
4196 EnterExit
ee("stat");
4200 Mutex::Locker
locker(lock
);
4201 EnterExit
ee("stat lock1");
4204 hoid
= get_uniform_random_object();
4205 in_flight_objects
.insert(hoid
);
4206 available_objects
.erase(hoid
);
4208 expected
= contents
[hoid
].data
.length();
4211 int r
= store
->stat(cid
, hoid
, &buf
);
4213 assert((uint64_t)buf
.st_size
== expected
);
4214 ASSERT_TRUE((uint64_t)buf
.st_size
== expected
);
4216 Mutex::Locker
locker(lock
);
4217 EnterExit
ee("stat lock2");
4220 in_flight_objects
.erase(hoid
);
4221 available_objects
.insert(hoid
);
4226 Mutex::Locker
locker(lock
);
4227 EnterExit
ee("unlink");
4230 ghobject_t to_remove
= get_uniform_random_object();
4231 ObjectStore::Transaction t
;
4232 t
.remove(cid
, to_remove
);
4234 available_objects
.erase(to_remove
);
4235 in_flight_objects
.insert(to_remove
);
4236 contents
.erase(to_remove
);
4237 int status
= store
->queue_transaction(osr
, std::move(t
), new C_SyntheticOnReadable(this, to_remove
));
4241 void print_internal_state() {
4242 Mutex::Locker
locker(lock
);
4243 cerr
<< "available_objects: " << available_objects
.size()
4244 << " in_flight_objects: " << in_flight_objects
.size()
4245 << " total objects: " << in_flight_objects
.size() + available_objects
.size()
4246 << " in_flight " << in_flight
<< std::endl
;
4251 void doSyntheticTest(boost::scoped_ptr
<ObjectStore
>& store
,
4253 uint64_t max_obj
, uint64_t max_wr
, uint64_t align
)
4255 ObjectStore::Sequencer
osr("test");
4256 MixedGenerator
gen(555);
4257 gen_type
rng(time(NULL
));
4258 coll_t
cid(spg_t(pg_t(0,555), shard_id_t::NO_SHARD
));
4260 g_ceph_context
->_conf
->set_val("bluestore_fsck_on_mount", "false");
4261 g_ceph_context
->_conf
->set_val("bluestore_fsck_on_umount", "false");
4262 g_ceph_context
->_conf
->apply_changes(NULL
);
4264 SyntheticWorkloadState
test_obj(store
.get(), &gen
, &rng
, &osr
, cid
,
4265 max_obj
, max_wr
, align
);
4267 for (int i
= 0; i
< num_ops
/10; ++i
) {
4268 if (!(i
% 500)) cerr
<< "seeding object " << i
<< std::endl
;
4271 for (int i
= 0; i
< num_ops
; ++i
) {
4273 cerr
<< "Op " << i
<< std::endl
;
4274 test_obj
.print_internal_state();
4276 boost::uniform_int
<> true_false(0, 999);
4277 int val
= true_false(rng
);
4279 test_obj
.fsck(true);
4280 } else if (val
> 997) {
4281 test_obj
.fsck(false);
4282 } else if (val
> 970) {
4284 } else if (val
> 950) {
4286 } else if (val
> 850) {
4288 } else if (val
> 800) {
4290 } else if (val
> 550) {
4292 } else if (val
> 500) {
4294 } else if (val
> 450) {
4295 test_obj
.clone_range();
4296 } else if (val
> 300) {
4298 } else if (val
> 100) {
4301 test_obj
.truncate();
4304 test_obj
.wait_for_done();
4305 test_obj
.shutdown();
4307 g_ceph_context
->_conf
->set_val("bluestore_fsck_on_mount", "true");
4308 g_ceph_context
->_conf
->set_val("bluestore_fsck_on_umount", "true");
4309 g_ceph_context
->_conf
->apply_changes(NULL
);
4312 TEST_P(StoreTest
, Synthetic
) {
4313 doSyntheticTest(store
, 10000, 400*1024, 40*1024, 0);
4317 TEST_P(StoreTestSpecificAUSize
, SyntheticMatrixSharding
) {
4318 if (string(GetParam()) != "bluestore")
4321 const char *m
[][10] = {
4322 { "bluestore_min_alloc_size", "4096", 0 }, // must be the first!
4323 { "num_ops", "50000", 0 },
4324 { "max_write", "65536", 0 },
4325 { "max_size", "262144", 0 },
4326 { "alignment", "4096", 0 },
4327 { "bluestore_max_blob_size", "65536", 0 },
4328 { "bluestore_extent_map_shard_min_size", "60", 0 },
4329 { "bluestore_extent_map_shard_max_size", "300", 0 },
4330 { "bluestore_extent_map_shard_target_size", "150", 0 },
4331 { "bluestore_default_buffered_read", "true", 0 },
4332 { "bluestore_default_buffered_write", "true", 0 },
4335 do_matrix(m
, store
, doSyntheticTest
);
4338 TEST_P(StoreTestSpecificAUSize
, ZipperPatternSharded
) {
4339 if(string(GetParam()) != "bluestore")
4341 StartDeferred(4096);
4344 ObjectStore::Sequencer
osr("test");
4346 ghobject_t
a(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
4348 ObjectStore::Transaction t
;
4349 t
.create_collection(cid
, 0);
4350 cerr
<< "Creating collection " << cid
<< std::endl
;
4351 r
= apply_transaction(store
, &osr
, std::move(t
));
4359 for (int i
=0; i
<1000; ++i
) {
4360 ObjectStore::Transaction t
;
4361 t
.write(cid
, a
, i
*2*len
, len
, bl
, 0);
4362 r
= apply_transaction(store
, &osr
, std::move(t
));
4365 for (int i
=0; i
<1000; ++i
) {
4366 ObjectStore::Transaction t
;
4367 t
.write(cid
, a
, i
*2*len
+ 1, len
, bl
, 0);
4368 r
= apply_transaction(store
, &osr
, std::move(t
));
4372 ObjectStore::Transaction t
;
4374 t
.remove_collection(cid
);
4375 cerr
<< "Cleaning" << std::endl
;
4376 r
= apply_transaction(store
, &osr
, std::move(t
));
4381 TEST_P(StoreTestSpecificAUSize
, SyntheticMatrixCsumAlgorithm
) {
4382 if (string(GetParam()) != "bluestore")
4385 const char *m
[][10] = {
4386 { "bluestore_min_alloc_size", "65536", 0 }, // must be the first!
4387 { "max_write", "65536", 0 },
4388 { "max_size", "1048576", 0 },
4389 { "alignment", "16", 0 },
4390 { "bluestore_csum_type", "crc32c", "crc32c_16", "crc32c_8", "xxhash32",
4391 "xxhash64", "none", 0 },
4392 { "bluestore_default_buffered_write", "false", 0 },
4395 do_matrix(m
, store
, doSyntheticTest
);
4398 TEST_P(StoreTestSpecificAUSize
, SyntheticMatrixCsumVsCompression
) {
4399 if (string(GetParam()) != "bluestore")
4402 const char *m
[][10] = {
4403 { "bluestore_min_alloc_size", "4096", "16384", 0 }, //to be the first!
4404 { "max_write", "131072", 0 },
4405 { "max_size", "262144", 0 },
4406 { "alignment", "512", 0 },
4407 { "bluestore_compression_mode", "force", 0},
4408 { "bluestore_compression_algorithm", "snappy", "zlib", 0 },
4409 { "bluestore_csum_type", "crc32c", 0 },
4410 { "bluestore_default_buffered_read", "true", "false", 0 },
4411 { "bluestore_default_buffered_write", "true", "false", 0 },
4412 { "bluestore_sync_submit_transaction", "false", 0 },
4415 do_matrix(m
, store
, doSyntheticTest
);
4418 TEST_P(StoreTestSpecificAUSize
, SyntheticMatrixCompression
) {
4419 if (string(GetParam()) != "bluestore")
4422 const char *m
[][10] = {
4423 { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first!
4424 { "max_write", "1048576", 0 },
4425 { "max_size", "4194304", 0 },
4426 { "alignment", "65536", 0 },
4427 { "bluestore_compression_mode", "force", "aggressive", "passive", "none", 0},
4428 { "bluestore_default_buffered_write", "false", 0 },
4429 { "bluestore_sync_submit_transaction", "true", 0 },
4432 do_matrix(m
, store
, doSyntheticTest
);
4435 TEST_P(StoreTestSpecificAUSize
, SyntheticMatrixCompressionAlgorithm
) {
4436 if (string(GetParam()) != "bluestore")
4439 const char *m
[][10] = {
4440 { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first!
4441 { "max_write", "1048576", 0 },
4442 { "max_size", "4194304", 0 },
4443 { "alignment", "65536", 0 },
4444 { "bluestore_compression_algorithm", "zlib", "snappy", 0 },
4445 { "bluestore_compression_mode", "force", 0 },
4446 { "bluestore_default_buffered_write", "false", 0 },
4449 do_matrix(m
, store
, doSyntheticTest
);
4452 TEST_P(StoreTestSpecificAUSize
, SyntheticMatrixNoCsum
) {
4453 if (string(GetParam()) != "bluestore")
4456 const char *m
[][10] = {
4457 { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first!
4458 { "max_write", "65536", 0 },
4459 { "max_size", "1048576", 0 },
4460 { "alignment", "512", 0 },
4461 { "bluestore_max_blob_size", "262144", 0 },
4462 { "bluestore_compression_mode", "force", "none", 0},
4463 { "bluestore_csum_type", "none", 0},
4464 { "bluestore_default_buffered_read", "true", "false", 0 },
4465 { "bluestore_default_buffered_write", "true", 0 },
4466 { "bluestore_sync_submit_transaction", "true", "false", 0 },
4469 do_matrix(m
, store
, doSyntheticTest
);
4472 TEST_P(StoreTestSpecificAUSize
, SyntheticMatrixPreferDeferred
) {
4473 if (string(GetParam()) != "bluestore")
4476 const char *m
[][10] = {
4477 { "bluestore_min_alloc_size", "4096", "65536", 0 }, // to be the first!
4478 { "max_write", "65536", 0 },
4479 { "max_size", "1048576", 0 },
4480 { "alignment", "512", 0 },
4481 { "bluestore_max_blob_size", "262144", 0 },
4482 { "bluestore_compression_mode", "force", "none", 0},
4483 { "bluestore_prefer_deferred_size", "32768", "0", 0},
4486 do_matrix(m
, store
, doSyntheticTest
);
4489 TEST_P(StoreTest
, AttrSynthetic
) {
4490 ObjectStore::Sequencer
osr("test");
4491 MixedGenerator
gen(447);
4492 gen_type
rng(time(NULL
));
4493 coll_t
cid(spg_t(pg_t(0,447),shard_id_t::NO_SHARD
));
4495 SyntheticWorkloadState
test_obj(store
.get(), &gen
, &rng
, &osr
, cid
, 40*1024, 4*1024, 0);
4497 for (int i
= 0; i
< 500; ++i
) {
4498 if (!(i
% 10)) cerr
<< "seeding object " << i
<< std::endl
;
4501 for (int i
= 0; i
< 1000; ++i
) {
4503 cerr
<< "Op " << i
<< std::endl
;
4504 test_obj
.print_internal_state();
4506 boost::uniform_int
<> true_false(0, 99);
4507 int val
= true_false(rng
);
4510 } else if (val
> 93) {
4512 } else if (val
> 75) {
4514 } else if (val
> 47) {
4515 test_obj
.setattrs();
4516 } else if (val
> 45) {
4518 } else if (val
> 37) {
4520 } else if (val
> 30) {
4521 test_obj
.getattrs();
4526 test_obj
.wait_for_done();
4527 test_obj
.shutdown();
4530 TEST_P(StoreTest
, HashCollisionTest
) {
4531 ObjectStore::Sequencer
osr("test");
4532 int64_t poolid
= 11;
4533 coll_t
cid(spg_t(pg_t(0,poolid
),shard_id_t::NO_SHARD
));
4536 ObjectStore::Transaction t
;
4537 t
.create_collection(cid
, 0);
4538 r
= apply_transaction(store
, &osr
, std::move(t
));
4542 for (int i
= 0; i
< 100; ++i
) base
.append("aaaaa");
4543 set
<ghobject_t
> created
;
4544 for (int n
= 0; n
< 10; ++n
) {
4546 sprintf(nbuf
, "n%d", n
);
4547 for (int i
= 0; i
< 1000; ++i
) {
4549 sprintf(buf
, "%d", i
);
4551 cerr
<< "Object n" << n
<< " "<< i
<< std::endl
;
4553 ghobject_t
hoid(hobject_t(string(buf
) + base
, string(), CEPH_NOSNAP
, 0, poolid
, string(nbuf
)));
4555 ObjectStore::Transaction t
;
4557 r
= apply_transaction(store
, &osr
, std::move(t
));
4560 created
.insert(hoid
);
4563 vector
<ghobject_t
> objects
;
4564 r
= store
->collection_list(cid
, ghobject_t(), ghobject_t::get_max(), INT_MAX
, &objects
, 0);
4566 set
<ghobject_t
> listed(objects
.begin(), objects
.end());
4567 cerr
<< "listed.size() is " << listed
.size() << " and created.size() is " << created
.size() << std::endl
;
4568 ASSERT_TRUE(listed
.size() == created
.size());
4571 ghobject_t current
, next
;
4573 r
= store
->collection_list(cid
, current
, ghobject_t::get_max(), 60,
4576 ASSERT_TRUE(sorted(objects
));
4577 for (vector
<ghobject_t
>::iterator i
= objects
.begin();
4580 if (listed
.count(*i
))
4581 cerr
<< *i
<< " repeated" << std::endl
;
4584 if (objects
.size() < 50) {
4585 ASSERT_TRUE(next
.is_max());
4591 cerr
<< "listed.size() is " << listed
.size() << std::endl
;
4592 ASSERT_TRUE(listed
.size() == created
.size());
4593 for (set
<ghobject_t
>::iterator i
= listed
.begin();
4596 ASSERT_TRUE(created
.count(*i
));
4599 for (set
<ghobject_t
>::iterator i
= created
.begin();
4602 ObjectStore::Transaction t
;
4604 r
= apply_transaction(store
, &osr
, std::move(t
));
4607 ObjectStore::Transaction t
;
4608 t
.remove_collection(cid
);
4609 r
= apply_transaction(store
, &osr
, std::move(t
));
4613 TEST_P(StoreTest
, ScrubTest
) {
4614 ObjectStore::Sequencer
osr("test");
4615 int64_t poolid
= 111;
4616 coll_t
cid(spg_t(pg_t(0, poolid
),shard_id_t(1)));
4619 ObjectStore::Transaction t
;
4620 t
.create_collection(cid
, 0);
4621 r
= apply_transaction(store
, &osr
, std::move(t
));
4624 string base
= "aaaaa";
4625 set
<ghobject_t
> created
;
4626 for (int i
= 0; i
< 1000; ++i
) {
4628 sprintf(buf
, "%d", i
);
4630 cerr
<< "Object " << i
<< std::endl
;
4632 ghobject_t
hoid(hobject_t(string(buf
) + base
, string(), CEPH_NOSNAP
, i
,
4634 ghobject_t::NO_GEN
, shard_id_t(1));
4636 ObjectStore::Transaction t
;
4638 r
= apply_transaction(store
, &osr
, std::move(t
));
4641 created
.insert(hoid
);
4644 // Add same hobject_t but different generation
4646 ghobject_t
hoid1(hobject_t("same-object", string(), CEPH_NOSNAP
, 0, poolid
, ""),
4647 ghobject_t::NO_GEN
, shard_id_t(1));
4648 ghobject_t
hoid2(hobject_t("same-object", string(), CEPH_NOSNAP
, 0, poolid
, ""), (gen_t
)1, shard_id_t(1));
4649 ghobject_t
hoid3(hobject_t("same-object", string(), CEPH_NOSNAP
, 0, poolid
, ""), (gen_t
)2, shard_id_t(1));
4650 ObjectStore::Transaction t
;
4651 t
.touch(cid
, hoid1
);
4652 t
.touch(cid
, hoid2
);
4653 t
.touch(cid
, hoid3
);
4654 r
= apply_transaction(store
, &osr
, std::move(t
));
4655 created
.insert(hoid1
);
4656 created
.insert(hoid2
);
4657 created
.insert(hoid3
);
4661 vector
<ghobject_t
> objects
;
4662 r
= store
->collection_list(cid
, ghobject_t(), ghobject_t::get_max(),
4663 INT_MAX
, &objects
, 0);
4665 set
<ghobject_t
> listed(objects
.begin(), objects
.end());
4666 cerr
<< "listed.size() is " << listed
.size() << " and created.size() is " << created
.size() << std::endl
;
4667 ASSERT_TRUE(listed
.size() == created
.size());
4670 ghobject_t current
, next
;
4672 r
= store
->collection_list(cid
, current
, ghobject_t::get_max(), 60,
4675 ASSERT_TRUE(sorted(objects
));
4676 for (vector
<ghobject_t
>::iterator i
= objects
.begin();
4677 i
!= objects
.end(); ++i
) {
4678 if (listed
.count(*i
))
4679 cerr
<< *i
<< " repeated" << std::endl
;
4682 if (objects
.size() < 50) {
4683 ASSERT_TRUE(next
.is_max());
4687 current
= next
.get_boundary();
4689 cerr
<< "listed.size() is " << listed
.size() << std::endl
;
4690 ASSERT_TRUE(listed
.size() == created
.size());
4691 for (set
<ghobject_t
>::iterator i
= listed
.begin();
4694 ASSERT_TRUE(created
.count(*i
));
4697 for (set
<ghobject_t
>::iterator i
= created
.begin();
4700 ObjectStore::Transaction t
;
4702 r
= apply_transaction(store
, &osr
, std::move(t
));
4705 ObjectStore::Transaction t
;
4706 t
.remove_collection(cid
);
4707 r
= apply_transaction(store
, &osr
, std::move(t
));
4712 TEST_P(StoreTest
, OMapTest
) {
4713 ObjectStore::Sequencer
osr("test");
4715 ghobject_t
hoid(hobject_t("tesomap", "", CEPH_NOSNAP
, 0, 0, ""));
4718 ObjectStore::Transaction t
;
4719 t
.create_collection(cid
, 0);
4720 r
= apply_transaction(store
, &osr
, std::move(t
));
4724 map
<string
, bufferlist
> attrs
;
4726 ObjectStore::Transaction t
;
4728 t
.omap_clear(cid
, hoid
);
4729 map
<string
, bufferlist
> start_set
;
4730 t
.omap_setkeys(cid
, hoid
, start_set
);
4731 r
= apply_transaction(store
, &osr
, std::move(t
));
4735 for (int i
= 0; i
< 100; i
++) {
4737 std::cout
<< "On iteration " << i
<< std::endl
;
4739 ObjectStore::Transaction t
;
4741 map
<string
, bufferlist
> cur_attrs
;
4742 r
= store
->omap_get(cid
, hoid
, &bl
, &cur_attrs
);
4744 for (map
<string
, bufferlist
>::iterator j
= attrs
.begin();
4747 bool correct
= cur_attrs
.count(j
->first
) && string(cur_attrs
[j
->first
].c_str()) == string(j
->second
.c_str());
4749 std::cout
<< j
->first
<< " is present in cur_attrs " << cur_attrs
.count(j
->first
) << " times " << std::endl
;
4750 if (cur_attrs
.count(j
->first
) > 0) {
4751 std::cout
<< j
->second
.c_str() << " : " << cur_attrs
[j
->first
].c_str() << std::endl
;
4754 ASSERT_EQ(correct
, true);
4756 ASSERT_EQ(attrs
.size(), cur_attrs
.size());
4759 snprintf(buf
, sizeof(buf
), "%d", i
);
4761 bufferptr
bp(buf
, strlen(buf
) + 1);
4763 map
<string
, bufferlist
> to_add
;
4764 to_add
.insert(pair
<string
, bufferlist
>("key-" + string(buf
), bl
));
4765 attrs
.insert(pair
<string
, bufferlist
>("key-" + string(buf
), bl
));
4766 t
.omap_setkeys(cid
, hoid
, to_add
);
4767 r
= apply_transaction(store
, &osr
, std::move(t
));
4772 while (attrs
.size()) {
4774 std::cout
<< "removal: On iteration " << i
<< std::endl
;
4776 ObjectStore::Transaction t
;
4778 map
<string
, bufferlist
> cur_attrs
;
4779 r
= store
->omap_get(cid
, hoid
, &bl
, &cur_attrs
);
4781 for (map
<string
, bufferlist
>::iterator j
= attrs
.begin();
4784 bool correct
= cur_attrs
.count(j
->first
) && string(cur_attrs
[j
->first
].c_str()) == string(j
->second
.c_str());
4786 std::cout
<< j
->first
<< " is present in cur_attrs " << cur_attrs
.count(j
->first
) << " times " << std::endl
;
4787 if (cur_attrs
.count(j
->first
) > 0) {
4788 std::cout
<< j
->second
.c_str() << " : " << cur_attrs
[j
->first
].c_str() << std::endl
;
4791 ASSERT_EQ(correct
, true);
4794 string to_remove
= attrs
.begin()->first
;
4795 set
<string
> keys_to_remove
;
4796 keys_to_remove
.insert(to_remove
);
4797 t
.omap_rmkeys(cid
, hoid
, keys_to_remove
);
4798 r
= apply_transaction(store
, &osr
, std::move(t
));
4801 attrs
.erase(to_remove
);
4808 bl1
.append("omap_header");
4809 ObjectStore::Transaction t
;
4810 t
.omap_setheader(cid
, hoid
, bl1
);
4811 r
= apply_transaction(store
, &osr
, std::move(t
));
4813 t
= ObjectStore::Transaction();
4816 bl2
.append("value");
4817 map
<string
, bufferlist
> to_add
;
4818 to_add
.insert(pair
<string
, bufferlist
>("key", bl2
));
4819 t
.omap_setkeys(cid
, hoid
, to_add
);
4820 r
= apply_transaction(store
, &osr
, std::move(t
));
4824 map
<string
, bufferlist
> cur_attrs
;
4825 r
= store
->omap_get(cid
, hoid
, &bl3
, &cur_attrs
);
4827 ASSERT_EQ(cur_attrs
.size(), size_t(1));
4828 ASSERT_TRUE(bl_eq(bl1
, bl3
));
4831 r
= store
->omap_get_keys(cid
, hoid
, &keys
);
4833 ASSERT_EQ(keys
.size(), size_t(1));
4836 // test omap_clear, omap_rmkey_range
4839 map
<string
,bufferlist
> to_set
;
4840 for (int n
=0; n
<10; ++n
) {
4841 to_set
[stringify(n
)].append("foo");
4845 ObjectStore::Transaction t
;
4846 t
.remove(cid
, hoid
);
4848 t
.omap_setheader(cid
, hoid
, h
);
4849 t
.omap_setkeys(cid
, hoid
, to_set
);
4850 r
= apply_transaction(store
, &osr
, std::move(t
));
4854 ObjectStore::Transaction t
;
4855 t
.omap_rmkeyrange(cid
, hoid
, "3", "7");
4856 r
= apply_transaction(store
, &osr
, std::move(t
));
4861 map
<string
,bufferlist
> m
;
4862 store
->omap_get(cid
, hoid
, &hdr
, &m
);
4863 ASSERT_EQ(6u, hdr
.length());
4864 ASSERT_TRUE(m
.count("2"));
4865 ASSERT_TRUE(!m
.count("3"));
4866 ASSERT_TRUE(!m
.count("6"));
4867 ASSERT_TRUE(m
.count("7"));
4868 ASSERT_TRUE(m
.count("8"));
4869 //cout << m << std::endl;
4870 ASSERT_EQ(6u, m
.size());
4873 ObjectStore::Transaction t
;
4874 t
.omap_clear(cid
, hoid
);
4875 r
= apply_transaction(store
, &osr
, std::move(t
));
4880 map
<string
,bufferlist
> m
;
4881 store
->omap_get(cid
, hoid
, &hdr
, &m
);
4882 ASSERT_EQ(0u, hdr
.length());
4883 ASSERT_EQ(0u, m
.size());
4887 ObjectStore::Transaction t
;
4888 t
.remove(cid
, hoid
);
4889 t
.remove_collection(cid
);
4890 r
= apply_transaction(store
, &osr
, std::move(t
));
4894 TEST_P(StoreTest
, OMapIterator
) {
4895 ObjectStore::Sequencer
osr("test");
4897 ghobject_t
hoid(hobject_t("tesomap", "", CEPH_NOSNAP
, 0, 0, ""));
4901 ObjectStore::Transaction t
;
4902 t
.create_collection(cid
, 0);
4903 r
= apply_transaction(store
, &osr
, std::move(t
));
4907 map
<string
, bufferlist
> attrs
;
4909 ObjectStore::Transaction t
;
4911 t
.omap_clear(cid
, hoid
);
4912 map
<string
, bufferlist
> start_set
;
4913 t
.omap_setkeys(cid
, hoid
, start_set
);
4914 r
= apply_transaction(store
, &osr
, std::move(t
));
4917 ObjectMap::ObjectMapIterator iter
;
4920 for (int i
= 0; i
< 100; i
++) {
4922 std::cout
<< "On iteration " << i
<< std::endl
;
4926 // FileStore may deadlock two active iterators over the same data
4927 iter
= ObjectMap::ObjectMapIterator();
4929 iter
= store
->get_omap_iterator(cid
, hoid
);
4930 for (iter
->seek_to_first(), count
=0; iter
->valid(); iter
->next(), count
++) {
4931 string key
= iter
->key();
4932 bufferlist value
= iter
->value();
4933 correct
= attrs
.count(key
) && (string(value
.c_str()) == string(attrs
[key
].c_str()));
4935 if (attrs
.count(key
) > 0) {
4936 std::cout
<< "key " << key
<< "in omap , " << value
.c_str() << " : " << attrs
[key
].c_str() << std::endl
;
4939 std::cout
<< "key " << key
<< "should not exists in omap" << std::endl
;
4941 ASSERT_EQ(correct
, true);
4943 ASSERT_EQ((int)attrs
.size(), count
);
4945 // FileStore may deadlock an active iterator vs apply_transaction
4946 iter
= ObjectMap::ObjectMapIterator();
4949 snprintf(buf
, sizeof(buf
), "%d", i
);
4951 bufferptr
bp(buf
, strlen(buf
) + 1);
4953 map
<string
, bufferlist
> to_add
;
4954 to_add
.insert(pair
<string
, bufferlist
>("key-" + string(buf
), bl
));
4955 attrs
.insert(pair
<string
, bufferlist
>("key-" + string(buf
), bl
));
4956 ObjectStore::Transaction t
;
4957 t
.omap_setkeys(cid
, hoid
, to_add
);
4958 r
= apply_transaction(store
, &osr
, std::move(t
));
4962 iter
= store
->get_omap_iterator(cid
, hoid
);
4964 string bound_key
= "key-5";
4965 iter
->lower_bound(bound_key
);
4966 correct
= bound_key
<= iter
->key();
4968 std::cout
<< "lower bound, bound key is " << bound_key
<< " < iter key is " << iter
->key() << std::endl
;
4970 ASSERT_EQ(correct
, true);
4972 iter
->upper_bound(bound_key
);
4973 correct
= iter
->key() > bound_key
;
4975 std::cout
<< "upper bound, bound key is " << bound_key
<< " >= iter key is " << iter
->key() << std::endl
;
4977 ASSERT_EQ(correct
, true);
4979 // FileStore may deadlock an active iterator vs apply_transaction
4980 iter
= ObjectMap::ObjectMapIterator();
4982 ObjectStore::Transaction t
;
4983 t
.remove(cid
, hoid
);
4984 t
.remove_collection(cid
);
4985 r
= apply_transaction(store
, &osr
, std::move(t
));
4990 TEST_P(StoreTest
, XattrTest
) {
4991 ObjectStore::Sequencer
osr("test");
4993 ghobject_t
hoid(hobject_t("tesomap", "", CEPH_NOSNAP
, 0, 0, ""));
4995 for (unsigned i
= 0; i
< 10000; ++i
) {
4999 for (unsigned i
= 0; i
< 10; ++i
) {
5004 ObjectStore::Transaction t
;
5005 t
.create_collection(cid
, 0);
5007 r
= apply_transaction(store
, &osr
, std::move(t
));
5011 map
<string
, bufferlist
> attrs
;
5013 ObjectStore::Transaction t
;
5014 t
.setattr(cid
, hoid
, "attr1", small
);
5015 attrs
["attr1"] = small
;
5016 t
.setattr(cid
, hoid
, "attr2", big
);
5017 attrs
["attr2"] = big
;
5018 t
.setattr(cid
, hoid
, "attr3", small
);
5019 attrs
["attr3"] = small
;
5020 t
.setattr(cid
, hoid
, "attr1", small
);
5021 attrs
["attr1"] = small
;
5022 t
.setattr(cid
, hoid
, "attr4", big
);
5023 attrs
["attr4"] = big
;
5024 t
.setattr(cid
, hoid
, "attr3", big
);
5025 attrs
["attr3"] = big
;
5026 r
= apply_transaction(store
, &osr
, std::move(t
));
5030 map
<string
, bufferptr
> aset
;
5031 store
->getattrs(cid
, hoid
, aset
);
5032 ASSERT_EQ(aset
.size(), attrs
.size());
5033 for (map
<string
, bufferptr
>::iterator i
= aset
.begin();
5037 bl
.push_back(i
->second
);
5038 ASSERT_TRUE(attrs
[i
->first
] == bl
);
5042 ObjectStore::Transaction t
;
5043 t
.rmattr(cid
, hoid
, "attr2");
5044 attrs
.erase("attr2");
5045 r
= apply_transaction(store
, &osr
, std::move(t
));
5050 store
->getattrs(cid
, hoid
, aset
);
5051 ASSERT_EQ(aset
.size(), attrs
.size());
5052 for (map
<string
, bufferptr
>::iterator i
= aset
.begin();
5056 bl
.push_back(i
->second
);
5057 ASSERT_TRUE(attrs
[i
->first
] == bl
);
5061 r
= store
->getattr(cid
, hoid
, "attr2", bp
);
5062 ASSERT_EQ(r
, -ENODATA
);
5064 r
= store
->getattr(cid
, hoid
, "attr3", bp
);
5068 ASSERT_TRUE(bl2
== attrs
["attr3"]);
5070 ObjectStore::Transaction t
;
5071 t
.remove(cid
, hoid
);
5072 t
.remove_collection(cid
);
5073 r
= apply_transaction(store
, &osr
, std::move(t
));
5079 unsigned num_objects
,
5080 unsigned common_suffix_size
,
5083 ObjectStore::Sequencer
osr("test");
5084 coll_t
cid(spg_t(pg_t(0,52),shard_id_t::NO_SHARD
));
5085 coll_t
tid(spg_t(pg_t(1<<common_suffix_size
,52),shard_id_t::NO_SHARD
));
5088 ObjectStore::Transaction t
;
5089 t
.create_collection(cid
, common_suffix_size
);
5090 r
= apply_transaction(store
, &osr
, std::move(t
));
5094 small
.append("small");
5096 ObjectStore::Transaction t
;
5097 for (uint32_t i
= 0; i
< (2 - (int)clones
)*num_objects
; ++i
) {
5098 stringstream objname
;
5099 objname
<< "obj" << i
;
5100 ghobject_t
a(hobject_t(
5104 i
<<common_suffix_size
,
5106 t
.write(cid
, a
, 0, small
.length(), small
,
5107 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
5109 objname
<< "-clone";
5110 ghobject_t
b(hobject_t(
5114 i
<<common_suffix_size
,
5119 r
= apply_transaction(store
, &osr
, std::move(t
));
5121 t
= ObjectStore::Transaction();
5124 r
= apply_transaction(store
, &osr
, std::move(t
));
5128 ObjectStore::Transaction t
;
5129 t
.create_collection(tid
, common_suffix_size
+ 1);
5130 t
.split_collection(cid
, common_suffix_size
+1, 1<<common_suffix_size
, tid
);
5131 r
= apply_transaction(store
, &osr
, std::move(t
));
5135 ObjectStore::Transaction t
;
5136 vector
<ghobject_t
> objects
;
5137 r
= store
->collection_list(cid
, ghobject_t(), ghobject_t::get_max(),
5138 INT_MAX
, &objects
, 0);
5140 ASSERT_EQ(objects
.size(), num_objects
);
5142 for (vector
<ghobject_t
>::iterator i
= objects
.begin();
5145 ASSERT_EQ(!!(i
->hobj
.get_hash() & (1<<common_suffix_size
)), 0u);
5149 r
= apply_transaction(store
, &osr
, std::move(t
));
5151 t
= ObjectStore::Transaction();
5156 r
= store
->collection_list(tid
, ghobject_t(), ghobject_t::get_max(),
5157 INT_MAX
, &objects
, 0);
5159 ASSERT_EQ(objects
.size(), num_objects
);
5160 for (vector
<ghobject_t
>::iterator i
= objects
.begin();
5163 ASSERT_EQ(!(i
->hobj
.get_hash() & (1<<common_suffix_size
)), 0u);
5167 r
= apply_transaction(store
, &osr
, std::move(t
));
5169 t
= ObjectStore::Transaction();
5173 t
.remove_collection(cid
);
5174 t
.remove_collection(tid
);
5175 r
= apply_transaction(store
, &osr
, std::move(t
));
5179 TEST_P(StoreTest
, ColSplitTest1
) {
5180 colsplittest(store
.get(), 10000, 11, false);
5182 TEST_P(StoreTest
, ColSplitTest1Clones
) {
5183 colsplittest(store
.get(), 10000, 11, true);
5185 TEST_P(StoreTest
, ColSplitTest2
) {
5186 colsplittest(store
.get(), 100, 7, false);
5188 TEST_P(StoreTest
, ColSplitTest2Clones
) {
5189 colsplittest(store
.get(), 100, 7, true);
5193 TEST_P(StoreTest
, ColSplitTest3
) {
5194 colsplittest(store
.get(), 100000, 25);
5199 * This test tests adding two different groups
5200 * of objects, each with 1 common prefix and 1
5201 * different prefix. We then remove half
5202 * in order to verify that the merging correctly
5203 * stops at the common prefix subdir. See bug
5205 TEST_P(StoreTest
, TwoHash
) {
5206 ObjectStore::Sequencer
osr("test");
5210 ObjectStore::Transaction t
;
5211 t
.create_collection(cid
, 0);
5212 r
= apply_transaction(store
, &osr
, std::move(t
));
5215 std::cout
<< "Making objects" << std::endl
;
5216 for (int i
= 0; i
< 360; ++i
) {
5217 ObjectStore::Transaction t
;
5221 o
.hobj
.set_hash((i
<< 16) | 0xA1);
5224 o
.hobj
.set_hash((i
<< 16) | 0xB1);
5226 r
= apply_transaction(store
, &osr
, std::move(t
));
5229 std::cout
<< "Removing half" << std::endl
;
5230 for (int i
= 1; i
< 8; ++i
) {
5231 ObjectStore::Transaction t
;
5234 o
.hobj
.set_hash((i
<< 16) | 0xA1);
5236 r
= apply_transaction(store
, &osr
, std::move(t
));
5239 std::cout
<< "Checking" << std::endl
;
5240 for (int i
= 1; i
< 8; ++i
) {
5241 ObjectStore::Transaction t
;
5243 o
.hobj
.set_hash((i
<< 16) | 0xA1);
5245 bool exists
= store
->exists(cid
, o
);
5246 ASSERT_EQ(exists
, false);
5250 o
.hobj
.set_hash(0xA1);
5252 bool exists
= store
->exists(cid
, o
);
5253 ASSERT_EQ(exists
, true);
5255 std::cout
<< "Cleanup" << std::endl
;
5256 for (int i
= 0; i
< 360; ++i
) {
5257 ObjectStore::Transaction t
;
5259 o
.hobj
.set_hash((i
<< 16) | 0xA1);
5262 o
.hobj
.set_hash((i
<< 16) | 0xB1);
5264 r
= apply_transaction(store
, &osr
, std::move(t
));
5267 ObjectStore::Transaction t
;
5268 t
.remove_collection(cid
);
5269 r
= apply_transaction(store
, &osr
, std::move(t
));
5273 TEST_P(StoreTest
, Rename
) {
5274 ObjectStore::Sequencer
osr("test");
5275 coll_t
cid(spg_t(pg_t(0, 2122),shard_id_t::NO_SHARD
));
5276 ghobject_t
srcoid(hobject_t("src_oid", "", CEPH_NOSNAP
, 0, 0, ""));
5277 ghobject_t
dstoid(hobject_t("dest_oid", "", CEPH_NOSNAP
, 0, 0, ""));
5283 ObjectStore::Transaction t
;
5284 t
.create_collection(cid
, 0);
5285 t
.write(cid
, srcoid
, 0, a
.length(), a
);
5286 r
= apply_transaction(store
, &osr
, std::move(t
));
5289 ASSERT_TRUE(store
->exists(cid
, srcoid
));
5291 ObjectStore::Transaction t
;
5292 t
.collection_move_rename(cid
, srcoid
, cid
, dstoid
);
5293 t
.write(cid
, srcoid
, 0, b
.length(), b
);
5294 t
.setattr(cid
, srcoid
, "attr", b
);
5295 r
= apply_transaction(store
, &osr
, std::move(t
));
5298 ASSERT_TRUE(store
->exists(cid
, srcoid
));
5299 ASSERT_TRUE(store
->exists(cid
, dstoid
));
5302 store
->read(cid
, srcoid
, 0, 3, bl
);
5303 ASSERT_TRUE(bl_eq(b
, bl
));
5304 store
->read(cid
, dstoid
, 0, 3, bl
);
5305 ASSERT_TRUE(bl_eq(a
, bl
));
5308 ObjectStore::Transaction t
;
5309 t
.remove(cid
, dstoid
);
5310 t
.collection_move_rename(cid
, srcoid
, cid
, dstoid
);
5311 r
= apply_transaction(store
, &osr
, std::move(t
));
5314 ASSERT_TRUE(store
->exists(cid
, dstoid
));
5315 ASSERT_FALSE(store
->exists(cid
, srcoid
));
5318 store
->read(cid
, dstoid
, 0, 3, bl
);
5319 ASSERT_TRUE(bl_eq(b
, bl
));
5322 ObjectStore::Transaction t
;
5323 t
.remove(cid
, dstoid
);
5324 t
.remove_collection(cid
);
5325 r
= apply_transaction(store
, &osr
, std::move(t
));
5330 TEST_P(StoreTest
, MoveRename
) {
5331 ObjectStore::Sequencer
osr("test");
5332 coll_t
cid(spg_t(pg_t(0, 212),shard_id_t::NO_SHARD
));
5333 ghobject_t
temp_oid(hobject_t("tmp_oid", "", CEPH_NOSNAP
, 0, 0, ""));
5334 ghobject_t
oid(hobject_t("dest_oid", "", CEPH_NOSNAP
, 0, 0, ""));
5337 ObjectStore::Transaction t
;
5338 t
.create_collection(cid
, 0);
5340 r
= apply_transaction(store
, &osr
, std::move(t
));
5343 ASSERT_TRUE(store
->exists(cid
, oid
));
5344 bufferlist data
, attr
;
5345 map
<string
, bufferlist
> omap
;
5346 data
.append("data payload");
5347 attr
.append("attr value");
5348 omap
["omap_key"].append("omap value");
5350 ObjectStore::Transaction t
;
5351 t
.touch(cid
, temp_oid
);
5352 t
.write(cid
, temp_oid
, 0, data
.length(), data
);
5353 t
.setattr(cid
, temp_oid
, "attr", attr
);
5354 t
.omap_setkeys(cid
, temp_oid
, omap
);
5355 r
= apply_transaction(store
, &osr
, std::move(t
));
5358 ASSERT_TRUE(store
->exists(cid
, temp_oid
));
5360 ObjectStore::Transaction t
;
5362 t
.collection_move_rename(cid
, temp_oid
, cid
, oid
);
5363 r
= apply_transaction(store
, &osr
, std::move(t
));
5366 ASSERT_TRUE(store
->exists(cid
, oid
));
5367 ASSERT_FALSE(store
->exists(cid
, temp_oid
));
5370 r
= store
->read(cid
, oid
, 0, 1000, newdata
);
5372 ASSERT_TRUE(bl_eq(data
, newdata
));
5374 r
= store
->getattr(cid
, oid
, "attr", newattr
);
5376 ASSERT_TRUE(bl_eq(attr
, newattr
));
5378 keys
.insert("omap_key");
5379 map
<string
, bufferlist
> newomap
;
5380 r
= store
->omap_get_values(cid
, oid
, keys
, &newomap
);
5382 ASSERT_EQ(1u, newomap
.size());
5383 ASSERT_TRUE(newomap
.count("omap_key"));
5384 ASSERT_TRUE(bl_eq(omap
["omap_key"], newomap
["omap_key"]));
5387 ObjectStore::Transaction t
;
5389 t
.remove_collection(cid
);
5390 r
= apply_transaction(store
, &osr
, std::move(t
));
5395 TEST_P(StoreTest
, BigRGWObjectName
) {
5396 ObjectStore::Sequencer
osr("test");
5397 coll_t
cid(spg_t(pg_t(0,12),shard_id_t::NO_SHARD
));
5400 "default.4106.50_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
5407 shard_id_t::NO_SHARD
);
5408 ghobject_t
oid2(oid
);
5409 oid2
.generation
= 17;
5410 ghobject_t
oidhead(oid
);
5411 oidhead
.generation
= ghobject_t::NO_GEN
;
5415 ObjectStore::Transaction t
;
5416 t
.create_collection(cid
, 0);
5417 t
.touch(cid
, oidhead
);
5418 t
.collection_move_rename(cid
, oidhead
, cid
, oid
);
5419 t
.touch(cid
, oidhead
);
5420 t
.collection_move_rename(cid
, oidhead
, cid
, oid2
);
5421 r
= apply_transaction(store
, &osr
, std::move(t
));
5426 ObjectStore::Transaction t
;
5428 r
= apply_transaction(store
, &osr
, std::move(t
));
5433 vector
<ghobject_t
> objects
;
5434 r
= store
->collection_list(cid
, ghobject_t(), ghobject_t::get_max(),
5435 INT_MAX
, &objects
, 0);
5437 ASSERT_EQ(objects
.size(), 1u);
5438 ASSERT_EQ(objects
[0], oid2
);
5441 ASSERT_FALSE(store
->exists(cid
, oid
));
5444 ObjectStore::Transaction t
;
5445 t
.remove(cid
, oid2
);
5446 t
.remove_collection(cid
);
5447 r
= apply_transaction(store
, &osr
, std::move(t
));
5453 TEST_P(StoreTest
, SetAllocHint
) {
5454 ObjectStore::Sequencer
osr("test");
5456 ghobject_t
hoid(hobject_t("test_hint", "", CEPH_NOSNAP
, 0, 0, ""));
5459 ObjectStore::Transaction t
;
5460 t
.create_collection(cid
, 0);
5462 r
= apply_transaction(store
, &osr
, std::move(t
));
5466 ObjectStore::Transaction t
;
5467 t
.set_alloc_hint(cid
, hoid
, 4*1024*1024, 1024*4, 0);
5468 r
= apply_transaction(store
, &osr
, std::move(t
));
5472 ObjectStore::Transaction t
;
5473 t
.remove(cid
, hoid
);
5474 r
= apply_transaction(store
, &osr
, std::move(t
));
5478 ObjectStore::Transaction t
;
5479 t
.set_alloc_hint(cid
, hoid
, 4*1024*1024, 1024*4, 0);
5480 r
= apply_transaction(store
, &osr
, std::move(t
));
5484 ObjectStore::Transaction t
;
5485 t
.remove_collection(cid
);
5486 r
= apply_transaction(store
, &osr
, std::move(t
));
5491 TEST_P(StoreTest
, TryMoveRename
) {
5492 ObjectStore::Sequencer
osr("test");
5494 ghobject_t
hoid(hobject_t("test_hint", "", CEPH_NOSNAP
, 0, -1, ""));
5495 ghobject_t
hoid2(hobject_t("test_hint2", "", CEPH_NOSNAP
, 0, -1, ""));
5498 ObjectStore::Transaction t
;
5499 t
.create_collection(cid
, 0);
5500 r
= apply_transaction(store
, &osr
, std::move(t
));
5504 ObjectStore::Transaction t
;
5505 t
.try_rename(cid
, hoid
, hoid2
);
5506 r
= apply_transaction(store
, &osr
, std::move(t
));
5510 ObjectStore::Transaction t
;
5512 r
= apply_transaction(store
, &osr
, std::move(t
));
5516 ObjectStore::Transaction t
;
5517 t
.try_rename(cid
, hoid
, hoid2
);
5518 r
= apply_transaction(store
, &osr
, std::move(t
));
5522 ASSERT_EQ(store
->stat(cid
, hoid
, &st
), -ENOENT
);
5523 ASSERT_EQ(store
->stat(cid
, hoid2
, &st
), 0);
5526 #if defined(HAVE_LIBAIO)
5527 TEST_P(StoreTest
, BluestoreOnOffCSumTest
) {
5528 if (string(GetParam()) != "bluestore")
5530 g_conf
->set_val("bluestore_csum_type", "crc32c");
5531 g_conf
->apply_changes(NULL
);
5533 ObjectStore::Sequencer
osr("test");
5536 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
5539 r
= store
->read(cid
, hoid
, 0, 5, in
);
5540 ASSERT_EQ(-ENOENT
, r
);
5543 ObjectStore::Transaction t
;
5544 t
.create_collection(cid
, 0);
5545 cerr
<< "Creating collection " << cid
<< std::endl
;
5546 r
= apply_transaction(store
, &osr
, std::move(t
));
5550 //write with csum enabled followed by read with csum disabled
5551 size_t block_size
= 64*1024;
5552 ObjectStore::Transaction t
;
5553 bufferlist bl
, orig
;
5554 bl
.append(std::string(block_size
, 'a'));
5556 t
.remove(cid
, hoid
);
5557 t
.set_alloc_hint(cid
, hoid
, 4*1024*1024, 1024*8, 0);
5558 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
5559 cerr
<< "Remove then create" << std::endl
;
5560 r
= apply_transaction(store
, &osr
, std::move(t
));
5563 g_conf
->set_val("bluestore_csum_type", "none");
5564 g_conf
->apply_changes(NULL
);
5567 r
= store
->read(cid
, hoid
, 0, block_size
, in
);
5568 ASSERT_EQ((int)block_size
, r
);
5569 ASSERT_TRUE(bl_eq(orig
, in
));
5573 //write with csum disabled followed by read with csum enabled
5575 size_t block_size
= 64*1024;
5576 ObjectStore::Transaction t
;
5577 bufferlist bl
, orig
;
5578 bl
.append(std::string(block_size
, 'a'));
5580 t
.remove(cid
, hoid
);
5581 t
.set_alloc_hint(cid
, hoid
, 4*1024*1024, 1024*8, 0);
5582 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
5583 cerr
<< "Remove then create" << std::endl
;
5584 r
= apply_transaction(store
, &osr
, std::move(t
));
5587 g_conf
->set_val("bluestore_csum_type", "crc32c");
5588 g_conf
->apply_changes(NULL
);
5591 r
= store
->read(cid
, hoid
, 0, block_size
, in
);
5592 ASSERT_EQ((int)block_size
, r
);
5593 ASSERT_TRUE(bl_eq(orig
, in
));
5596 //'mixed' non-overlapping writes to the same blob
5598 ObjectStore::Transaction t
;
5599 bufferlist bl
, orig
;
5600 size_t block_size
= 8000;
5601 bl
.append(std::string(block_size
, 'a'));
5603 t
.remove(cid
, hoid
);
5604 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
5605 cerr
<< "Remove then create" << std::endl
;
5606 r
= apply_transaction(store
, &osr
, std::move(t
));
5609 g_conf
->set_val("bluestore_csum_type", "none");
5610 g_conf
->apply_changes(NULL
);
5612 ObjectStore::Transaction t2
;
5613 t2
.write(cid
, hoid
, block_size
*2, bl
.length(), bl
);
5614 cerr
<< "Append 'unprotected'" << std::endl
;
5615 r
= apply_transaction(store
, &osr
, std::move(t2
));
5619 r
= store
->read(cid
, hoid
, 0, block_size
, in
);
5620 ASSERT_EQ((int)block_size
, r
);
5621 ASSERT_TRUE(bl_eq(orig
, in
));
5623 r
= store
->read(cid
, hoid
, block_size
*2, block_size
, in
);
5624 ASSERT_EQ((int)block_size
, r
);
5625 ASSERT_TRUE(bl_eq(orig
, in
));
5627 g_conf
->set_val("bluestore_csum_type", "crc32c");
5628 g_conf
->apply_changes(NULL
);
5630 r
= store
->read(cid
, hoid
, 0, block_size
, in
);
5631 ASSERT_EQ((int)block_size
, r
);
5632 ASSERT_TRUE(bl_eq(orig
, in
));
5634 r
= store
->read(cid
, hoid
, block_size
*2, block_size
, in
);
5635 ASSERT_EQ((int)block_size
, r
);
5636 ASSERT_TRUE(bl_eq(orig
, in
));
5639 //partially blob overwrite under a different csum enablement mode
5641 ObjectStore::Transaction t
;
5642 bufferlist bl
, orig
, orig2
;
5643 size_t block_size0
= 0x10000;
5644 size_t block_size
= 9000;
5645 size_t block_size2
= 5000;
5646 bl
.append(std::string(block_size0
, 'a'));
5647 t
.remove(cid
, hoid
);
5648 t
.set_alloc_hint(cid
, hoid
, 4*1024*1024, 1024*8, 0);
5649 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
5650 cerr
<< "Remove then create" << std::endl
;
5651 r
= apply_transaction(store
, &osr
, std::move(t
));
5654 g_conf
->set_val("bluestore_csum_type", "none");
5655 g_conf
->apply_changes(NULL
);
5657 ObjectStore::Transaction t2
;
5659 bl
.append(std::string(block_size
, 'b'));
5660 t2
.write(cid
, hoid
, 0, bl
.length(), bl
);
5661 t2
.write(cid
, hoid
, block_size0
, bl
.length(), bl
);
5662 cerr
<< "Overwrite with unprotected data" << std::endl
;
5663 r
= apply_transaction(store
, &osr
, std::move(t2
));
5668 orig
.append( std::string(block_size0
- block_size
, 'a'));
5671 r
= store
->read(cid
, hoid
, 0, block_size0
, in
);
5672 ASSERT_EQ((int)block_size0
, r
);
5673 ASSERT_TRUE(bl_eq(orig
, in
));
5675 r
= store
->read(cid
, hoid
, block_size0
, block_size
, in
);
5676 ASSERT_EQ((int)block_size
, r
);
5677 ASSERT_TRUE(bl_eq(orig2
, in
));
5679 g_conf
->set_val("bluestore_csum_type", "crc32c");
5680 g_conf
->apply_changes(NULL
);
5682 ObjectStore::Transaction t3
;
5684 bl
.append(std::string(block_size2
, 'c'));
5685 t3
.write(cid
, hoid
, block_size0
, bl
.length(), bl
);
5686 cerr
<< "Overwrite with protected data" << std::endl
;
5687 r
= apply_transaction(store
, &osr
, std::move(t3
));
5692 orig
.append( std::string(block_size
- block_size2
, 'b'));
5693 r
= store
->read(cid
, hoid
, block_size0
, block_size
, in
);
5694 ASSERT_EQ((int)block_size
, r
);
5695 ASSERT_TRUE(bl_eq(orig
, in
));
5699 ObjectStore::Transaction t
;
5700 t
.remove(cid
, hoid
);
5701 t
.remove_collection(cid
);
5702 cerr
<< "Cleaning" << std::endl
;
5703 r
= apply_transaction(store
, &osr
, std::move(t
));
5709 INSTANTIATE_TEST_CASE_P(
5715 #if defined(HAVE_LIBAIO)
5720 // Note: instantiate all stores to preserve store numbering order only
5721 INSTANTIATE_TEST_CASE_P(
5723 StoreTestSpecificAUSize
,
5727 #if defined(HAVE_LIBAIO)
5734 // Google Test may not support value-parameterized tests with some
5735 // compilers. If we use conditional compilation to compile out all
5736 // code referring to the gtest_main library, MSVC linker will not link
5737 // that library at all and consequently complain about missing entry
5738 // point defined in that library (fatal error LNK1561: entry point
5739 // must be defined). This dummy test keeps gtest_main linked in.
5740 TEST(DummyTest
, ValueParameterizedTestsAreNotSupportedOnThisPlatform
) {}
5744 void doMany4KWritesTest(boost::scoped_ptr
<ObjectStore
>& store
,
5745 unsigned max_objects
,
5747 unsigned max_object_size
,
5748 unsigned max_write_size
,
5749 unsigned write_alignment
,
5750 store_statfs_t
* res_stat
)
5752 ObjectStore::Sequencer
osr("test");
5753 MixedGenerator
gen(555);
5754 gen_type
rng(time(NULL
));
5755 coll_t
cid(spg_t(pg_t(0,555), shard_id_t::NO_SHARD
));
5757 SyntheticWorkloadState
test_obj(store
.get(),
5766 for (unsigned i
= 0; i
< max_objects
; ++i
) {
5767 if (!(i
% 500)) cerr
<< "seeding object " << i
<< std::endl
;
5770 for (unsigned i
= 0; i
< max_ops
; ++i
) {
5772 cerr
<< "Op " << i
<< std::endl
;
5773 test_obj
.print_internal_state();
5777 test_obj
.wait_for_done();
5779 test_obj
.statfs(*res_stat
);
5781 test_obj
.shutdown();
5784 TEST_P(StoreTestSpecificAUSize
, Many4KWritesTest
) {
5785 if (string(GetParam()) != "bluestore")
5788 StartDeferred(0x10000);
5790 store_statfs_t res_stat
;
5791 unsigned max_object
= 4*1024*1024;
5793 doMany4KWritesTest(store
, 1, 1000, 4*1024*1024, 4*1024, 0, &res_stat
);
5795 ASSERT_LE(res_stat
.stored
, max_object
);
5796 ASSERT_EQ(res_stat
.allocated
, max_object
);
5799 TEST_P(StoreTestSpecificAUSize
, Many4KWritesNoCSumTest
) {
5800 if (string(GetParam()) != "bluestore")
5802 StartDeferred(0x10000);
5803 g_conf
->set_val("bluestore_csum_type", "none");
5804 g_ceph_context
->_conf
->apply_changes(NULL
);
5805 store_statfs_t res_stat
;
5806 unsigned max_object
= 4*1024*1024;
5808 doMany4KWritesTest(store
, 1, 1000, max_object
, 4*1024, 0, &res_stat
);
5810 ASSERT_LE(res_stat
.stored
, max_object
);
5811 ASSERT_EQ(res_stat
.allocated
, max_object
);
5812 g_conf
->set_val("bluestore_csum_type", "crc32c");
5815 TEST_P(StoreTestSpecificAUSize
, TooManyBlobsTest
) {
5816 if (string(GetParam()) != "bluestore")
5818 StartDeferred(0x10000);
5819 store_statfs_t res_stat
;
5820 unsigned max_object
= 4*1024*1024;
5821 doMany4KWritesTest(store
, 1, 1000, max_object
, 4*1024, 0, &res_stat
);
5822 ASSERT_LE(res_stat
.stored
, max_object
);
5823 ASSERT_EQ(res_stat
.allocated
, max_object
);
5826 #if defined(HAVE_LIBAIO)
5827 void get_mempool_stats(uint64_t* total_bytes
, uint64_t* total_items
)
5829 uint64_t onode_allocated
= mempool::bluestore_cache_onode::allocated_bytes();
5830 uint64_t other_allocated
= mempool::bluestore_cache_other::allocated_bytes();
5832 uint64_t onode_items
= mempool::bluestore_cache_onode::allocated_items();
5833 uint64_t other_items
= mempool::bluestore_cache_other::allocated_items();
5834 cout
<< "onode(" << onode_allocated
<< "/" << onode_items
5835 << ") other(" << other_allocated
<< "/" << other_items
5836 << ")" << std::endl
;
5837 *total_bytes
= onode_allocated
+ other_allocated
;
5838 *total_items
= onode_items
;
5841 TEST_P(StoreTestSpecificAUSize
, OnodeSizeTracking
) {
5843 if (string(GetParam()) != "bluestore")
5846 size_t block_size
= 4096;
5847 StartDeferred(block_size
);
5848 g_conf
->set_val("bluestore_compression_mode", "none");
5849 g_conf
->set_val("bluestore_csum_type", "none");
5850 g_conf
->set_val("bluestore_cache_size_hdd", "400000000");
5851 g_conf
->set_val("bluestore_cache_size_ssd", "400000000");
5852 g_conf
->apply_changes(NULL
);
5854 ObjectStore::Sequencer
osr("test");
5857 ghobject_t
hoid(hobject_t("test_hint", "", CEPH_NOSNAP
, 0, -1, ""));
5858 size_t obj_size
= 4 * 1024 * 1024;
5859 uint64_t total_bytes
, total_bytes2
;
5860 uint64_t total_onodes
;
5861 get_mempool_stats(&total_bytes
, &total_onodes
);
5862 ASSERT_EQ(total_onodes
, 0u);
5865 ObjectStore::Transaction t
;
5866 t
.create_collection(cid
, 0);
5867 r
= apply_transaction(store
, &osr
, std::move(t
));
5871 ObjectStore::Transaction t
;
5872 bufferlist bl
, orig
, orig2
;
5874 bl
.append(std::string(obj_size
, 'a'));
5875 t
.write(cid
, hoid
, 0, bl
.length(), bl
);
5876 r
= apply_transaction(store
, &osr
, std::move(t
));
5879 get_mempool_stats(&total_bytes
, &total_onodes
);
5880 ASSERT_NE(total_bytes
, 0u);
5881 ASSERT_EQ(total_onodes
, 1u);
5884 ObjectStore::Transaction t
;
5885 t
.truncate(cid
, hoid
, 0);
5886 r
= apply_transaction(store
, &osr
, std::move(t
));
5890 for(size_t i
= 0; i
< 1; ++i
) {
5892 bl
.append(std::string(block_size
* (i
+1), 'a'));
5893 for( size_t j
= 0; j
< obj_size
; j
+= bl
.length()) {
5894 ObjectStore::Transaction t
;
5895 t
.write(cid
, hoid
, j
, bl
.length(), bl
);
5896 r
= apply_transaction(store
, &osr
, std::move(t
));
5899 get_mempool_stats(&total_bytes2
, &total_onodes
);
5900 ASSERT_NE(total_bytes2
, 0u);
5901 ASSERT_EQ(total_onodes
, 1u);
5904 cout
<<" mempool dump:\n";
5905 JSONFormatter
f(true);
5906 f
.open_object_section("transaction");
5914 for (size_t i
= 0; i
< obj_size
; i
+= 0x1000) {
5915 store
->read(cid
, hoid
, i
, 0x1000, bl
);
5918 get_mempool_stats(&total_bytes
, &total_onodes
);
5919 ASSERT_NE(total_bytes
, 0u);
5920 ASSERT_EQ(total_onodes
, 1u);
5923 cout
<<" mempool dump:\n";
5924 JSONFormatter
f(true);
5925 f
.open_object_section("transaction");
5932 ObjectStore::Transaction t
;
5933 t
.remove(cid
, hoid
);
5934 t
.remove_collection(cid
);
5935 cerr
<< "Cleaning" << std::endl
;
5936 r
= apply_transaction(store
, &osr
, std::move(t
));
5939 g_ceph_context
->_conf
->set_val("bluestore_cache_size_hdd", "4000000");
5940 g_ceph_context
->_conf
->set_val("bluestore_cache_size_ssd", "4000000");
5941 g_conf
->set_val("bluestore_compression_mode", "none");
5942 g_conf
->set_val("bluestore_csum_type", "crc32c");
5946 TEST_P(StoreTestSpecificAUSize
, BlobReuseOnOverwrite
) {
5948 if (string(GetParam()) != "bluestore")
5951 size_t block_size
= 4096;
5952 StartDeferred(block_size
);
5953 g_conf
->set_val("bluestore_max_blob_size", "65536");
5954 g_conf
->apply_changes(NULL
);
5956 ObjectStore::Sequencer
osr("test");
5959 ghobject_t
hoid(hobject_t("test_hint", "", CEPH_NOSNAP
, 0, -1, ""));
5961 const PerfCounters
* logger
= store
->get_perf_counters();
5964 ObjectStore::Transaction t
;
5965 t
.create_collection(cid
, 0);
5966 r
= apply_transaction(store
, &osr
, std::move(t
));
5970 ObjectStore::Transaction t
;
5973 bl
.append(std::string(block_size
* 2, 'a'));
5974 t
.write(cid
, hoid
, 0, bl
.length(), bl
, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
5975 r
= apply_transaction(store
, &osr
, std::move(t
));
5979 // overwrite at the beginning
5980 ObjectStore::Transaction t
;
5983 bl
.append(std::string(block_size
, 'b'));
5984 t
.write(cid
, hoid
, 0, bl
.length(), bl
, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
5985 r
= apply_transaction(store
, &osr
, std::move(t
));
5990 ObjectStore::Transaction t
;
5993 bl
.append(std::string(block_size
* 2, 'c'));
5994 t
.write(cid
, hoid
, block_size
* 2, bl
.length(), bl
, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
5995 r
= apply_transaction(store
, &osr
, std::move(t
));
5999 // append with a gap
6000 ObjectStore::Transaction t
;
6003 bl
.append(std::string(block_size
* 2, 'd'));
6004 t
.write(cid
, hoid
, block_size
* 5, bl
.length(), bl
, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6005 r
= apply_transaction(store
, &osr
, std::move(t
));
6009 // We need to issue a read to trigger cache stat update that refresh
6010 // perf counters. additionally we need to wait some time for mempool
6011 // thread to update stats.
6013 bufferlist bl
, expected
;
6014 r
= store
->read(cid
, hoid
, 0, block_size
, bl
);
6015 ASSERT_EQ(r
, (int)block_size
);
6016 expected
.append(string(block_size
, 'b'));
6017 ASSERT_TRUE(bl_eq(expected
, bl
));
6018 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 1u);
6019 ASSERT_EQ(logger
->get(l_bluestore_extents
), 2u);
6023 ObjectStore::Transaction t
;
6026 bl
.append(std::string(block_size
* 2, 'e'));
6028 // Currently we are unable to reuse blob when overwriting in a single step
6029 t
.write(cid
, hoid
, block_size
* 6, bl
.length(), bl
, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6030 r
= apply_transaction(store
, &osr
, std::move(t
));
6034 // We need to issue a read to trigger cache stat update that refresh
6035 // perf counters. additionally we need to wait some time for mempool
6036 // thread to update stats.
6038 bufferlist bl
, expected
;
6039 r
= store
->read(cid
, hoid
, 0, block_size
, bl
);
6040 ASSERT_EQ(r
, (int)block_size
);
6041 expected
.append(string(block_size
, 'b'));
6042 ASSERT_TRUE(bl_eq(expected
, bl
));
6043 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 1u);
6044 ASSERT_EQ(logger
->get(l_bluestore_extents
), 2u);
6048 ObjectStore::Transaction t
;
6051 bl
.append(std::string(block_size
, 'f'));
6053 t
.write(cid
, hoid
, block_size
* 4, bl
.length(), bl
, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6054 r
= apply_transaction(store
, &osr
, std::move(t
));
6058 // we need to wait some time for mempool
6059 // thread to update stats to be able to check blob/extent numbers from
6063 bufferlist bl
, expected
;
6064 r
= store
->read(cid
, hoid
, 0, block_size
, bl
);
6065 ASSERT_EQ(r
, (int)block_size
);
6066 expected
.append(string(block_size
, 'b'));
6067 ASSERT_TRUE(bl_eq(expected
, bl
));
6071 r
= store
->read(cid
, hoid
, block_size
, block_size
, bl
);
6072 ASSERT_EQ(r
, (int)block_size
);
6073 expected
.append(string(block_size
, 'a'));
6074 ASSERT_TRUE(bl_eq(expected
, bl
));
6078 r
= store
->read(cid
, hoid
, block_size
* 2, block_size
* 2, bl
);
6079 ASSERT_EQ(r
, (int)block_size
* 2);
6080 expected
.append(string(block_size
* 2, 'c'));
6081 ASSERT_TRUE(bl_eq(expected
, bl
));
6085 r
= store
->read(cid
, hoid
, block_size
* 4, block_size
, bl
);
6086 ASSERT_EQ(r
, (int)block_size
);
6087 expected
.append(string(block_size
, 'f'));
6088 ASSERT_TRUE(bl_eq(expected
, bl
));
6092 r
= store
->read(cid
, hoid
, block_size
* 5, block_size
, bl
);
6093 ASSERT_EQ(r
, (int)block_size
);
6094 expected
.append(string(block_size
, 'd'));
6095 ASSERT_TRUE(bl_eq(expected
, bl
));
6099 r
= store
->read(cid
, hoid
, block_size
* 5, block_size
* 3, bl
);
6100 ASSERT_EQ(r
, (int)block_size
* 3);
6101 expected
.append(string(block_size
, 'd'));
6102 expected
.append(string(block_size
* 2, 'e'));
6103 ASSERT_TRUE(bl_eq(expected
, bl
));
6105 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 1u);
6106 ASSERT_EQ(logger
->get(l_bluestore_extents
), 1u);
6110 ObjectStore::Transaction t
;
6111 t
.remove(cid
, hoid
);
6112 t
.remove_collection(cid
);
6113 cerr
<< "Cleaning" << std::endl
;
6114 r
= apply_transaction(store
, &osr
, std::move(t
));
6117 g_conf
->set_val("bluestore_max_blob_size", "0");
6121 TEST_P(StoreTestSpecificAUSize
, BlobReuseOnOverwriteReverse
) {
6123 if (string(GetParam()) != "bluestore")
6126 size_t block_size
= 4096;
6127 StartDeferred(block_size
);
6128 g_conf
->set_val("bluestore_max_blob_size", "65536");
6130 g_conf
->apply_changes(NULL
);
6132 ObjectStore::Sequencer
osr("test");
6135 ghobject_t
hoid(hobject_t("test_hint", "", CEPH_NOSNAP
, 0, -1, ""));
6137 const PerfCounters
* logger
= store
->get_perf_counters();
6139 ObjectStore::Transaction t
;
6140 t
.create_collection(cid
, 0);
6141 r
= apply_transaction(store
, &osr
, std::move(t
));
6145 ObjectStore::Transaction t
;
6148 bl
.append(std::string(block_size
* 2, 'a'));
6149 t
.write(cid
, hoid
, block_size
* 10, bl
.length(), bl
,
6150 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6151 r
= apply_transaction(store
, &osr
, std::move(t
));
6156 ObjectStore::Transaction t
;
6159 bl
.append(std::string(block_size
, 'b'));
6160 t
.write(cid
, hoid
, block_size
* 9, bl
.length(), bl
,
6161 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6162 r
= apply_transaction(store
, &osr
, std::move(t
));
6166 // We need to issue a read to trigger cache stat update that refresh
6167 // perf counters. additionally we need to wait some time for mempool
6168 // thread to update stats.
6170 bufferlist bl
, expected
;
6171 r
= store
->read(cid
, hoid
, block_size
* 9, block_size
* 2, bl
);
6172 ASSERT_EQ(r
, (int)block_size
* 2);
6173 expected
.append(string(block_size
, 'b'));
6174 expected
.append(string(block_size
, 'a'));
6175 ASSERT_TRUE(bl_eq(expected
, bl
));
6176 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 1u);
6177 ASSERT_EQ(logger
->get(l_bluestore_extents
), 1u);
6182 // prepend existing with a gap
6183 ObjectStore::Transaction t
;
6186 bl
.append(std::string(block_size
, 'c'));
6187 t
.write(cid
, hoid
, block_size
* 7, bl
.length(), bl
,
6188 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6189 r
= apply_transaction(store
, &osr
, std::move(t
));
6193 // We need to issue a read to trigger cache stat update that refresh
6194 // perf counters. additionally we need to wait some time for mempool
6195 // thread to update stats.
6197 bufferlist bl
, expected
;
6198 r
= store
->read(cid
, hoid
, block_size
* 7, block_size
* 3, bl
);
6199 ASSERT_EQ(r
, (int)block_size
* 3);
6200 expected
.append(string(block_size
, 'c'));
6201 expected
.append(string(block_size
, 0));
6202 expected
.append(string(block_size
, 'b'));
6203 ASSERT_TRUE(bl_eq(expected
, bl
));
6204 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 1u);
6205 ASSERT_EQ(logger
->get(l_bluestore_extents
), 2u);
6209 // append after existing with a gap
6210 ObjectStore::Transaction t
;
6213 bl
.append(std::string(block_size
, 'd'));
6214 t
.write(cid
, hoid
, block_size
* 13, bl
.length(), bl
,
6215 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6216 r
= apply_transaction(store
, &osr
, std::move(t
));
6220 // We need to issue a read to trigger cache stat update that refresh
6221 // perf counters. additionally we need to wait some time for mempool
6222 // thread to update stats.
6224 bufferlist bl
, expected
;
6225 r
= store
->read(cid
, hoid
, block_size
* 11, block_size
* 3, bl
);
6226 ASSERT_EQ(r
, (int)block_size
* 3);
6227 expected
.append(string(block_size
, 'a'));
6228 expected
.append(string(block_size
, 0));
6229 expected
.append(string(block_size
, 'd'));
6230 ASSERT_TRUE(bl_eq(expected
, bl
));
6231 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 1u);
6232 ASSERT_EQ(logger
->get(l_bluestore_extents
), 3u);
6236 // append twice to the next max_blob slot
6237 ObjectStore::Transaction t
;
6240 bl
.append(std::string(block_size
, 'e'));
6241 t
.write(cid
, hoid
, block_size
* 17, bl
.length(), bl
,
6242 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6243 t
.write(cid
, hoid
, block_size
* 19, bl
.length(), bl
,
6244 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6245 r
= apply_transaction(store
, &osr
, std::move(t
));
6249 // We need to issue a read to trigger cache stat update that refresh
6250 // perf counters. additionally we need to wait some time for mempool
6251 // thread to update stats.
6253 bufferlist bl
, expected
;
6254 r
= store
->read(cid
, hoid
, block_size
* 17, block_size
* 3, bl
);
6255 ASSERT_EQ(r
, (int)block_size
* 3);
6256 expected
.append(string(block_size
, 'e'));
6257 expected
.append(string(block_size
, 0));
6258 expected
.append(string(block_size
, 'e'));
6259 ASSERT_TRUE(bl_eq(expected
, bl
));
6260 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 2u);
6261 ASSERT_EQ(logger
->get(l_bluestore_extents
), 5u);
6264 // fill gaps at the second slot
6265 ObjectStore::Transaction t
;
6268 bl
.append(std::string(block_size
, 'f'));
6269 t
.write(cid
, hoid
, block_size
* 16, bl
.length(), bl
,
6270 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6271 t
.write(cid
, hoid
, block_size
* 18, bl
.length(), bl
,
6272 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6273 r
= apply_transaction(store
, &osr
, std::move(t
));
6277 // We need to issue a read to trigger cache stat update that refresh
6278 // perf counters. additionally we need to wait some time for mempool
6279 // thread to update stats.
6281 bufferlist bl
, expected
;
6282 r
= store
->read(cid
, hoid
, block_size
* 16, block_size
* 4, bl
);
6283 ASSERT_EQ(r
, (int)block_size
* 4);
6284 expected
.append(string(block_size
, 'f'));
6285 expected
.append(string(block_size
, 'e'));
6286 expected
.append(string(block_size
, 'f'));
6287 expected
.append(string(block_size
, 'e'));
6288 ASSERT_TRUE(bl_eq(expected
, bl
));
6289 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 2u);
6290 ASSERT_EQ(logger
->get(l_bluestore_extents
), 4u);
6293 ObjectStore::Transaction t
;
6294 t
.remove(cid
, hoid
);
6295 t
.remove_collection(cid
);
6296 cerr
<< "Cleaning" << std::endl
;
6297 r
= apply_transaction(store
, &osr
, std::move(t
));
6300 g_conf
->set_val("bluestore_max_blob_size", "0");
6303 TEST_P(StoreTestSpecificAUSize
, BlobReuseOnSmallOverwrite
) {
6305 if (string(GetParam()) != "bluestore")
6308 size_t block_size
= 4096;
6309 StartDeferred(block_size
);
6310 g_conf
->set_val("bluestore_max_blob_size", "65536");
6311 g_conf
->apply_changes(NULL
);
6313 ObjectStore::Sequencer
osr("test");
6316 ghobject_t
hoid(hobject_t("test_hint", "", CEPH_NOSNAP
, 0, -1, ""));
6318 const PerfCounters
* logger
= store
->get_perf_counters();
6321 ObjectStore::Transaction t
;
6322 t
.create_collection(cid
, 0);
6323 r
= apply_transaction(store
, &osr
, std::move(t
));
6327 ObjectStore::Transaction t
;
6330 bl
.append(std::string(block_size
, 'a'));
6331 t
.write(cid
, hoid
, 0, bl
.length(), bl
, CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6332 t
.write(cid
, hoid
, block_size
* 2, bl
.length(), bl
,
6333 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6334 r
= apply_transaction(store
, &osr
, std::move(t
));
6338 // write small into the gap
6339 ObjectStore::Transaction t
;
6342 bl
.append(std::string(3, 'b'));
6343 t
.write(cid
, hoid
, block_size
+ 1, bl
.length(), bl
,
6344 CEPH_OSD_OP_FLAG_FADVISE_WILLNEED
);
6345 r
= apply_transaction(store
, &osr
, std::move(t
));
6349 // We need to issue a read to trigger cache stat update that refresh
6350 // perf counters. additionally we need to wait some time for mempool
6351 // thread to update stats.
6353 bufferlist bl
, expected
;
6354 r
= store
->read(cid
, hoid
, 0, block_size
* 3, bl
);
6355 ASSERT_EQ(r
, (int)block_size
* 3);
6356 expected
.append(string(block_size
, 'a'));
6357 expected
.append(string(1, 0));
6358 expected
.append(string(3, 'b'));
6359 expected
.append(string(block_size
- 4, 0));
6360 expected
.append(string(block_size
, 'a'));
6361 ASSERT_TRUE(bl_eq(expected
, bl
));
6363 ASSERT_EQ(logger
->get(l_bluestore_blobs
), 1u);
6364 ASSERT_EQ(logger
->get(l_bluestore_extents
), 3u);
6367 ObjectStore::Transaction t
;
6368 t
.remove(cid
, hoid
);
6369 t
.remove_collection(cid
);
6370 cerr
<< "Cleaning" << std::endl
;
6371 r
= apply_transaction(store
, &osr
, std::move(t
));
6374 g_conf
->set_val("bluestore_max_blob_size", "0");
6377 // The test case to reproduce an issue when write happens
6378 // to a zero space between the extents sharing the same spanning blob
6379 // with unloaded shard map.
6380 // Second extent might be filled with zeros this way due to wrong result
6381 // returned by has_any_extents() call in do_write_small. The latter is caused
6382 // by incompletly loaded extent map.
6383 TEST_P(StoreTestSpecificAUSize
, SmallWriteOnShardedExtents
) {
6384 if (string(GetParam()) != "bluestore")
6387 size_t block_size
= 0x10000;
6388 StartDeferred(block_size
);
6390 g_conf
->set_val("bluestore_csum_type", "xxhash64");
6391 g_conf
->set_val("bluestore_max_target_blob", "524288"); // for sure
6393 g_conf
->apply_changes(NULL
);
6395 ObjectStore::Sequencer
osr("test");
6398 ghobject_t
hoid1(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
6401 ObjectStore::Transaction t
;
6402 t
.create_collection(cid
, 0);
6403 r
= apply_transaction(store
, &osr
, std::move(t
));
6407 //doing some tricks to have sharded extents/spanning objects
6408 ObjectStore::Transaction t
;
6411 bl
.append(std::string(0x80000, 'a'));
6412 t
.write(cid
, hoid1
, 0, bl
.length(), bl
, 0);
6413 t
.zero(cid
, hoid1
, 0x719e0, 0x75b0 );
6414 r
= apply_transaction(store
, &osr
, std::move(t
));
6417 bl2
.append(std::string(0x70000, 'b'));
6418 t
.write(cid
, hoid1
, 0, bl2
.length(), bl2
, 0);
6419 t
.zero(cid
, hoid1
, 0, 0x50000);
6420 r
= apply_transaction(store
, &osr
, std::move(t
));
6428 // do a write to zero space in between some extents sharing the same blob
6429 ObjectStore::Transaction t
;
6432 bl
.append(std::string(0x6520, 'c'));
6433 t
.write(cid
, hoid1
, 0x71c00, bl
.length(), bl
, 0);
6435 r
= apply_transaction(store
, &osr
, std::move(t
));
6440 ObjectStore::Transaction t
;
6441 bufferlist bl
, expected
;
6443 r
= store
->read(cid
, hoid1
, 0x70000, 0x9c00, bl
);
6444 ASSERT_EQ(r
, (int)0x9c00);
6445 expected
.append(string(0x19e0, 'a'));
6446 expected
.append(string(0x220, 0));
6447 expected
.append(string(0x6520, 'c'));
6448 expected
.append(string(0xe70, 0));
6449 expected
.append(string(0xc70, 'a'));
6450 ASSERT_TRUE(bl_eq(expected
, bl
));
6456 ObjectStore::Transaction t
;
6457 t
.remove(cid
, hoid1
);
6458 t
.remove_collection(cid
);
6459 cerr
<< "Cleaning" << std::endl
;
6460 r
= apply_transaction(store
, &osr
, std::move(t
));
6463 g_conf
->set_val("bluestore_max_target_blob", "524288");
6464 g_conf
->set_val("bluestore_csum_type", "crc32c");
6467 TEST_P(StoreTestSpecificAUSize
, ExcessiveFragmentation
) {
6468 if (string(GetParam()) != "bluestore")
6471 g_conf
->set_val("bluestore_block_size",
6472 stringify((uint64_t)2048 * 1024 * 1024));
6474 ASSERT_EQ(g_conf
->get_val
<uint64_t>("bluefs_alloc_size"),
6477 size_t block_size
= 0x10000;
6478 StartDeferred(block_size
);
6480 ObjectStore::Sequencer
osr("test");
6483 ghobject_t
hoid1(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
6484 ghobject_t
hoid2(hobject_t(sobject_t("Object 2", CEPH_NOSNAP
)));
6487 ObjectStore::Transaction t
;
6488 t
.create_collection(cid
, 0);
6489 r
= apply_transaction(store
, &osr
, std::move(t
));
6493 // create 2x400MB objects in a way that their pextents are interleaved
6494 ObjectStore::Transaction t
;
6497 bl
.append(std::string(block_size
* 4, 'a')); // 256KB
6499 while(offs
< (uint64_t)400 * 1024 * 1024) {
6500 t
.write(cid
, hoid1
, offs
, bl
.length(), bl
, 0);
6501 t
.write(cid
, hoid2
, offs
, bl
.length(), bl
, 0);
6502 r
= apply_transaction(store
, &osr
, std::move(t
));
6504 offs
+= bl
.length();
6505 if( (offs
% (100 * 1024 * 1024)) == 0) {
6506 std::cout
<<"written " << offs
<< std::endl
;
6510 std::cout
<<"written 800MB"<<std::endl
;
6512 // Partially overwrite objects with 100MB each leaving space
6513 // fragmented and occuping still unfragmented space at the end
6514 // So we'll have enough free space but it'll lack long enough (e.g. 1MB)
6515 // contiguous pextents.
6516 ObjectStore::Transaction t
;
6519 bl
.append(std::string(block_size
* 4, 'a'));
6521 while(offs
< 112 * 1024 * 1024) {
6522 t
.write(cid
, hoid1
, offs
, bl
.length(), bl
, 0);
6523 t
.write(cid
, hoid2
, offs
, bl
.length(), bl
, 0);
6524 r
= apply_transaction(store
, &osr
, std::move(t
));
6526 // this will produce high fragmentation if original allocations
6528 offs
+= bl
.length();
6529 if( (offs
% (10 * 1024 * 1024)) == 0) {
6530 std::cout
<<"written " << offs
<< std::endl
;
6535 // remove one of the object producing much free space
6536 // and hence triggering bluefs rebalance.
6537 // Which should fail as there is no long enough pextents.
6538 ObjectStore::Transaction t
;
6539 t
.remove(cid
, hoid2
);
6540 r
= apply_transaction(store
, &osr
, std::move(t
));
6545 (int)g_conf
->get_val
<double>("bluestore_bluefs_balance_interval");
6546 std::cout
<<"sleeping... " << std::endl
;
6550 // touch another object to triggerrebalance
6551 ObjectStore::Transaction t
;
6552 t
.touch(cid
, hoid1
);
6553 r
= apply_transaction(store
, &osr
, std::move(t
));
6557 ObjectStore::Transaction t
;
6558 t
.remove(cid
, hoid1
);
6559 t
.remove(cid
, hoid2
);
6560 t
.remove_collection(cid
);
6561 cerr
<< "Cleaning" << std::endl
;
6562 r
= apply_transaction(store
, &osr
, std::move(t
));
6565 g_conf
->set_val("bluestore_block_size",
6566 stringify(DEF_STORE_TEST_BLOCKDEV_SIZE
));
6569 #endif //#if defined(HAVE_LIBAIO)
6571 TEST_P(StoreTest
, KVDBHistogramTest
) {
6572 if (string(GetParam()) != "bluestore")
6575 ObjectStore::Sequencer
osr("test");
6579 string
base("testobj.");
6581 bufferptr
ap(0x1000);
6582 memset(ap
.c_str(), 'a', 0x1000);
6585 ObjectStore::Transaction t
;
6586 t
.create_collection(cid
, 0);
6587 r
= apply_transaction(store
, &osr
, std::move(t
));
6590 for (int i
= 0; i
< NUM_OBJS
; ++i
) {
6591 ObjectStore::Transaction t
;
6593 snprintf(buf
, sizeof(buf
), "%d", i
);
6594 ghobject_t
hoid(hobject_t(sobject_t(base
+ string(buf
), CEPH_NOSNAP
)));
6595 t
.write(cid
, hoid
, 0, 0x1000, a
);
6596 r
= apply_transaction(store
, &osr
, std::move(t
));
6600 Formatter
*f
= Formatter::create("store_test", "json-pretty", "json-pretty");
6601 store
->generate_db_histogram(f
);
6606 TEST_P(StoreTest
, KVDBStatsTest
) {
6607 if (string(GetParam()) != "bluestore")
6610 g_conf
->set_val("rocksdb_perf", "true");
6611 g_conf
->set_val("rocksdb_collect_compaction_stats", "true");
6612 g_conf
->set_val("rocksdb_collect_extended_stats","true");
6613 g_conf
->set_val("rocksdb_collect_memory_stats","true");
6614 g_ceph_context
->_conf
->apply_changes(NULL
);
6615 int r
= store
->umount();
6617 r
= store
->mount(); //to force rocksdb stats
6620 ObjectStore::Sequencer
osr("test");
6623 string
base("testobj.");
6625 bufferptr
ap(0x1000);
6626 memset(ap
.c_str(), 'a', 0x1000);
6629 ObjectStore::Transaction t
;
6630 t
.create_collection(cid
, 0);
6631 r
= apply_transaction(store
, &osr
, std::move(t
));
6634 for (int i
= 0; i
< NUM_OBJS
; ++i
) {
6635 ObjectStore::Transaction t
;
6637 snprintf(buf
, sizeof(buf
), "%d", i
);
6638 ghobject_t
hoid(hobject_t(sobject_t(base
+ string(buf
), CEPH_NOSNAP
)));
6639 t
.write(cid
, hoid
, 0, 0x1000, a
);
6640 r
= apply_transaction(store
, &osr
, std::move(t
));
6644 Formatter
*f
= Formatter::create("store_test", "json-pretty", "json-pretty");
6645 store
->get_db_statistics(f
);
6648 g_conf
->set_val("rocksdb_perf", "false");
6649 g_conf
->set_val("rocksdb_collect_compaction_stats", "false");
6650 g_conf
->set_val("rocksdb_collect_extended_stats","false");
6651 g_conf
->set_val("rocksdb_collect_memory_stats","false");
6654 #if defined(HAVE_LIBAIO)
6655 TEST_P(StoreTestSpecificAUSize
, garbageCollection
) {
6656 ObjectStore::Sequencer
osr("test");
6659 int buf_len
= 256 * 1024;
6660 int overlap_offset
= 64 * 1024;
6661 int write_offset
= buf_len
;
6662 if (string(GetParam()) != "bluestore")
6665 #define WRITE_AT(offset, _length) {\
6666 ObjectStore::Transaction t;\
6667 if ((uint64_t)_length != bl.length()) { \
6668 buffer::ptr p(bl.c_str(), _length);\
6670 bl_tmp.push_back(p);\
6671 t.write(cid, hoid, offset, bl_tmp.length(), bl_tmp);\
6673 t.write(cid, hoid, offset, bl.length(), bl);\
6675 r = apply_transaction(store, &osr, std::move(t));\
6679 StartDeferred(65536);
6681 g_conf
->set_val("bluestore_compression_max_blob_size", "524288");
6682 g_conf
->set_val("bluestore_compression_min_blob_size", "262144");
6683 g_conf
->set_val("bluestore_compression_mode", "force");
6684 g_conf
->apply_changes(NULL
);
6686 ghobject_t
hoid(hobject_t(sobject_t("Object 1", CEPH_NOSNAP
)));
6689 r
= store
->read(cid
, hoid
, 0, 5, in
);
6690 ASSERT_EQ(-ENOENT
, r
);
6693 ObjectStore::Transaction t
;
6694 t
.create_collection(cid
, 0);
6695 cerr
<< "Creating collection " << cid
<< std::endl
;
6696 r
= apply_transaction(store
, &osr
, std::move(t
));
6701 data
.resize(buf_len
);
6705 bool exists
= store
->exists(cid
, hoid
);
6706 ASSERT_TRUE(!exists
);
6708 ObjectStore::Transaction t
;
6710 cerr
<< "Creating object " << hoid
<< std::endl
;
6711 r
= apply_transaction(store
, &osr
, std::move(t
));
6714 exists
= store
->exists(cid
, hoid
);
6715 ASSERT_EQ(true, exists
);
6719 for(size_t i
= 0; i
< data
.size(); i
++)
6725 struct store_statfs_t statfs
;
6726 WRITE_AT(0, buf_len
);
6727 int r
= store
->statfs(&statfs
);
6729 ASSERT_EQ(statfs
.compressed_allocated
, 0x10000);
6732 struct store_statfs_t statfs
;
6733 WRITE_AT(write_offset
- 2 * overlap_offset
, buf_len
);
6734 int r
= store
->statfs(&statfs
);
6736 ASSERT_EQ(statfs
.compressed_allocated
, 0x20000);
6737 const PerfCounters
* counters
= store
->get_perf_counters();
6738 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0u);
6742 struct store_statfs_t statfs
;
6743 WRITE_AT(write_offset
- overlap_offset
, buf_len
);
6744 int r
= store
->statfs(&statfs
);
6746 ASSERT_EQ(statfs
.compressed_allocated
, 0x20000);
6747 const PerfCounters
* counters
= store
->get_perf_counters();
6748 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0x10000u
);
6751 struct store_statfs_t statfs
;
6752 WRITE_AT(write_offset
- 3 * overlap_offset
, buf_len
);
6753 int r
= store
->statfs(&statfs
);
6755 ASSERT_EQ(statfs
.compressed_allocated
, 0x20000);
6756 const PerfCounters
* counters
= store
->get_perf_counters();
6757 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0x20000u
);
6760 struct store_statfs_t statfs
;
6761 WRITE_AT(write_offset
+ 1, overlap_offset
-1);
6762 int r
= store
->statfs(&statfs
);
6764 ASSERT_EQ(statfs
.compressed_allocated
, 0x20000);
6765 const PerfCounters
* counters
= store
->get_perf_counters();
6766 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0x20000u
);
6769 struct store_statfs_t statfs
;
6770 WRITE_AT(write_offset
+ 1, overlap_offset
);
6771 int r
= store
->statfs(&statfs
);
6773 ASSERT_EQ(statfs
.compressed_allocated
, 0x10000);
6774 const PerfCounters
* counters
= store
->get_perf_counters();
6775 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0x3ffffu
);
6778 struct store_statfs_t statfs
;
6779 WRITE_AT(0, buf_len
-1);
6780 int r
= store
->statfs(&statfs
);
6782 ASSERT_EQ(statfs
.compressed_allocated
, 0x10000);
6783 const PerfCounters
* counters
= store
->get_perf_counters();
6784 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0x40001u
);
6786 g_conf
->set_val("bluestore_gc_enable_total_threshold", "1"); //forbid GC when saving = 0
6788 struct store_statfs_t statfs
;
6789 WRITE_AT(1, overlap_offset
-2);
6790 WRITE_AT(overlap_offset
* 2 + 1, overlap_offset
-2);
6791 int r
= store
->statfs(&statfs
);
6793 ASSERT_EQ(statfs
.compressed_allocated
, 0x10000);
6794 const PerfCounters
* counters
= store
->get_perf_counters();
6795 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0x40001u
);
6798 struct store_statfs_t statfs
;
6799 WRITE_AT(overlap_offset
+ 1, overlap_offset
-2);
6800 int r
= store
->statfs(&statfs
);
6802 ASSERT_EQ(statfs
.compressed_allocated
, 0x0);
6803 const PerfCounters
* counters
= store
->get_perf_counters();
6804 ASSERT_EQ(counters
->get(l_bluestore_gc_merged
), 0x40007u
);
6807 ObjectStore::Transaction t
;
6808 t
.remove(cid
, hoid
);
6809 cerr
<< "Cleaning" << std::endl
;
6810 r
= apply_transaction(store
, &osr
, std::move(t
));
6814 g_conf
->set_val("bluestore_gc_enable_total_threshold", "0");
6815 g_conf
->set_val("bluestore_compression_min_blob_size", "0");
6816 g_conf
->set_val("bluestore_compression_max_blob_size", "0");
6817 g_conf
->set_val("bluestore_compression_mode", "none");
6818 g_conf
->apply_changes(NULL
);
6822 TEST_P(StoreTestSpecificAUSize
, fsckOnUnalignedDevice
) {
6823 if (string(GetParam()) != "bluestore")
6826 g_conf
->set_val("bluestore_block_size", stringify(0x280005000)); //10 Gb + 4K
6827 g_conf
->set_val("bluestore_fsck_on_mount", "false");
6828 g_conf
->set_val("bluestore_fsck_on_umount", "false");
6829 StartDeferred(0x4000);
6831 ASSERT_EQ(store
->fsck(false), 0); // do fsck explicitly
6834 g_conf
->set_val("bluestore_fsck_on_mount", "true");
6835 g_conf
->set_val("bluestore_fsck_on_umount", "true");
6836 g_conf
->set_val("bluestore_block_size", stringify(0x280000000)); // 10 Gb
6837 g_conf
->apply_changes(NULL
);
6840 TEST_P(StoreTestSpecificAUSize
, fsckOnUnalignedDevice2
) {
6841 if (string(GetParam()) != "bluestore")
6844 g_conf
->set_val("bluestore_block_size", stringify(0x280005000)); //10 Gb + 20K
6845 g_conf
->set_val("bluestore_fsck_on_mount", "false");
6846 g_conf
->set_val("bluestore_fsck_on_umount", "false");
6847 StartDeferred(0x1000);
6849 ASSERT_EQ(store
->fsck(false), 0); // do fsck explicitly
6852 g_conf
->set_val("bluestore_block_size", stringify(0x280000000)); // 10 Gb
6853 g_conf
->set_val("bluestore_fsck_on_mount", "true");
6854 g_conf
->set_val("bluestore_fsck_on_umount", "true");
6855 g_conf
->apply_changes(NULL
);
6858 int main(int argc
, char **argv
) {
6859 vector
<const char*> args
;
6860 argv_to_vec(argc
, (const char **)argv
, args
);
6863 auto cct
= global_init(NULL
, args
, CEPH_ENTITY_TYPE_CLIENT
,
6864 CODE_ENVIRONMENT_UTILITY
, 0);
6865 common_init_finish(g_ceph_context
);
6867 g_ceph_context
->_conf
->set_val("osd_journal_size", "400");
6868 g_ceph_context
->_conf
->set_val("filestore_index_retry_probability", "0.5");
6869 g_ceph_context
->_conf
->set_val("filestore_op_thread_timeout", "1000");
6870 g_ceph_context
->_conf
->set_val("filestore_op_thread_suicide_timeout", "10000");
6871 //g_ceph_context->_conf->set_val("filestore_fiemap", "true");
6872 g_ceph_context
->_conf
->set_val("bluestore_fsck_on_mount", "true");
6873 g_ceph_context
->_conf
->set_val("bluestore_fsck_on_umount", "true");
6874 g_ceph_context
->_conf
->set_val("bluestore_debug_misc", "true");
6875 g_ceph_context
->_conf
->set_val("bluestore_debug_small_allocations", "4");
6876 g_ceph_context
->_conf
->set_val("bluestore_debug_freelist", "true");
6877 g_ceph_context
->_conf
->set_val("bluestore_clone_cow", "true");
6878 g_ceph_context
->_conf
->set_val("bluestore_max_alloc_size", "196608");
6880 // set small cache sizes so we see trimming during Synthetic tests
6881 g_ceph_context
->_conf
->set_val("bluestore_cache_size_hdd", "4000000");
6882 g_ceph_context
->_conf
->set_val("bluestore_cache_size_ssd", "4000000");
6884 // very short *_max prealloc so that we fall back to async submits
6885 g_ceph_context
->_conf
->set_val("bluestore_blobid_prealloc", "10");
6886 g_ceph_context
->_conf
->set_val("bluestore_nid_prealloc", "10");
6887 g_ceph_context
->_conf
->set_val("bluestore_debug_randomize_serial_transaction",
6890 g_ceph_context
->_conf
->set_val("bdev_debug_aio", "true");
6892 // specify device size
6893 g_ceph_context
->_conf
->set_val("bluestore_block_size",
6894 stringify(DEF_STORE_TEST_BLOCKDEV_SIZE
));
6896 g_ceph_context
->_conf
->set_val(
6897 "enable_experimental_unrecoverable_data_corrupting_features", "*");
6898 g_ceph_context
->_conf
->apply_changes(NULL
);
6900 ::testing::InitGoogleTest(&argc
, argv
);
6901 return RUN_ALL_TESTS();
6906 * compile-command: "cd ../.. ; make ceph_test_objectstore &&
6907 * ./ceph_test_objectstore \
6908 * --gtest_filter=*.collect_metadata* --log-to-stderr=true --debug-filestore=20