1 // Copyright (c) 2016, Red Hat, Inc. All rights reserved.
2 // This source code is licensed under both the GPLv2 (found in the
3 // COPYING file in the root directory) and Apache 2.0 License
4 // (found in the LICENSE.Apache file in the root directory).
8 #include "rocksdb/utilities/env_librados.h"
9 #include <rados/librados.hpp>
10 #include "env/mock_env.h"
11 #include "util/testharness.h"
13 #include "rocksdb/db.h"
14 #include "rocksdb/slice.h"
15 #include "rocksdb/options.h"
16 #include "util/random.h"
19 #include "rocksdb/utilities/transaction_db.h"
22 typedef std::chrono::high_resolution_clock high_resolution_clock
;
23 typedef std::chrono::milliseconds milliseconds
;
25 explicit Timer(bool run
= false)
32 _start
= high_resolution_clock::now();
34 milliseconds
Elapsed() const
36 return std::chrono::duration_cast
<milliseconds
>(high_resolution_clock::now() - _start
);
38 template <typename T
, typename Traits
>
39 friend std::basic_ostream
<T
, Traits
>& operator<<(std::basic_ostream
<T
, Traits
>& out
, const Timer
& timer
)
41 return out
<< timer
.Elapsed().count();
44 high_resolution_clock::time_point _start
;
49 class EnvLibradosTest
: public testing::Test
{
51 // we will use all of these below
52 const std::string db_name
= "env_librados_test_db";
53 const std::string db_pool
= db_name
+ "_pool";
54 const char *keyring
= "admin";
55 const char *config
= "../ceph/src/ceph.conf";
58 const EnvOptions soptions_
;
61 : env_(new EnvLibrados(db_name
, config
, db_pool
)) {
65 librados::Rados rados
;
68 ret
= rados
.init("admin"); // just use the client.admin keyring
69 if (ret
< 0) { // let's handle any error that might have come back
70 std::cerr
<< "couldn't initialize rados! error " << ret
<< std::endl
;
75 ret
= rados
.conf_read_file(config
);
77 // This could fail if the config file is malformed, but it'd be hard.
78 std::cerr
<< "failed to parse config file " << config
79 << "! error" << ret
<< std::endl
;
85 * next, we actually connect to the cluster
88 ret
= rados
.connect();
90 std::cerr
<< "couldn't connect to cluster! error " << ret
<< std::endl
;
96 * And now we're done, so let's remove our pool and then
97 * shut down the connection gracefully.
99 int delete_ret
= rados
.pool_delete(db_pool
.c_str());
100 if (delete_ret
< 0) {
102 std::cerr
<< "We failed to delete our test pool!" << db_pool
<< delete_ret
<< std::endl
;
109 TEST_F(EnvLibradosTest
, Basics
) {
111 unique_ptr
<WritableFile
> writable_file
;
112 std::vector
<std::string
> children
;
114 ASSERT_OK(env_
->CreateDir("/dir"));
115 // Check that the directory is empty.
116 ASSERT_EQ(Status::NotFound(), env_
->FileExists("/dir/non_existent"));
117 ASSERT_TRUE(!env_
->GetFileSize("/dir/non_existent", &file_size
).ok());
118 ASSERT_OK(env_
->GetChildren("/dir", &children
));
119 ASSERT_EQ(0U, children
.size());
122 ASSERT_OK(env_
->NewWritableFile("/dir/f", &writable_file
, soptions_
));
123 writable_file
.reset();
125 // Check that the file exists.
126 ASSERT_OK(env_
->FileExists("/dir/f"));
127 ASSERT_OK(env_
->GetFileSize("/dir/f", &file_size
));
128 ASSERT_EQ(0U, file_size
);
129 ASSERT_OK(env_
->GetChildren("/dir", &children
));
130 ASSERT_EQ(1U, children
.size());
131 ASSERT_EQ("f", children
[0]);
133 // Write to the file.
134 ASSERT_OK(env_
->NewWritableFile("/dir/f", &writable_file
, soptions_
));
135 ASSERT_OK(writable_file
->Append("abc"));
136 writable_file
.reset();
139 // Check for expected size.
140 ASSERT_OK(env_
->GetFileSize("/dir/f", &file_size
));
141 ASSERT_EQ(3U, file_size
);
144 // Check that renaming works.
145 ASSERT_TRUE(!env_
->RenameFile("/dir/non_existent", "/dir/g").ok());
146 ASSERT_OK(env_
->RenameFile("/dir/f", "/dir/g"));
147 ASSERT_EQ(Status::NotFound(), env_
->FileExists("/dir/f"));
148 ASSERT_OK(env_
->FileExists("/dir/g"));
149 ASSERT_OK(env_
->GetFileSize("/dir/g", &file_size
));
150 ASSERT_EQ(3U, file_size
);
152 // Check that opening non-existent file fails.
153 unique_ptr
<SequentialFile
> seq_file
;
154 unique_ptr
<RandomAccessFile
> rand_file
;
156 !env_
->NewSequentialFile("/dir/non_existent", &seq_file
, soptions_
).ok());
157 ASSERT_TRUE(!seq_file
);
158 ASSERT_TRUE(!env_
->NewRandomAccessFile("/dir/non_existent", &rand_file
,
160 ASSERT_TRUE(!rand_file
);
162 // Check that deleting works.
163 ASSERT_TRUE(!env_
->DeleteFile("/dir/non_existent").ok());
164 ASSERT_OK(env_
->DeleteFile("/dir/g"));
165 ASSERT_EQ(Status::NotFound(), env_
->FileExists("/dir/g"));
166 ASSERT_OK(env_
->GetChildren("/dir", &children
));
167 ASSERT_EQ(0U, children
.size());
168 ASSERT_OK(env_
->DeleteDir("/dir"));
171 TEST_F(EnvLibradosTest
, ReadWrite
) {
172 unique_ptr
<WritableFile
> writable_file
;
173 unique_ptr
<SequentialFile
> seq_file
;
174 unique_ptr
<RandomAccessFile
> rand_file
;
178 ASSERT_OK(env_
->CreateDir("/dir"));
180 ASSERT_OK(env_
->NewWritableFile("/dir/f", &writable_file
, soptions_
));
181 ASSERT_OK(writable_file
->Append("hello "));
182 ASSERT_OK(writable_file
->Append("world"));
183 writable_file
.reset();
185 // Read sequentially.
186 ASSERT_OK(env_
->NewSequentialFile("/dir/f", &seq_file
, soptions_
));
187 ASSERT_OK(seq_file
->Read(5, &result
, scratch
)); // Read "hello".
188 ASSERT_EQ(0, result
.compare("hello"));
189 ASSERT_OK(seq_file
->Skip(1));
190 ASSERT_OK(seq_file
->Read(1000, &result
, scratch
)); // Read "world".
191 ASSERT_EQ(0, result
.compare("world"));
192 ASSERT_OK(seq_file
->Read(1000, &result
, scratch
)); // Try reading past EOF.
193 ASSERT_EQ(0U, result
.size());
194 ASSERT_OK(seq_file
->Skip(100)); // Try to skip past end of file.
195 ASSERT_OK(seq_file
->Read(1000, &result
, scratch
));
196 ASSERT_EQ(0U, result
.size());
199 ASSERT_OK(env_
->NewRandomAccessFile("/dir/f", &rand_file
, soptions_
));
200 ASSERT_OK(rand_file
->Read(6, 5, &result
, scratch
)); // Read "world".
201 ASSERT_EQ(0, result
.compare("world"));
202 ASSERT_OK(rand_file
->Read(0, 5, &result
, scratch
)); // Read "hello".
203 ASSERT_EQ(0, result
.compare("hello"));
204 ASSERT_OK(rand_file
->Read(10, 100, &result
, scratch
)); // Read "d".
205 ASSERT_EQ(0, result
.compare("d"));
208 ASSERT_OK(rand_file
->Read(1000, 5, &result
, scratch
));
211 TEST_F(EnvLibradosTest
, Locks
) {
212 FileLock
* lock
= nullptr;
213 unique_ptr
<WritableFile
> writable_file
;
215 ASSERT_OK(env_
->CreateDir("/dir"));
217 ASSERT_OK(env_
->NewWritableFile("/dir/f", &writable_file
, soptions_
));
219 // These are no-ops, but we test they return success.
220 ASSERT_OK(env_
->LockFile("some file", &lock
));
221 ASSERT_OK(env_
->UnlockFile(lock
));
223 ASSERT_OK(env_
->LockFile("/dir/f", &lock
));
224 ASSERT_OK(env_
->UnlockFile(lock
));
227 TEST_F(EnvLibradosTest
, Misc
) {
228 std::string test_dir
;
229 ASSERT_OK(env_
->GetTestDirectory(&test_dir
));
230 ASSERT_TRUE(!test_dir
.empty());
232 unique_ptr
<WritableFile
> writable_file
;
233 ASSERT_TRUE(!env_
->NewWritableFile("/a/b", &writable_file
, soptions_
).ok());
235 ASSERT_OK(env_
->NewWritableFile("/a", &writable_file
, soptions_
));
236 // These are no-ops, but we test they return success.
237 ASSERT_OK(writable_file
->Sync());
238 ASSERT_OK(writable_file
->Flush());
239 ASSERT_OK(writable_file
->Close());
240 writable_file
.reset();
243 TEST_F(EnvLibradosTest
, LargeWrite
) {
244 const size_t kWriteSize
= 300 * 1024;
245 char* scratch
= new char[kWriteSize
* 2];
247 std::string write_data
;
248 for (size_t i
= 0; i
< kWriteSize
; ++i
) {
249 write_data
.append(1, 'h');
252 unique_ptr
<WritableFile
> writable_file
;
253 ASSERT_OK(env_
->CreateDir("/dir"));
254 ASSERT_OK(env_
->NewWritableFile("/dir/g", &writable_file
, soptions_
));
255 ASSERT_OK(writable_file
->Append("foo"));
256 ASSERT_OK(writable_file
->Append(write_data
));
257 writable_file
.reset();
259 unique_ptr
<SequentialFile
> seq_file
;
261 ASSERT_OK(env_
->NewSequentialFile("/dir/g", &seq_file
, soptions_
));
262 ASSERT_OK(seq_file
->Read(3, &result
, scratch
)); // Read "foo".
263 ASSERT_EQ(0, result
.compare("foo"));
266 std::string read_data
;
267 while (read
< kWriteSize
) {
268 ASSERT_OK(seq_file
->Read(kWriteSize
- read
, &result
, scratch
));
269 read_data
.append(result
.data(), result
.size());
270 read
+= result
.size();
272 ASSERT_TRUE(write_data
== read_data
);
276 TEST_F(EnvLibradosTest
, FrequentlySmallWrite
) {
277 const size_t kWriteSize
= 1 << 10;
278 char* scratch
= new char[kWriteSize
* 2];
280 std::string write_data
;
281 for (size_t i
= 0; i
< kWriteSize
; ++i
) {
282 write_data
.append(1, 'h');
285 unique_ptr
<WritableFile
> writable_file
;
286 ASSERT_OK(env_
->CreateDir("/dir"));
287 ASSERT_OK(env_
->NewWritableFile("/dir/g", &writable_file
, soptions_
));
288 ASSERT_OK(writable_file
->Append("foo"));
290 for (size_t i
= 0; i
< kWriteSize
; ++i
) {
291 ASSERT_OK(writable_file
->Append("h"));
293 writable_file
.reset();
295 unique_ptr
<SequentialFile
> seq_file
;
297 ASSERT_OK(env_
->NewSequentialFile("/dir/g", &seq_file
, soptions_
));
298 ASSERT_OK(seq_file
->Read(3, &result
, scratch
)); // Read "foo".
299 ASSERT_EQ(0, result
.compare("foo"));
302 std::string read_data
;
303 while (read
< kWriteSize
) {
304 ASSERT_OK(seq_file
->Read(kWriteSize
- read
, &result
, scratch
));
305 read_data
.append(result
.data(), result
.size());
306 read
+= result
.size();
308 ASSERT_TRUE(write_data
== read_data
);
312 TEST_F(EnvLibradosTest
, Truncate
) {
313 const size_t kWriteSize
= 300 * 1024;
314 const size_t truncSize
= 1024;
315 std::string write_data
;
316 for (size_t i
= 0; i
< kWriteSize
; ++i
) {
317 write_data
.append(1, 'h');
320 unique_ptr
<WritableFile
> writable_file
;
321 ASSERT_OK(env_
->CreateDir("/dir"));
322 ASSERT_OK(env_
->NewWritableFile("/dir/g", &writable_file
, soptions_
));
323 ASSERT_OK(writable_file
->Append(write_data
));
324 ASSERT_EQ(writable_file
->GetFileSize(), kWriteSize
);
325 ASSERT_OK(writable_file
->Truncate(truncSize
));
326 ASSERT_EQ(writable_file
->GetFileSize(), truncSize
);
327 writable_file
.reset();
330 TEST_F(EnvLibradosTest
, DBBasics
) {
331 std::string kDBPath
= "/tmp/DBBasics";
334 // Optimize RocksDB. This is the easiest way to get RocksDB to perform well
335 options
.IncreaseParallelism();
336 options
.OptimizeLevelStyleCompaction();
337 // create the DB if it's not already present
338 options
.create_if_missing
= true;
342 Status s
= DB::Open(options
, kDBPath
, &db
);
346 s
= db
->Put(WriteOptions(), "key1", "value");
350 s
= db
->Get(ReadOptions(), "key1", &value
);
352 assert(value
== "value");
354 // atomically apply a set of updates
357 batch
.Delete("key1");
358 batch
.Put("key2", value
);
359 s
= db
->Write(WriteOptions(), &batch
);
362 s
= db
->Get(ReadOptions(), "key1", &value
);
363 assert(s
.IsNotFound());
365 db
->Get(ReadOptions(), "key2", &value
);
366 assert(value
== "value");
371 TEST_F(EnvLibradosTest
, DBLoadKeysInRandomOrder
) {
372 char key
[20] = {0}, value
[20] = {0};
373 int max_loop
= 1 << 10;
375 std::cout
<< "Test size : loop(" << max_loop
<< ")" << std::endl
;
376 /**********************************
378 ***********************************/
379 std::string kDBPath1
= "/tmp/DBLoadKeysInRandomOrder1";
382 // Optimize Rocksdb. This is the easiest way to get RocksDB to perform well
383 options1
.IncreaseParallelism();
384 options1
.OptimizeLevelStyleCompaction();
385 // create the DB if it's not already present
386 options1
.create_if_missing
= true;
389 Status s1
= DB::Open(options1
, kDBPath1
, &db1
);
392 rocksdb::Random64
r1(time(nullptr));
395 for (int i
= 0; i
< max_loop
; ++i
) {
399 (unsigned long)r1
.Uniform(std::numeric_limits
<uint64_t>::max()));
403 (unsigned long)r1
.Uniform(std::numeric_limits
<uint64_t>::max()));
405 s1
= db1
->Put(WriteOptions(), key
, value
);
408 std::cout
<< "Time by default : " << timer
<< "ms" << std::endl
;
411 /**********************************
413 ***********************************/
414 std::string kDBPath2
= "/tmp/DBLoadKeysInRandomOrder2";
417 // Optimize RocksDB. This is the easiest way to get RocksDB to perform well
418 options2
.IncreaseParallelism();
419 options2
.OptimizeLevelStyleCompaction();
420 // create the DB if it's not already present
421 options2
.create_if_missing
= true;
425 Status s2
= DB::Open(options2
, kDBPath2
, &db2
);
428 rocksdb::Random64
r2(time(nullptr));
431 for (int i
= 0; i
< max_loop
; ++i
) {
435 (unsigned long)r2
.Uniform(std::numeric_limits
<uint64_t>::max()));
439 (unsigned long)r2
.Uniform(std::numeric_limits
<uint64_t>::max()));
441 s2
= db2
->Put(WriteOptions(), key
, value
);
444 std::cout
<< "Time by librados : " << timer
<< "ms" << std::endl
;
448 TEST_F(EnvLibradosTest
, DBBulkLoadKeysInRandomOrder
) {
449 char key
[20] = {0}, value
[20] = {0};
450 int max_loop
= 1 << 6;
451 int bulk_size
= 1 << 15;
453 std::cout
<< "Test size : loop(" << max_loop
<< "); bulk_size(" << bulk_size
<< ")" << std::endl
;
454 /**********************************
456 ***********************************/
457 std::string kDBPath1
= "/tmp/DBBulkLoadKeysInRandomOrder1";
460 // Optimize Rocksdb. This is the easiest way to get RocksDB to perform well
461 options1
.IncreaseParallelism();
462 options1
.OptimizeLevelStyleCompaction();
463 // create the DB if it's not already present
464 options1
.create_if_missing
= true;
467 Status s1
= DB::Open(options1
, kDBPath1
, &db1
);
470 rocksdb::Random64
r1(time(nullptr));
473 for (int i
= 0; i
< max_loop
; ++i
) {
475 for (int j
= 0; j
< bulk_size
; ++j
) {
479 (unsigned long)r1
.Uniform(std::numeric_limits
<uint64_t>::max()));
483 (unsigned long)r1
.Uniform(std::numeric_limits
<uint64_t>::max()));
484 batch
.Put(key
, value
);
486 s1
= db1
->Write(WriteOptions(), &batch
);
489 std::cout
<< "Time by default : " << timer
<< "ms" << std::endl
;
492 /**********************************
494 ***********************************/
495 std::string kDBPath2
= "/tmp/DBBulkLoadKeysInRandomOrder2";
498 // Optimize RocksDB. This is the easiest way to get RocksDB to perform well
499 options2
.IncreaseParallelism();
500 options2
.OptimizeLevelStyleCompaction();
501 // create the DB if it's not already present
502 options2
.create_if_missing
= true;
506 Status s2
= DB::Open(options2
, kDBPath2
, &db2
);
509 rocksdb::Random64
r2(time(nullptr));
512 for (int i
= 0; i
< max_loop
; ++i
) {
514 for (int j
= 0; j
< bulk_size
; ++j
) {
518 (unsigned long)r2
.Uniform(std::numeric_limits
<uint64_t>::max()));
522 (unsigned long)r2
.Uniform(std::numeric_limits
<uint64_t>::max()));
523 batch
.Put(key
, value
);
525 s2
= db2
->Write(WriteOptions(), &batch
);
528 std::cout
<< "Time by librados : " << timer
<< "ms" << std::endl
;
532 TEST_F(EnvLibradosTest
, DBBulkLoadKeysInSequentialOrder
) {
533 char key
[20] = {0}, value
[20] = {0};
534 int max_loop
= 1 << 6;
535 int bulk_size
= 1 << 15;
537 std::cout
<< "Test size : loop(" << max_loop
<< "); bulk_size(" << bulk_size
<< ")" << std::endl
;
538 /**********************************
540 ***********************************/
541 std::string kDBPath1
= "/tmp/DBBulkLoadKeysInSequentialOrder1";
544 // Optimize Rocksdb. This is the easiest way to get RocksDB to perform well
545 options1
.IncreaseParallelism();
546 options1
.OptimizeLevelStyleCompaction();
547 // create the DB if it's not already present
548 options1
.create_if_missing
= true;
551 Status s1
= DB::Open(options1
, kDBPath1
, &db1
);
554 rocksdb::Random64
r1(time(nullptr));
557 for (int i
= 0; i
< max_loop
; ++i
) {
559 for (int j
= 0; j
< bulk_size
; ++j
) {
563 (long long)(i
* bulk_size
+ j
));
567 (unsigned long)r1
.Uniform(std::numeric_limits
<uint64_t>::max()));
568 batch
.Put(key
, value
);
570 s1
= db1
->Write(WriteOptions(), &batch
);
573 std::cout
<< "Time by default : " << timer
<< "ms" << std::endl
;
576 /**********************************
578 ***********************************/
579 std::string kDBPath2
= "/tmp/DBBulkLoadKeysInSequentialOrder2";
582 // Optimize RocksDB. This is the easiest way to get RocksDB to perform well
583 options2
.IncreaseParallelism();
584 options2
.OptimizeLevelStyleCompaction();
585 // create the DB if it's not already present
586 options2
.create_if_missing
= true;
590 Status s2
= DB::Open(options2
, kDBPath2
, &db2
);
593 rocksdb::Random64
r2(time(nullptr));
596 for (int i
= 0; i
< max_loop
; ++i
) {
598 for (int j
= 0; j
< bulk_size
; ++j
) {
602 (unsigned long)r2
.Uniform(std::numeric_limits
<uint64_t>::max()));
606 (unsigned long)r2
.Uniform(std::numeric_limits
<uint64_t>::max()));
607 batch
.Put(key
, value
);
609 s2
= db2
->Write(WriteOptions(), &batch
);
612 std::cout
<< "Time by librados : " << timer
<< "ms" << std::endl
;
616 TEST_F(EnvLibradosTest
, DBRandomRead
) {
617 char key
[20] = {0}, value
[20] = {0};
618 int max_loop
= 1 << 6;
619 int bulk_size
= 1 << 10;
620 int read_loop
= 1 << 20;
622 std::cout
<< "Test size : keys_num(" << max_loop
<< ", " << bulk_size
<< "); read_loop(" << read_loop
<< ")" << std::endl
;
623 /**********************************
625 ***********************************/
626 std::string kDBPath1
= "/tmp/DBRandomRead1";
629 // Optimize Rocksdb. This is the easiest way to get RocksDB to perform well
630 options1
.IncreaseParallelism();
631 options1
.OptimizeLevelStyleCompaction();
632 // create the DB if it's not already present
633 options1
.create_if_missing
= true;
636 Status s1
= DB::Open(options1
, kDBPath1
, &db1
);
639 rocksdb::Random64
r1(time(nullptr));
642 for (int i
= 0; i
< max_loop
; ++i
) {
644 for (int j
= 0; j
< bulk_size
; ++j
) {
648 (long long)(i
* bulk_size
+ j
));
652 (unsigned long)r1
.Uniform(std::numeric_limits
<uint64_t>::max()));
653 batch
.Put(key
, value
);
655 s1
= db1
->Write(WriteOptions(), &batch
);
659 int base1
= 0, offset1
= 0;
660 for (int i
= 0; i
< read_loop
; ++i
) {
661 base1
= r1
.Uniform(max_loop
);
662 offset1
= r1
.Uniform(bulk_size
);
667 (long long)(base1
* bulk_size
+ offset1
));
668 s1
= db1
->Get(ReadOptions(), key
, &value1
);
671 std::cout
<< "Time by default : " << timer
<< "ms" << std::endl
;
674 /**********************************
676 ***********************************/
677 std::string kDBPath2
= "/tmp/DBRandomRead2";
680 // Optimize RocksDB. This is the easiest way to get RocksDB to perform well
681 options2
.IncreaseParallelism();
682 options2
.OptimizeLevelStyleCompaction();
683 // create the DB if it's not already present
684 options2
.create_if_missing
= true;
688 Status s2
= DB::Open(options2
, kDBPath2
, &db2
);
691 rocksdb::Random64
r2(time(nullptr));
693 for (int i
= 0; i
< max_loop
; ++i
) {
695 for (int j
= 0; j
< bulk_size
; ++j
) {
699 (long long)(i
* bulk_size
+ j
));
703 (unsigned long)r2
.Uniform(std::numeric_limits
<uint64_t>::max()));
704 batch
.Put(key
, value
);
706 s2
= db2
->Write(WriteOptions(), &batch
);
711 int base2
= 0, offset2
= 0;
712 for (int i
= 0; i
< read_loop
; ++i
) {
713 base2
= r2
.Uniform(max_loop
);
714 offset2
= r2
.Uniform(bulk_size
);
719 (long long)(base2
* bulk_size
+ offset2
));
720 s2
= db2
->Get(ReadOptions(), key
, &value2
);
722 std::cout
<< s2
.ToString() << std::endl
;
726 std::cout
<< "Time by librados : " << timer
<< "ms" << std::endl
;
730 class EnvLibradosMutipoolTest
: public testing::Test
{
732 // we will use all of these below
733 const std::string client_name
= "client.admin";
734 const std::string cluster_name
= "ceph";
735 const uint64_t flags
= 0;
736 const std::string db_name
= "env_librados_test_db";
737 const std::string db_pool
= db_name
+ "_pool";
738 const std::string wal_dir
= "/wal";
739 const std::string wal_pool
= db_name
+ "_wal_pool";
740 const size_t write_buffer_size
= 1 << 20;
741 const char *keyring
= "admin";
742 const char *config
= "../ceph/src/ceph.conf";
745 const EnvOptions soptions_
;
747 EnvLibradosMutipoolTest() {
748 env_
= new EnvLibrados(client_name
, cluster_name
, flags
, db_name
, config
, db_pool
, wal_dir
, wal_pool
, write_buffer_size
);
750 ~EnvLibradosMutipoolTest() {
752 librados::Rados rados
;
755 ret
= rados
.init("admin"); // just use the client.admin keyring
756 if (ret
< 0) { // let's handle any error that might have come back
757 std::cerr
<< "couldn't initialize rados! error " << ret
<< std::endl
;
762 ret
= rados
.conf_read_file(config
);
764 // This could fail if the config file is malformed, but it'd be hard.
765 std::cerr
<< "failed to parse config file " << config
766 << "! error" << ret
<< std::endl
;
772 * next, we actually connect to the cluster
775 ret
= rados
.connect();
777 std::cerr
<< "couldn't connect to cluster! error " << ret
<< std::endl
;
783 * And now we're done, so let's remove our pool and then
784 * shut down the connection gracefully.
786 int delete_ret
= rados
.pool_delete(db_pool
.c_str());
787 if (delete_ret
< 0) {
789 std::cerr
<< "We failed to delete our test pool!" << db_pool
<< delete_ret
<< std::endl
;
792 delete_ret
= rados
.pool_delete(wal_pool
.c_str());
793 if (delete_ret
< 0) {
795 std::cerr
<< "We failed to delete our test pool!" << wal_pool
<< delete_ret
<< std::endl
;
802 TEST_F(EnvLibradosMutipoolTest
, Basics
) {
804 unique_ptr
<WritableFile
> writable_file
;
805 std::vector
<std::string
> children
;
806 std::vector
<std::string
> v
= {"/tmp/dir1", "/tmp/dir2", "/tmp/dir3", "/tmp/dir4", "dir"};
808 for (size_t i
= 0; i
< v
.size(); ++i
) {
809 std::string dir
= v
[i
];
810 std::string dir_non_existent
= dir
+ "/non_existent";
811 std::string dir_f
= dir
+ "/f";
812 std::string dir_g
= dir
+ "/g";
814 ASSERT_OK(env_
->CreateDir(dir
.c_str()));
815 // Check that the directory is empty.
816 ASSERT_EQ(Status::NotFound(), env_
->FileExists(dir_non_existent
.c_str()));
817 ASSERT_TRUE(!env_
->GetFileSize(dir_non_existent
.c_str(), &file_size
).ok());
818 ASSERT_OK(env_
->GetChildren(dir
.c_str(), &children
));
819 ASSERT_EQ(0U, children
.size());
822 ASSERT_OK(env_
->NewWritableFile(dir_f
.c_str(), &writable_file
, soptions_
));
823 writable_file
.reset();
825 // Check that the file exists.
826 ASSERT_OK(env_
->FileExists(dir_f
.c_str()));
827 ASSERT_OK(env_
->GetFileSize(dir_f
.c_str(), &file_size
));
828 ASSERT_EQ(0U, file_size
);
829 ASSERT_OK(env_
->GetChildren(dir
.c_str(), &children
));
830 ASSERT_EQ(1U, children
.size());
831 ASSERT_EQ("f", children
[0]);
833 // Write to the file.
834 ASSERT_OK(env_
->NewWritableFile(dir_f
.c_str(), &writable_file
, soptions_
));
835 ASSERT_OK(writable_file
->Append("abc"));
836 writable_file
.reset();
839 // Check for expected size.
840 ASSERT_OK(env_
->GetFileSize(dir_f
.c_str(), &file_size
));
841 ASSERT_EQ(3U, file_size
);
844 // Check that renaming works.
845 ASSERT_TRUE(!env_
->RenameFile(dir_non_existent
.c_str(), dir_g
.c_str()).ok());
846 ASSERT_OK(env_
->RenameFile(dir_f
.c_str(), dir_g
.c_str()));
847 ASSERT_EQ(Status::NotFound(), env_
->FileExists(dir_f
.c_str()));
848 ASSERT_OK(env_
->FileExists(dir_g
.c_str()));
849 ASSERT_OK(env_
->GetFileSize(dir_g
.c_str(), &file_size
));
850 ASSERT_EQ(3U, file_size
);
852 // Check that opening non-existent file fails.
853 unique_ptr
<SequentialFile
> seq_file
;
854 unique_ptr
<RandomAccessFile
> rand_file
;
856 !env_
->NewSequentialFile(dir_non_existent
.c_str(), &seq_file
, soptions_
).ok());
857 ASSERT_TRUE(!seq_file
);
858 ASSERT_TRUE(!env_
->NewRandomAccessFile(dir_non_existent
.c_str(), &rand_file
,
860 ASSERT_TRUE(!rand_file
);
862 // Check that deleting works.
863 ASSERT_TRUE(!env_
->DeleteFile(dir_non_existent
.c_str()).ok());
864 ASSERT_OK(env_
->DeleteFile(dir_g
.c_str()));
865 ASSERT_EQ(Status::NotFound(), env_
->FileExists(dir_g
.c_str()));
866 ASSERT_OK(env_
->GetChildren(dir
.c_str(), &children
));
867 ASSERT_EQ(0U, children
.size());
868 ASSERT_OK(env_
->DeleteDir(dir
.c_str()));
872 TEST_F(EnvLibradosMutipoolTest
, DBBasics
) {
873 std::string kDBPath
= "/tmp/DBBasics";
874 std::string walPath
= "/tmp/wal";
877 // Optimize RocksDB. This is the easiest way to get RocksDB to perform well
878 options
.IncreaseParallelism();
879 options
.OptimizeLevelStyleCompaction();
880 // create the DB if it's not already present
881 options
.create_if_missing
= true;
883 options
.wal_dir
= walPath
;
886 Status s
= DB::Open(options
, kDBPath
, &db
);
890 s
= db
->Put(WriteOptions(), "key1", "value");
894 s
= db
->Get(ReadOptions(), "key1", &value
);
896 assert(value
== "value");
898 // atomically apply a set of updates
901 batch
.Delete("key1");
902 batch
.Put("key2", value
);
903 s
= db
->Write(WriteOptions(), &batch
);
906 s
= db
->Get(ReadOptions(), "key1", &value
);
907 assert(s
.IsNotFound());
909 db
->Get(ReadOptions(), "key2", &value
);
910 assert(value
== "value");
915 TEST_F(EnvLibradosMutipoolTest
, DBBulkLoadKeysInRandomOrder
) {
916 char key
[20] = {0}, value
[20] = {0};
917 int max_loop
= 1 << 6;
918 int bulk_size
= 1 << 15;
920 std::cout
<< "Test size : loop(" << max_loop
<< "); bulk_size(" << bulk_size
<< ")" << std::endl
;
921 /**********************************
923 ***********************************/
924 std::string kDBPath1
= "/tmp/DBBulkLoadKeysInRandomOrder1";
925 std::string walPath
= "/tmp/wal";
928 // Optimize Rocksdb. This is the easiest way to get RocksDB to perform well
929 options1
.IncreaseParallelism();
930 options1
.OptimizeLevelStyleCompaction();
931 // create the DB if it's not already present
932 options1
.create_if_missing
= true;
935 Status s1
= DB::Open(options1
, kDBPath1
, &db1
);
938 rocksdb::Random64
r1(time(nullptr));
941 for (int i
= 0; i
< max_loop
; ++i
) {
943 for (int j
= 0; j
< bulk_size
; ++j
) {
947 (unsigned long)r1
.Uniform(std::numeric_limits
<uint64_t>::max()));
951 (unsigned long)r1
.Uniform(std::numeric_limits
<uint64_t>::max()));
952 batch
.Put(key
, value
);
954 s1
= db1
->Write(WriteOptions(), &batch
);
957 std::cout
<< "Time by default : " << timer
<< "ms" << std::endl
;
960 /**********************************
962 ***********************************/
963 std::string kDBPath2
= "/tmp/DBBulkLoadKeysInRandomOrder2";
966 // Optimize RocksDB. This is the easiest way to get RocksDB to perform well
967 options2
.IncreaseParallelism();
968 options2
.OptimizeLevelStyleCompaction();
969 // create the DB if it's not already present
970 options2
.create_if_missing
= true;
972 options2
.wal_dir
= walPath
;
975 Status s2
= DB::Open(options2
, kDBPath2
, &db2
);
977 std::cerr
<< s2
.ToString() << std::endl
;
981 rocksdb::Random64
r2(time(nullptr));
984 for (int i
= 0; i
< max_loop
; ++i
) {
986 for (int j
= 0; j
< bulk_size
; ++j
) {
990 (unsigned long)r2
.Uniform(std::numeric_limits
<uint64_t>::max()));
994 (unsigned long)r2
.Uniform(std::numeric_limits
<uint64_t>::max()));
995 batch
.Put(key
, value
);
997 s2
= db2
->Write(WriteOptions(), &batch
);
1000 std::cout
<< "Time by librados : " << timer
<< "ms" << std::endl
;
1004 TEST_F(EnvLibradosMutipoolTest
, DBTransactionDB
) {
1005 std::string kDBPath
= "/tmp/DBTransactionDB";
1008 TransactionDBOptions txn_db_options
;
1009 options
.create_if_missing
= true;
1011 TransactionDB
* txn_db
;
1013 Status s
= TransactionDB::Open(options
, txn_db_options
, kDBPath
, &txn_db
);
1016 WriteOptions write_options
;
1017 ReadOptions read_options
;
1018 TransactionOptions txn_options
;
1021 ////////////////////////////////////////////////////////
1023 // Simple OptimisticTransaction Example ("Read Committed")
1025 ////////////////////////////////////////////////////////
1027 // Start a transaction
1028 Transaction
* txn
= txn_db
->BeginTransaction(write_options
);
1031 // Read a key in this transaction
1032 s
= txn
->Get(read_options
, "abc", &value
);
1033 assert(s
.IsNotFound());
1035 // Write a key in this transaction
1036 s
= txn
->Put("abc", "def");
1039 // Read a key OUTSIDE this transaction. Does not affect txn.
1040 s
= txn_db
->Get(read_options
, "abc", &value
);
1042 // Write a key OUTSIDE of this transaction.
1043 // Does not affect txn since this is an unrelated key. If we wrote key 'abc'
1044 // here, the transaction would fail to commit.
1045 s
= txn_db
->Put(write_options
, "xyz", "zzz");
1047 // Commit transaction
1052 ////////////////////////////////////////////////////////
1054 // "Repeatable Read" (Snapshot Isolation) Example
1055 // -- Using a single Snapshot
1057 ////////////////////////////////////////////////////////
1059 // Set a snapshot at start of transaction by setting set_snapshot=true
1060 txn_options
.set_snapshot
= true;
1061 txn
= txn_db
->BeginTransaction(write_options
, txn_options
);
1063 const Snapshot
* snapshot
= txn
->GetSnapshot();
1065 // Write a key OUTSIDE of transaction
1066 s
= txn_db
->Put(write_options
, "abc", "xyz");
1069 // Attempt to read a key using the snapshot. This will fail since
1070 // the previous write outside this txn conflicts with this read.
1071 read_options
.snapshot
= snapshot
;
1072 s
= txn
->GetForUpdate(read_options
, "abc", &value
);
1078 // Clear snapshot from read options since it is no longer valid
1079 read_options
.snapshot
= nullptr;
1082 ////////////////////////////////////////////////////////
1084 // "Read Committed" (Monotonic Atomic Views) Example
1085 // --Using multiple Snapshots
1087 ////////////////////////////////////////////////////////
1089 // In this example, we set the snapshot multiple times. This is probably
1090 // only necessary if you have very strict isolation requirements to
1093 // Set a snapshot at start of transaction
1094 txn_options
.set_snapshot
= true;
1095 txn
= txn_db
->BeginTransaction(write_options
, txn_options
);
1097 // Do some reads and writes to key "x"
1098 read_options
.snapshot
= txn_db
->GetSnapshot();
1099 s
= txn
->Get(read_options
, "x", &value
);
1102 // Do a write outside of the transaction to key "y"
1103 s
= txn_db
->Put(write_options
, "y", "y");
1105 // Set a new snapshot in the transaction
1107 txn
->SetSavePoint();
1108 read_options
.snapshot
= txn_db
->GetSnapshot();
1110 // Do some reads and writes to key "y"
1111 // Since the snapshot was advanced, the write done outside of the
1112 // transaction does not conflict.
1113 s
= txn
->GetForUpdate(read_options
, "y", &value
);
1116 // Decide we want to revert the last write from this transaction.
1117 txn
->RollbackToSavePoint();
1123 // Clear snapshot from read options since it is no longer valid
1124 read_options
.snapshot
= nullptr;
1128 DestroyDB(kDBPath
, options
);
1131 } // namespace rocksdb
1133 int main(int argc
, char** argv
) {
1134 ::testing::InitGoogleTest(&argc
, argv
);
1135 return RUN_ALL_TESTS();
1141 int main(int argc
, char** argv
) {
1142 fprintf(stderr
, "SKIPPED as EnvMirror is not supported in ROCKSDB_LITE\n");
1146 #endif // !ROCKSDB_LITE