1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
11 #include "global/global_init.h"
12 #include "common/ceph_argparse.h"
13 #include "include/stringify.h"
14 #include "common/errno.h"
15 #include <gtest/gtest.h>
17 #include "os/bluestore/BlueFS.h"
19 string
get_temp_bdev(uint64_t size
)
22 string fn
= "ceph_test_bluefs.tmp.block." + stringify(getpid())
23 + "." + stringify(++n
);
24 int fd
= ::open(fn
.c_str(), O_CREAT
|O_RDWR
|O_TRUNC
, 0644);
26 int r
= ::ftruncate(fd
, size
);
32 char* gen_buffer(uint64_t size
)
34 char *buffer
= new char[size
];
35 boost::random::random_device rand
;
36 rand
.generate(buffer
, buffer
+ size
);
41 void rm_temp_bdev(string f
)
47 uint64_t size
= 1048576 * 128;
48 string fn
= get_temp_bdev(size
);
50 BlueFS
fs(g_ceph_context
);
51 ASSERT_EQ(0, fs
.add_block_device(BlueFS::BDEV_DB
, fn
));
52 fs
.add_block_extent(BlueFS::BDEV_DB
, 1048576, size
- 1048576);
53 ASSERT_EQ(0, fs
.mkfs(fsid
));
57 TEST(BlueFS
, mkfs_mount
) {
58 uint64_t size
= 1048576 * 128;
59 string fn
= get_temp_bdev(size
);
60 BlueFS
fs(g_ceph_context
);
61 ASSERT_EQ(0, fs
.add_block_device(BlueFS::BDEV_DB
, fn
));
62 fs
.add_block_extent(BlueFS::BDEV_DB
, 1048576, size
- 1048576);
64 ASSERT_EQ(0, fs
.mkfs(fsid
));
65 ASSERT_EQ(0, fs
.mount());
66 ASSERT_EQ(fs
.get_total(BlueFS::BDEV_DB
), size
- 1048576);
67 ASSERT_LT(fs
.get_free(BlueFS::BDEV_DB
), size
- 1048576);
72 TEST(BlueFS
, write_read
) {
73 uint64_t size
= 1048576 * 128;
74 string fn
= get_temp_bdev(size
);
75 BlueFS
fs(g_ceph_context
);
76 ASSERT_EQ(0, fs
.add_block_device(BlueFS::BDEV_DB
, fn
));
77 fs
.add_block_extent(BlueFS::BDEV_DB
, 1048576, size
- 1048576);
79 ASSERT_EQ(0, fs
.mkfs(fsid
));
80 ASSERT_EQ(0, fs
.mount());
82 BlueFS::FileWriter
*h
;
83 ASSERT_EQ(0, fs
.mkdir("dir"));
84 ASSERT_EQ(0, fs
.open_for_write("dir", "file", &h
, false));
92 BlueFS::FileReader
*h
;
93 ASSERT_EQ(0, fs
.open_for_read("dir", "file", &h
));
95 BlueFS::FileReaderBuffer
buf(4096);
96 ASSERT_EQ(9, fs
.read(h
, &buf
, 0, 1024, &bl
, NULL
));
97 ASSERT_EQ(0, strncmp("foobarbaz", bl
.c_str(), 9));
104 TEST(BlueFS
, small_appends
) {
105 uint64_t size
= 1048576 * 128;
106 string fn
= get_temp_bdev(size
);
107 BlueFS
fs(g_ceph_context
);
108 ASSERT_EQ(0, fs
.add_block_device(BlueFS::BDEV_DB
, fn
));
109 fs
.add_block_extent(BlueFS::BDEV_DB
, 1048576, size
- 1048576);
111 ASSERT_EQ(0, fs
.mkfs(fsid
));
112 ASSERT_EQ(0, fs
.mount());
114 BlueFS::FileWriter
*h
;
115 ASSERT_EQ(0, fs
.mkdir("dir"));
116 ASSERT_EQ(0, fs
.open_for_write("dir", "file", &h
, false));
117 for (unsigned i
= 0; i
< 10000; ++i
) {
118 h
->append("abcdeabcdeabcdeabcdeabcdeabc", 23);
124 BlueFS::FileWriter
*h
;
125 ASSERT_EQ(0, fs
.open_for_write("dir", "file_sync", &h
, false));
126 for (unsigned i
= 0; i
< 1000; ++i
) {
127 h
->append("abcdeabcdeabcdeabcdeabcdeabc", 23);
136 #define ALLOC_SIZE 4096
138 void write_data(BlueFS
&fs
, uint64_t rationed_bytes
)
140 BlueFS::FileWriter
*h
;
142 uint64_t written_bytes
= 0;
143 rationed_bytes
-= ALLOC_SIZE
;
146 ss
<< std::this_thread::get_id();
147 dir
.append(ss
.str());
149 dir
.append(to_string(j
));
150 ASSERT_EQ(0, fs
.mkdir(dir
));
152 string file
= "file.";
153 file
.append(to_string(j
));
154 ASSERT_EQ(0, fs
.open_for_write(dir
, file
, &h
, false));
156 char *buf
= gen_buffer(ALLOC_SIZE
);
157 bufferptr bp
= buffer::claim_char(ALLOC_SIZE
, buf
);
159 h
->append(bl
.c_str(), bl
.length());
165 written_bytes
+= g_conf
->bluefs_alloc_size
;
168 if ((rationed_bytes
- written_bytes
) <= g_conf
->bluefs_alloc_size
) {
174 void create_single_file(BlueFS
&fs
)
176 BlueFS::FileWriter
*h
;
178 string dir
= "dir.test";
179 ASSERT_EQ(0, fs
.mkdir(dir
));
180 string file
= "testfile";
181 ASSERT_EQ(0, fs
.open_for_write(dir
, file
, &h
, false));
183 char *buf
= gen_buffer(ALLOC_SIZE
);
184 bufferptr bp
= buffer::claim_char(ALLOC_SIZE
, buf
);
186 h
->append(bl
.c_str(), bl
.length());
191 void write_single_file(BlueFS
&fs
, uint64_t rationed_bytes
)
193 BlueFS::FileWriter
*h
;
195 string dir
= "dir.test";
196 string file
= "testfile";
198 uint64_t written_bytes
= 0;
199 rationed_bytes
-= ALLOC_SIZE
;
201 ASSERT_EQ(0, fs
.open_for_write(dir
, file
, &h
, false));
203 char *buf
= gen_buffer(ALLOC_SIZE
);
204 bufferptr bp
= buffer::claim_char(ALLOC_SIZE
, buf
);
206 h
->append(bl
.c_str(), bl
.length());
212 written_bytes
+= g_conf
->bluefs_alloc_size
;
214 if ((rationed_bytes
- written_bytes
) <= g_conf
->bluefs_alloc_size
) {
220 bool writes_done
= false;
222 void sync_fs(BlueFS
&fs
)
225 if (writes_done
== true)
233 void do_join(std::thread
& t
)
238 void join_all(std::vector
<std::thread
>& v
)
240 std::for_each(v
.begin(),v
.end(),do_join
);
243 #define NUM_WRITERS 3
244 #define NUM_SYNC_THREADS 1
246 #define NUM_SINGLE_FILE_WRITERS 1
247 #define NUM_MULTIPLE_FILE_WRITERS 2
249 TEST(BlueFS
, test_flush_1
) {
250 uint64_t size
= 1048576 * 128;
251 string fn
= get_temp_bdev(size
);
252 g_ceph_context
->_conf
->set_val(
255 g_ceph_context
->_conf
->apply_changes(NULL
);
257 BlueFS
fs(g_ceph_context
);
258 ASSERT_EQ(0, fs
.add_block_device(BlueFS::BDEV_DB
, fn
));
259 fs
.add_block_extent(BlueFS::BDEV_DB
, 1048576, size
- 1048576);
261 ASSERT_EQ(0, fs
.mkfs(fsid
));
262 ASSERT_EQ(0, fs
.mount());
264 std::vector
<std::thread
> write_thread_multiple
;
265 uint64_t effective_size
= size
- (32 * 1048576); // leaving the last 32 MB for log compaction
266 uint64_t per_thread_bytes
= (effective_size
/(NUM_MULTIPLE_FILE_WRITERS
+ NUM_SINGLE_FILE_WRITERS
));
267 for (int i
=0; i
<NUM_MULTIPLE_FILE_WRITERS
; i
++) {
268 write_thread_multiple
.push_back(std::thread(write_data
, std::ref(fs
), per_thread_bytes
));
271 create_single_file(fs
);
272 std::vector
<std::thread
> write_thread_single
;
273 for (int i
=0; i
<NUM_SINGLE_FILE_WRITERS
; i
++) {
274 write_thread_single
.push_back(std::thread(write_single_file
, std::ref(fs
), per_thread_bytes
));
277 join_all(write_thread_single
);
278 join_all(write_thread_multiple
);
284 TEST(BlueFS
, test_flush_2
) {
285 uint64_t size
= 1048576 * 256;
286 string fn
= get_temp_bdev(size
);
287 g_ceph_context
->_conf
->set_val(
290 g_ceph_context
->_conf
->apply_changes(NULL
);
292 BlueFS
fs(g_ceph_context
);
293 ASSERT_EQ(0, fs
.add_block_device(BlueFS::BDEV_DB
, fn
));
294 fs
.add_block_extent(BlueFS::BDEV_DB
, 1048576, size
- 1048576);
296 ASSERT_EQ(0, fs
.mkfs(fsid
));
297 ASSERT_EQ(0, fs
.mount());
299 uint64_t effective_size
= size
- (128 * 1048576); // leaving the last 32 MB for log compaction
300 uint64_t per_thread_bytes
= (effective_size
/(NUM_WRITERS
));
301 std::vector
<std::thread
> write_thread_multiple
;
302 for (int i
=0; i
<NUM_WRITERS
; i
++) {
303 write_thread_multiple
.push_back(std::thread(write_data
, std::ref(fs
), per_thread_bytes
));
306 join_all(write_thread_multiple
);
312 TEST(BlueFS
, test_flush_3
) {
313 uint64_t size
= 1048576 * 256;
314 string fn
= get_temp_bdev(size
);
315 g_ceph_context
->_conf
->set_val(
318 g_ceph_context
->_conf
->apply_changes(NULL
);
320 BlueFS
fs(g_ceph_context
);
321 ASSERT_EQ(0, fs
.add_block_device(BlueFS::BDEV_DB
, fn
));
322 fs
.add_block_extent(BlueFS::BDEV_DB
, 1048576, size
- 1048576);
324 ASSERT_EQ(0, fs
.mkfs(fsid
));
325 ASSERT_EQ(0, fs
.mount());
327 std::vector
<std::thread
> write_threads
;
328 uint64_t effective_size
= size
- (64 * 1048576); // leaving the last 11 MB for log compaction
329 uint64_t per_thread_bytes
= (effective_size
/(NUM_WRITERS
));
330 for (int i
=0; i
<NUM_WRITERS
; i
++) {
331 write_threads
.push_back(std::thread(write_data
, std::ref(fs
), per_thread_bytes
));
334 std::vector
<std::thread
> sync_threads
;
335 for (int i
=0; i
<NUM_SYNC_THREADS
; i
++) {
336 sync_threads
.push_back(std::thread(sync_fs
, std::ref(fs
)));
339 join_all(write_threads
);
341 join_all(sync_threads
);
347 TEST(BlueFS
, test_simple_compaction_sync
) {
348 g_ceph_context
->_conf
->set_val(
349 "bluefs_compact_log_sync",
351 uint64_t size
= 1048576 * 128;
352 string fn
= get_temp_bdev(size
);
354 BlueFS
fs(g_ceph_context
);
355 ASSERT_EQ(0, fs
.add_block_device(BlueFS::BDEV_DB
, fn
));
356 fs
.add_block_extent(BlueFS::BDEV_DB
, 1048576, size
- 1048576);
358 ASSERT_EQ(0, fs
.mkfs(fsid
));
359 ASSERT_EQ(0, fs
.mount());
361 BlueFS::FileWriter
*h
;
362 for (int i
=0; i
<10; i
++) {
364 dir
.append(to_string(i
));
365 ASSERT_EQ(0, fs
.mkdir(dir
));
366 for (int j
=0; j
<10; j
++) {
367 string file
= "file.";
368 file
.append(to_string(j
));
369 ASSERT_EQ(0, fs
.open_for_write(dir
, file
, &h
, false));
371 char *buf
= gen_buffer(4096);
372 bufferptr bp
= buffer::claim_char(4096, buf
);
374 h
->append(bl
.c_str(), bl
.length());
382 for (int i
=0; i
<10; i
+=2) {
384 dir
.append(to_string(i
));
385 for (int j
=0; j
<10; j
+=2) {
386 string file
= "file.";
387 file
.append(to_string(j
));
388 fs
.unlink(dir
, file
);
400 TEST(BlueFS
, test_simple_compaction_async
) {
401 g_ceph_context
->_conf
->set_val(
402 "bluefs_compact_log_sync",
404 uint64_t size
= 1048576 * 128;
405 string fn
= get_temp_bdev(size
);
407 BlueFS
fs(g_ceph_context
);
408 ASSERT_EQ(0, fs
.add_block_device(BlueFS::BDEV_DB
, fn
));
409 fs
.add_block_extent(BlueFS::BDEV_DB
, 1048576, size
- 1048576);
411 ASSERT_EQ(0, fs
.mkfs(fsid
));
412 ASSERT_EQ(0, fs
.mount());
414 BlueFS::FileWriter
*h
;
415 for (int i
=0; i
<10; i
++) {
417 dir
.append(to_string(i
));
418 ASSERT_EQ(0, fs
.mkdir(dir
));
419 for (int j
=0; j
<10; j
++) {
420 string file
= "file.";
421 file
.append(to_string(j
));
422 ASSERT_EQ(0, fs
.open_for_write(dir
, file
, &h
, false));
424 char *buf
= gen_buffer(4096);
425 bufferptr bp
= buffer::claim_char(4096, buf
);
427 h
->append(bl
.c_str(), bl
.length());
435 for (int i
=0; i
<10; i
+=2) {
437 dir
.append(to_string(i
));
438 for (int j
=0; j
<10; j
+=2) {
439 string file
= "file.";
440 file
.append(to_string(j
));
441 fs
.unlink(dir
, file
);
453 TEST(BlueFS
, test_compaction_sync
) {
454 uint64_t size
= 1048576 * 128;
455 string fn
= get_temp_bdev(size
);
456 g_ceph_context
->_conf
->set_val(
459 g_ceph_context
->_conf
->set_val(
460 "bluefs_compact_log_sync",
463 BlueFS
fs(g_ceph_context
);
464 ASSERT_EQ(0, fs
.add_block_device(BlueFS::BDEV_DB
, fn
));
465 fs
.add_block_extent(BlueFS::BDEV_DB
, 1048576, size
- 1048576);
467 ASSERT_EQ(0, fs
.mkfs(fsid
));
468 ASSERT_EQ(0, fs
.mount());
470 std::vector
<std::thread
> write_threads
;
471 uint64_t effective_size
= size
- (32 * 1048576); // leaving the last 32 MB for log compaction
472 uint64_t per_thread_bytes
= (effective_size
/(NUM_WRITERS
));
473 for (int i
=0; i
<NUM_WRITERS
; i
++) {
474 write_threads
.push_back(std::thread(write_data
, std::ref(fs
), per_thread_bytes
));
477 std::vector
<std::thread
> sync_threads
;
478 for (int i
=0; i
<NUM_SYNC_THREADS
; i
++) {
479 sync_threads
.push_back(std::thread(sync_fs
, std::ref(fs
)));
482 join_all(write_threads
);
484 join_all(sync_threads
);
491 TEST(BlueFS
, test_compaction_async
) {
492 uint64_t size
= 1048576 * 128;
493 string fn
= get_temp_bdev(size
);
494 g_ceph_context
->_conf
->set_val(
497 g_ceph_context
->_conf
->set_val(
498 "bluefs_compact_log_sync",
501 BlueFS
fs(g_ceph_context
);
502 ASSERT_EQ(0, fs
.add_block_device(BlueFS::BDEV_DB
, fn
));
503 fs
.add_block_extent(BlueFS::BDEV_DB
, 1048576, size
- 1048576);
505 ASSERT_EQ(0, fs
.mkfs(fsid
));
506 ASSERT_EQ(0, fs
.mount());
508 std::vector
<std::thread
> write_threads
;
509 uint64_t effective_size
= size
- (32 * 1048576); // leaving the last 32 MB for log compaction
510 uint64_t per_thread_bytes
= (effective_size
/(NUM_WRITERS
));
511 for (int i
=0; i
<NUM_WRITERS
; i
++) {
512 write_threads
.push_back(std::thread(write_data
, std::ref(fs
), per_thread_bytes
));
515 std::vector
<std::thread
> sync_threads
;
516 for (int i
=0; i
<NUM_SYNC_THREADS
; i
++) {
517 sync_threads
.push_back(std::thread(sync_fs
, std::ref(fs
)));
520 join_all(write_threads
);
522 join_all(sync_threads
);
529 TEST(BlueFS
, test_replay
) {
530 uint64_t size
= 1048576 * 128;
531 string fn
= get_temp_bdev(size
);
532 g_ceph_context
->_conf
->set_val(
535 g_ceph_context
->_conf
->set_val(
536 "bluefs_compact_log_sync",
539 BlueFS
fs(g_ceph_context
);
540 ASSERT_EQ(0, fs
.add_block_device(BlueFS::BDEV_DB
, fn
));
541 fs
.add_block_extent(BlueFS::BDEV_DB
, 1048576, size
- 1048576);
543 ASSERT_EQ(0, fs
.mkfs(fsid
));
544 ASSERT_EQ(0, fs
.mount());
546 std::vector
<std::thread
> write_threads
;
547 uint64_t effective_size
= size
- (32 * 1048576); // leaving the last 32 MB for log compaction
548 uint64_t per_thread_bytes
= (effective_size
/(NUM_WRITERS
));
549 for (int i
=0; i
<NUM_WRITERS
; i
++) {
550 write_threads
.push_back(std::thread(write_data
, std::ref(fs
), per_thread_bytes
));
553 std::vector
<std::thread
> sync_threads
;
554 for (int i
=0; i
<NUM_SYNC_THREADS
; i
++) {
555 sync_threads
.push_back(std::thread(sync_fs
, std::ref(fs
)));
558 join_all(write_threads
);
560 join_all(sync_threads
);
564 // remount and check log can replay safe?
570 int main(int argc
, char **argv
) {
571 vector
<const char*> args
;
572 argv_to_vec(argc
, (const char **)argv
, args
);
575 vector
<const char *> def_args
;
576 def_args
.push_back("--debug-bluefs=1/20");
577 def_args
.push_back("--debug-bdev=1/20");
579 auto cct
= global_init(&def_args
, args
, CEPH_ENTITY_TYPE_CLIENT
,
580 CODE_ENVIRONMENT_UTILITY
,
582 common_init_finish(g_ceph_context
);
583 g_ceph_context
->_conf
->set_val(
584 "enable_experimental_unrecoverable_data_corrupting_features",
586 g_ceph_context
->_conf
->apply_changes(NULL
);
588 ::testing::InitGoogleTest(&argc
, argv
);
589 return RUN_ALL_TESTS();