ceph/src/test/objectstore/test_bluefs.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include <stdio.h>
   5 #include <string.h>
   6 #include <iostream>
   7 #include <time.h>
   8 #include <fcntl.h>
   9 #include <unistd.h>
  10 #include <random>
  11 #include <thread>
  12 #include "global/global_init.h"
  13 #include "common/ceph_argparse.h"
  14 #include "include/stringify.h"
  15 #include "include/scope_guard.h"
  16 #include "common/errno.h"
  17 #include <gtest/gtest.h>
  18
  19 #include "os/bluestore/BlueFS.h"
  20
  21 std::unique_ptr<char[]> gen_buffer(uint64_t size)
  22 {
  23     std::unique_ptr<char[]> buffer = std::make_unique<char[]>(size);
  24     std::independent_bits_engine<std::default_random_engine, CHAR_BIT, unsigned char> e;
  25     std::generate(buffer.get(), buffer.get()+size, std::ref(e));
  26     return buffer;
  27 }
  28
  29 class TempBdev {
  30 public:
  31   TempBdev(uint64_t size)
  32     : path{get_temp_bdev(size)}
  33   {}
  34   ~TempBdev() {
  35     rm_temp_bdev(path);
  36   }
  37   const std::string path;
  38 private:
  39   static string get_temp_bdev(uint64_t size)
  40   {
  41     static int n = 0;
  42     string fn = "ceph_test_bluefs.tmp.block." + stringify(getpid())
  43       + "." + stringify(++n);
  44     int fd = ::open(fn.c_str(), O_CREAT|O_RDWR|O_TRUNC, 0644);
  45     ceph_assert(fd >= 0);
  46     int r = ::ftruncate(fd, size);
  47     ceph_assert(r >= 0);
  48     ::close(fd);
  49     return fn;
  50   }
  51   static void rm_temp_bdev(string f)
  52   {
  53     ::unlink(f.c_str());
  54   }
  55 };
  56
  57 TEST(BlueFS, mkfs) {
  58   uint64_t size = 1048576 * 128;
  59   TempBdev bdev{size};
  60   uuid_d fsid;
  61   BlueFS fs(g_ceph_context);
  62   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
  63   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
  64   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
  65 }
  66
  67 TEST(BlueFS, mkfs_mount) {
  68   uint64_t size = 1048576 * 128;
  69   TempBdev bdev{size};
  70   BlueFS fs(g_ceph_context);
  71   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
  72   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
  73   uuid_d fsid;
  74   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
  75   ASSERT_EQ(0, fs.mount());
  76   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
  77   ASSERT_EQ(fs.get_total(BlueFS::BDEV_DB), size - 1048576);
  78   ASSERT_LT(fs.get_free(BlueFS::BDEV_DB), size - 1048576);
  79   fs.umount();
  80 }
  81
  82 TEST(BlueFS, mkfs_mount_duplicate_gift) {
  83   uint64_t size = 1048576 * 128;
  84   TempBdev bdev{ size };
  85   {
  86     BlueFS fs(g_ceph_context);
  87     ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
  88     fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
  89     uuid_d fsid;
  90     ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
  91     ASSERT_EQ(0, fs.mount());
  92
  93     {
  94       BlueFS::FileWriter *h;
  95       ASSERT_EQ(0, fs.mkdir("dir"));
  96       ASSERT_EQ(0, fs.open_for_write("dir", "file1", &h, false));
  97       h->append("foo", 3);
  98       h->append("bar", 3);
  99       h->append("baz", 3);
 100       fs.fsync(h);
 101       fs.close_writer(h);
 102     }
 103
 104     fs.umount();
 105   }
 106
 107   {
 108     BlueFS fs(g_ceph_context);
 109     ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 110     ASSERT_EQ(0, fs.mount());
 111     // free allocation presumably allocated for file1
 112     fs.debug_inject_duplicate_gift(BlueFS::BDEV_DB, 5 * 1048576, 1048576);
 113     {
 114       // overwrite file1 with file2
 115       BlueFS::FileWriter *h;
 116       ASSERT_EQ(0, fs.open_for_write("dir", "file2", &h, false));
 117       h->append("foo", 3);
 118       h->append("bar", 3);
 119       h->append("baz", 3);
 120       fs.fsync(h);
 121       fs.close_writer(h);
 122     }
 123     fs.umount();
 124   }
 125
 126   g_ceph_context->_conf.set_val_or_die("bluefs_log_replay_check_allocations", "true");
 127   g_ceph_context->_conf.apply_changes(nullptr);
 128
 129   {
 130     // this should fail
 131     BlueFS fs(g_ceph_context);
 132     ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 133     ASSERT_NE(0, fs.mount());
 134   }
 135 }
 136
 137 TEST(BlueFS, write_read) {
 138   uint64_t size = 1048576 * 128;
 139   TempBdev bdev{size};
 140   BlueFS fs(g_ceph_context);
 141   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 142   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 143   uuid_d fsid;
 144   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 145   ASSERT_EQ(0, fs.mount());
 146   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 147   {
 148     BlueFS::FileWriter *h;
 149     ASSERT_EQ(0, fs.mkdir("dir"));
 150     ASSERT_EQ(0, fs.open_for_write("dir", "file", &h, false));
 151     h->append("foo", 3);
 152     h->append("bar", 3);
 153     h->append("baz", 3);
 154     fs.fsync(h);
 155     fs.close_writer(h);
 156   }
 157   {
 158     BlueFS::FileReader *h;
 159     ASSERT_EQ(0, fs.open_for_read("dir", "file", &h));
 160     bufferlist bl;
 161     BlueFS::FileReaderBuffer buf(4096);
 162     ASSERT_EQ(9, fs.read(h, &buf, 0, 1024, &bl, NULL));
 163     ASSERT_EQ(0, strncmp("foobarbaz", bl.c_str(), 9));
 164     delete h;
 165   }
 166   fs.umount();
 167 }
 168
 169 TEST(BlueFS, small_appends) {
 170   uint64_t size = 1048576 * 128;
 171   TempBdev bdev{size};
 172   BlueFS fs(g_ceph_context);
 173   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 174   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 175   uuid_d fsid;
 176   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 177   ASSERT_EQ(0, fs.mount());
 178   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 179   {
 180     BlueFS::FileWriter *h;
 181     ASSERT_EQ(0, fs.mkdir("dir"));
 182     ASSERT_EQ(0, fs.open_for_write("dir", "file", &h, false));
 183     for (unsigned i = 0; i < 10000; ++i) {
 184       h->append("abcdeabcdeabcdeabcdeabcdeabc", 23);
 185     }
 186     fs.fsync(h);
 187     fs.close_writer(h);
 188   }
 189   {
 190     BlueFS::FileWriter *h;
 191     ASSERT_EQ(0, fs.open_for_write("dir", "file_sync", &h, false));
 192     for (unsigned i = 0; i < 1000; ++i) {
 193       h->append("abcdeabcdeabcdeabcdeabcdeabc", 23);
 194       ASSERT_EQ(0, fs.fsync(h));
 195     }
 196     fs.close_writer(h);
 197   }
 198   fs.umount();
 199 }
 200
 201 TEST(BlueFS, very_large_write) {
 202   // we'll write a ~3G file, so allocate more than that for the whole fs
 203   uint64_t size = 1048576 * 1024 * 8ull;
 204   TempBdev bdev{size};
 205   BlueFS fs(g_ceph_context);
 206
 207   bool old = g_ceph_context->_conf.get_val<bool>("bluefs_buffered_io");
 208   g_ceph_context->_conf.set_val("bluefs_buffered_io", "false");
 209
 210   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 211   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 212   uuid_d fsid;
 213   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 214   ASSERT_EQ(0, fs.mount());
 215   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 216   char buf[1048571]; // this is biggish, but intentionally not evenly aligned
 217   for (unsigned i = 0; i < sizeof(buf); ++i) {
 218     buf[i] = i;
 219   }
 220   {
 221     BlueFS::FileWriter *h;
 222     ASSERT_EQ(0, fs.mkdir("dir"));
 223     ASSERT_EQ(0, fs.open_for_write("dir", "bigfile", &h, false));
 224     for (unsigned i = 0; i < 3*1024*1048576ull / sizeof(buf); ++i) {
 225       h->append(buf, sizeof(buf));
 226     }
 227     fs.fsync(h);
 228     fs.close_writer(h);
 229   }
 230   {
 231     BlueFS::FileReader *h;
 232     ASSERT_EQ(0, fs.open_for_read("dir", "bigfile", &h));
 233     bufferlist bl;
 234     BlueFS::FileReaderBuffer readbuf(10485760);
 235     for (unsigned i = 0; i < 3*1024*1048576ull / sizeof(buf); ++i) {
 236       bl.clear();
 237       fs.read(h, &readbuf, i * sizeof(buf), sizeof(buf), &bl, NULL);
 238       int r = memcmp(buf, bl.c_str(), sizeof(buf));
 239       if (r) {
 240         cerr << "read got mismatch at offset " << i*sizeof(buf) << " r " << r
 241              << std::endl;
 242       }
 243       ASSERT_EQ(0, r);
 244     }
 245     delete h;
 246   }
 247   fs.umount();
 248
 249   g_ceph_context->_conf.set_val("bluefs_buffered_io", stringify((int)old));
 250 }
 251
 252 #define ALLOC_SIZE 4096
 253
 254 void write_data(BlueFS &fs, uint64_t rationed_bytes)
 255 {
 256     int j=0, r=0;
 257     uint64_t written_bytes = 0;
 258     rationed_bytes -= ALLOC_SIZE;
 259     stringstream ss;
 260     string dir = "dir.";
 261     ss << std::this_thread::get_id();
 262     dir.append(ss.str());
 263     dir.append(".");
 264     dir.append(to_string(j));
 265     ASSERT_EQ(0, fs.mkdir(dir));
 266     while (1) {
 267       string file = "file.";
 268       file.append(to_string(j));
 269       BlueFS::FileWriter *h;
 270       ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false));
 271       ASSERT_NE(nullptr, h);
 272       auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); });
 273       bufferlist bl;
 274       std::unique_ptr<char[]> buf = gen_buffer(ALLOC_SIZE);
 275       bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get());
 276       bl.push_back(bp);
 277       h->append(bl.c_str(), bl.length());
 278       r = fs.fsync(h);
 279       if (r < 0) {
 280          break;
 281       }
 282       written_bytes += g_conf()->bluefs_alloc_size;
 283       j++;
 284       if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) {
 285         break;
 286       }
 287     }
 288 }
 289
 290 void create_single_file(BlueFS &fs)
 291 {
 292     BlueFS::FileWriter *h;
 293     stringstream ss;
 294     string dir = "dir.test";
 295     ASSERT_EQ(0, fs.mkdir(dir));
 296     string file = "testfile";
 297     ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false));
 298     bufferlist bl;
 299     std::unique_ptr<char[]> buf = gen_buffer(ALLOC_SIZE);
 300     bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get());
 301     bl.push_back(bp);
 302     h->append(bl.c_str(), bl.length());
 303     fs.fsync(h);
 304     fs.close_writer(h);
 305 }
 306
 307 void write_single_file(BlueFS &fs, uint64_t rationed_bytes)
 308 {
 309     stringstream ss;
 310     const string dir = "dir.test";
 311     const string file = "testfile";
 312     uint64_t written_bytes = 0;
 313     rationed_bytes -= ALLOC_SIZE;
 314     while (1) {
 315       BlueFS::FileWriter *h;
 316       ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false));
 317       ASSERT_NE(nullptr, h);
 318       auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); });
 319       bufferlist bl;
 320       std::unique_ptr<char[]> buf = gen_buffer(ALLOC_SIZE);
 321       bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get());
 322       bl.push_back(bp);
 323       h->append(bl.c_str(), bl.length());
 324       int r = fs.fsync(h);
 325       if (r < 0) {
 326          break;
 327       }
 328       written_bytes += g_conf()->bluefs_alloc_size;
 329       if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) {
 330         break;
 331       }
 332     }
 333 }
 334
 335 bool writes_done = false;
 336
 337 void sync_fs(BlueFS &fs)
 338 {
 339     while (1) {
 340       if (writes_done == true)
 341         break;
 342       fs.sync_metadata(false);
 343       sleep(1);
 344     }
 345 }
 346
 347
 348 void do_join(std::thread& t)
 349 {
 350     t.join();
 351 }
 352
 353 void join_all(std::vector<std::thread>& v)
 354 {
 355     std::for_each(v.begin(),v.end(),do_join);
 356 }
 357
 358 #define NUM_WRITERS 3
 359 #define NUM_SYNC_THREADS 1
 360
 361 #define NUM_SINGLE_FILE_WRITERS 1
 362 #define NUM_MULTIPLE_FILE_WRITERS 2
 363
 364 TEST(BlueFS, test_flush_1) {
 365   uint64_t size = 1048576 * 128;
 366   TempBdev bdev{size};
 367   g_ceph_context->_conf.set_val(
 368     "bluefs_alloc_size",
 369     "65536");
 370   g_ceph_context->_conf.apply_changes(nullptr);
 371
 372   BlueFS fs(g_ceph_context);
 373   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 374   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 375   uuid_d fsid;
 376   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 377   ASSERT_EQ(0, fs.mount());
 378   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 379   {
 380     std::vector<std::thread> write_thread_multiple;
 381     uint64_t effective_size = size - (32 * 1048576); // leaving the last 32 MB for log compaction
 382     uint64_t per_thread_bytes = (effective_size/(NUM_MULTIPLE_FILE_WRITERS + NUM_SINGLE_FILE_WRITERS));
 383     for (int i=0; i<NUM_MULTIPLE_FILE_WRITERS ; i++) {
 384       write_thread_multiple.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 385     }
 386
 387     create_single_file(fs);
 388     std::vector<std::thread> write_thread_single;
 389     for (int i=0; i<NUM_SINGLE_FILE_WRITERS; i++) {
 390       write_thread_single.push_back(std::thread(write_single_file, std::ref(fs), per_thread_bytes));
 391     }
 392
 393     join_all(write_thread_single);
 394     join_all(write_thread_multiple);
 395   }
 396   fs.umount();
 397 }
 398
 399 TEST(BlueFS, test_flush_2) {
 400   uint64_t size = 1048576 * 256;
 401   TempBdev bdev{size};
 402   g_ceph_context->_conf.set_val(
 403     "bluefs_alloc_size",
 404     "65536");
 405   g_ceph_context->_conf.apply_changes(nullptr);
 406
 407   BlueFS fs(g_ceph_context);
 408   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 409   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 410   uuid_d fsid;
 411   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 412   ASSERT_EQ(0, fs.mount());
 413   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 414   {
 415     uint64_t effective_size = size - (128 * 1048576); // leaving the last 32 MB for log compaction
 416     uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS));
 417     std::vector<std::thread> write_thread_multiple;
 418     for (int i=0; i<NUM_WRITERS; i++) {
 419       write_thread_multiple.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 420     }
 421
 422     join_all(write_thread_multiple);
 423   }
 424   fs.umount();
 425 }
 426
 427 TEST(BlueFS, test_flush_3) {
 428   uint64_t size = 1048576 * 256;
 429   TempBdev bdev{size};
 430   g_ceph_context->_conf.set_val(
 431     "bluefs_alloc_size",
 432     "65536");
 433   g_ceph_context->_conf.apply_changes(nullptr);
 434
 435   BlueFS fs(g_ceph_context);
 436   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 437   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 438   uuid_d fsid;
 439   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 440   ASSERT_EQ(0, fs.mount());
 441   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 442   {
 443     std::vector<std::thread> write_threads;
 444     uint64_t effective_size = size - (64 * 1048576); // leaving the last 11 MB for log compaction
 445     uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS));
 446     for (int i=0; i<NUM_WRITERS; i++) {
 447       write_threads.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 448     }
 449
 450     std::vector<std::thread> sync_threads;
 451     for (int i=0; i<NUM_SYNC_THREADS; i++) {
 452       sync_threads.push_back(std::thread(sync_fs, std::ref(fs)));
 453     }
 454
 455     join_all(write_threads);
 456     writes_done = true;
 457     join_all(sync_threads);
 458   }
 459   fs.umount();
 460 }
 461
 462 TEST(BlueFS, test_simple_compaction_sync) {
 463   g_ceph_context->_conf.set_val(
 464     "bluefs_compact_log_sync",
 465     "true");
 466   uint64_t size = 1048576 * 128;
 467   TempBdev bdev{size};
 468
 469   BlueFS fs(g_ceph_context);
 470   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 471   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 472   uuid_d fsid;
 473   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 474   ASSERT_EQ(0, fs.mount());
 475   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 476   {
 477     for (int i=0; i<10; i++) {
 478        string dir = "dir.";
 479        dir.append(to_string(i));
 480        ASSERT_EQ(0, fs.mkdir(dir));
 481        for (int j=0; j<10; j++) {
 482           string file = "file.";
 483           file.append(to_string(j));
 484           BlueFS::FileWriter *h;
 485           ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false));
 486           ASSERT_NE(nullptr, h);
 487           auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); });
 488           bufferlist bl;
 489           std::unique_ptr<char[]> buf = gen_buffer(4096);
 490           bufferptr bp = buffer::claim_char(4096, buf.get());
 491           bl.push_back(bp);
 492           h->append(bl.c_str(), bl.length());
 493           fs.fsync(h);
 494        }
 495     }
 496   }
 497   {
 498     for (int i=0; i<10; i+=2) {
 499        string dir = "dir.";
 500        dir.append(to_string(i));
 501        for (int j=0; j<10; j++) {
 502           string file = "file.";
 503           file.append(to_string(j));
 504           fs.unlink(dir, file);
 505           fs.sync_metadata(false);
 506        }
 507        ASSERT_EQ(0, fs.rmdir(dir));
 508        fs.sync_metadata(false);
 509     }
 510   }
 511   fs.compact_log();
 512   fs.umount();
 513 }
 514
 515 TEST(BlueFS, test_simple_compaction_async) {
 516   g_ceph_context->_conf.set_val(
 517     "bluefs_compact_log_sync",
 518     "false");
 519   uint64_t size = 1048576 * 128;
 520   TempBdev bdev{size};
 521
 522   BlueFS fs(g_ceph_context);
 523   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 524   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 525   uuid_d fsid;
 526   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 527   ASSERT_EQ(0, fs.mount());
 528   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 529   {
 530     for (int i=0; i<10; i++) {
 531        string dir = "dir.";
 532        dir.append(to_string(i));
 533        ASSERT_EQ(0, fs.mkdir(dir));
 534        for (int j=0; j<10; j++) {
 535           string file = "file.";
 536           file.append(to_string(j));
 537           BlueFS::FileWriter *h;
 538           ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false));
 539           ASSERT_NE(nullptr, h);
 540           auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); });
 541           bufferlist bl;
 542           std::unique_ptr<char[]> buf = gen_buffer(4096);
 543           bufferptr bp = buffer::claim_char(4096, buf.get());
 544           bl.push_back(bp);
 545           h->append(bl.c_str(), bl.length());
 546           fs.fsync(h);
 547        }
 548     }
 549   }
 550   {
 551     for (int i=0; i<10; i+=2) {
 552        string dir = "dir.";
 553        dir.append(to_string(i));
 554        for (int j=0; j<10; j++) {
 555           string file = "file.";
 556           file.append(to_string(j));
 557           fs.unlink(dir, file);
 558           fs.sync_metadata(false);
 559        }
 560        ASSERT_EQ(0, fs.rmdir(dir));
 561        fs.sync_metadata(false);
 562     }
 563   }
 564   fs.compact_log();
 565   fs.umount();
 566 }
 567
 568 TEST(BlueFS, test_compaction_sync) {
 569   uint64_t size = 1048576 * 128;
 570   TempBdev bdev{size};
 571   g_ceph_context->_conf.set_val(
 572     "bluefs_alloc_size",
 573     "65536");
 574   g_ceph_context->_conf.set_val(
 575     "bluefs_compact_log_sync",
 576     "true");
 577
 578   BlueFS fs(g_ceph_context);
 579   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 580   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 581   uuid_d fsid;
 582   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 583   ASSERT_EQ(0, fs.mount());
 584   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 585   {
 586     std::vector<std::thread> write_threads;
 587     uint64_t effective_size = size - (32 * 1048576); // leaving the last 32 MB for log compaction
 588     uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS));
 589     for (int i=0; i<NUM_WRITERS; i++) {
 590       write_threads.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 591     }
 592
 593     std::vector<std::thread> sync_threads;
 594     for (int i=0; i<NUM_SYNC_THREADS; i++) {
 595       sync_threads.push_back(std::thread(sync_fs, std::ref(fs)));
 596     }
 597
 598     join_all(write_threads);
 599     writes_done = true;
 600     join_all(sync_threads);
 601     fs.compact_log();
 602   }
 603   fs.umount();
 604 }
 605
 606 TEST(BlueFS, test_compaction_async) {
 607   uint64_t size = 1048576 * 128;
 608   TempBdev bdev{size};
 609   g_ceph_context->_conf.set_val(
 610     "bluefs_alloc_size",
 611     "65536");
 612   g_ceph_context->_conf.set_val(
 613     "bluefs_compact_log_sync",
 614     "false");
 615
 616   BlueFS fs(g_ceph_context);
 617   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 618   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 619   uuid_d fsid;
 620   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 621   ASSERT_EQ(0, fs.mount());
 622   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 623   {
 624     std::vector<std::thread> write_threads;
 625     uint64_t effective_size = size - (32 * 1048576); // leaving the last 32 MB for log compaction
 626     uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS));
 627     for (int i=0; i<NUM_WRITERS; i++) {
 628       write_threads.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 629     }
 630
 631     std::vector<std::thread> sync_threads;
 632     for (int i=0; i<NUM_SYNC_THREADS; i++) {
 633       sync_threads.push_back(std::thread(sync_fs, std::ref(fs)));
 634     }
 635
 636     join_all(write_threads);
 637     writes_done = true;
 638     join_all(sync_threads);
 639     fs.compact_log();
 640   }
 641   fs.umount();
 642 }
 643
 644 TEST(BlueFS, test_replay) {
 645   uint64_t size = 1048576 * 128;
 646   TempBdev bdev{size};
 647   g_ceph_context->_conf.set_val(
 648     "bluefs_alloc_size",
 649     "65536");
 650   g_ceph_context->_conf.set_val(
 651     "bluefs_compact_log_sync",
 652     "false");
 653
 654   BlueFS fs(g_ceph_context);
 655   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 656   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 657   uuid_d fsid;
 658   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 659   ASSERT_EQ(0, fs.mount());
 660   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 661   {
 662     std::vector<std::thread> write_threads;
 663     uint64_t effective_size = size - (32 * 1048576); // leaving the last 32 MB for log compaction
 664     uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS));
 665     for (int i=0; i<NUM_WRITERS; i++) {
 666       write_threads.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 667     }
 668
 669     std::vector<std::thread> sync_threads;
 670     for (int i=0; i<NUM_SYNC_THREADS; i++) {
 671       sync_threads.push_back(std::thread(sync_fs, std::ref(fs)));
 672     }
 673
 674     join_all(write_threads);
 675     writes_done = true;
 676     join_all(sync_threads);
 677     fs.compact_log();
 678   }
 679   fs.umount();
 680   // remount and check log can replay safe?
 681   ASSERT_EQ(0, fs.mount());
 682   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 683   fs.umount();
 684 }
 685
 686 int main(int argc, char **argv) {
 687   vector<const char*> args;
 688   argv_to_vec(argc, (const char **)argv, args);
 689
 690   map<string,string> defaults = {
 691     { "debug_bluefs", "1/20" },
 692     { "debug_bdev", "1/20" }
 693   };
 694
 695   auto cct = global_init(&defaults, args, CEPH_ENTITY_TYPE_CLIENT,
 696                          CODE_ENVIRONMENT_UTILITY,
 697                          CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
 698   common_init_finish(g_ceph_context);
 699   g_ceph_context->_conf.set_val(
 700     "enable_experimental_unrecoverable_data_corrupting_features",
 701     "*");
 702   g_ceph_context->_conf.apply_changes(nullptr);
 703
 704   ::testing::InitGoogleTest(&argc, argv);
 705   return RUN_ALL_TESTS();
 706 }