ceph/src/test/objectstore/test_bluefs.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include <stdio.h>
   5 #include <string.h>
   6 #include <iostream>
   7 #include <time.h>
   8 #include <fcntl.h>
   9 #include <unistd.h>
  10 #include <random>
  11 #include <thread>
  12 #include "global/global_init.h"
  13 #include "common/ceph_argparse.h"
  14 #include "include/stringify.h"
  15 #include "include/scope_guard.h"
  16 #include "common/errno.h"
  17 #include <gtest/gtest.h>
  18
  19 #include "os/bluestore/BlueFS.h"
  20
  21 std::unique_ptr<char[]> gen_buffer(uint64_t size)
  22 {
  23     std::unique_ptr<char[]> buffer = std::make_unique<char[]>(size);
  24     std::independent_bits_engine<std::default_random_engine, CHAR_BIT, unsigned char> e;
  25     std::generate(buffer.get(), buffer.get()+size, std::ref(e));
  26     return buffer;
  27 }
  28
  29 class TempBdev {
  30 public:
  31   TempBdev(uint64_t size)
  32     : path{get_temp_bdev(size)}
  33   {}
  34   ~TempBdev() {
  35     rm_temp_bdev(path);
  36   }
  37   const std::string path;
  38 private:
  39   static string get_temp_bdev(uint64_t size)
  40   {
  41     static int n = 0;
  42     string fn = "ceph_test_bluefs.tmp.block." + stringify(getpid())
  43       + "." + stringify(++n);
  44     int fd = ::open(fn.c_str(), O_CREAT|O_RDWR|O_TRUNC, 0644);
  45     ceph_assert(fd >= 0);
  46     int r = ::ftruncate(fd, size);
  47     ceph_assert(r >= 0);
  48     ::close(fd);
  49     return fn;
  50   }
  51   static void rm_temp_bdev(string f)
  52   {
  53     ::unlink(f.c_str());
  54   }
  55 };
  56
  57 TEST(BlueFS, mkfs) {
  58   uint64_t size = 1048576 * 128;
  59   TempBdev bdev{size};
  60   uuid_d fsid;
  61   BlueFS fs(g_ceph_context);
  62   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
  63   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
  64   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
  65 }
  66
  67 TEST(BlueFS, mkfs_mount) {
  68   uint64_t size = 1048576 * 128;
  69   TempBdev bdev{size};
  70   BlueFS fs(g_ceph_context);
  71   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
  72   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
  73   uuid_d fsid;
  74   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
  75   ASSERT_EQ(0, fs.mount());
  76   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
  77   ASSERT_EQ(fs.get_total(BlueFS::BDEV_DB), size - 1048576);
  78   ASSERT_LT(fs.get_free(BlueFS::BDEV_DB), size - 1048576);
  79   fs.umount();
  80 }
  81
  82 TEST(BlueFS, mkfs_mount_duplicate_gift) {
  83   uint64_t size = 1048576 * 128;
  84   TempBdev bdev{ size };
  85   bluefs_extent_t dup_ext;
  86   {
  87     BlueFS fs(g_ceph_context);
  88     ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
  89     fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
  90     uuid_d fsid;
  91     ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
  92     ASSERT_EQ(0, fs.mount());
  93
  94     {
  95       BlueFS::FileWriter *h;
  96       ASSERT_EQ(0, fs.mkdir("dir"));
  97       ASSERT_EQ(0, fs.open_for_write("dir", "file1", &h, false));
  98       h->append("foo", 3);
  99       h->append("bar", 3);
 100       h->append("baz", 3);
 101       fs.fsync(h);
 102       ceph_assert(h->file->fnode.extents.size() > 0);
 103       dup_ext = h->file->fnode.extents[0];
 104       ceph_assert(dup_ext.bdev == BlueFS::BDEV_DB);
 105       fs.close_writer(h);
 106     }
 107
 108     fs.umount();
 109   }
 110
 111   {
 112     BlueFS fs(g_ceph_context);
 113     ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 114     ASSERT_EQ(0, fs.mount());
 115     // free allocation presumably allocated for file1
 116     std::cout << "duplicate extent: " << std::hex
 117       << dup_ext.offset << "~" << dup_ext.length
 118       << std::dec << std::endl;
 119     fs.debug_inject_duplicate_gift(BlueFS::BDEV_DB, dup_ext.offset, dup_ext.length);
 120     {
 121       // overwrite file1 with file2
 122       BlueFS::FileWriter *h;
 123       ASSERT_EQ(0, fs.open_for_write("dir", "file2", &h, false));
 124       h->append("foo", 3);
 125       h->append("bar", 3);
 126       h->append("baz", 3);
 127       fs.fsync(h);
 128       fs.close_writer(h);
 129     }
 130     fs.umount();
 131   }
 132
 133   g_ceph_context->_conf.set_val_or_die("bluefs_log_replay_check_allocations", "true");
 134   g_ceph_context->_conf.apply_changes(nullptr);
 135
 136   {
 137     // this should fail
 138     BlueFS fs(g_ceph_context);
 139     ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 140     ASSERT_NE(0, fs.mount());
 141   }
 142 }
 143
 144 TEST(BlueFS, write_read) {
 145   uint64_t size = 1048576 * 128;
 146   TempBdev bdev{size};
 147   BlueFS fs(g_ceph_context);
 148   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 149   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 150   uuid_d fsid;
 151   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 152   ASSERT_EQ(0, fs.mount());
 153   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 154   {
 155     BlueFS::FileWriter *h;
 156     ASSERT_EQ(0, fs.mkdir("dir"));
 157     ASSERT_EQ(0, fs.open_for_write("dir", "file", &h, false));
 158     h->append("foo", 3);
 159     h->append("bar", 3);
 160     h->append("baz", 3);
 161     fs.fsync(h);
 162     fs.close_writer(h);
 163   }
 164   {
 165     BlueFS::FileReader *h;
 166     ASSERT_EQ(0, fs.open_for_read("dir", "file", &h));
 167     bufferlist bl;
 168     BlueFS::FileReaderBuffer buf(4096);
 169     ASSERT_EQ(9, fs.read(h, &buf, 0, 1024, &bl, NULL));
 170     ASSERT_EQ(0, strncmp("foobarbaz", bl.c_str(), 9));
 171     delete h;
 172   }
 173   fs.umount();
 174 }
 175
 176 TEST(BlueFS, small_appends) {
 177   uint64_t size = 1048576 * 128;
 178   TempBdev bdev{size};
 179   BlueFS fs(g_ceph_context);
 180   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 181   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 182   uuid_d fsid;
 183   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 184   ASSERT_EQ(0, fs.mount());
 185   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 186   {
 187     BlueFS::FileWriter *h;
 188     ASSERT_EQ(0, fs.mkdir("dir"));
 189     ASSERT_EQ(0, fs.open_for_write("dir", "file", &h, false));
 190     for (unsigned i = 0; i < 10000; ++i) {
 191       h->append("abcdeabcdeabcdeabcdeabcdeabc", 23);
 192     }
 193     fs.fsync(h);
 194     fs.close_writer(h);
 195   }
 196   {
 197     BlueFS::FileWriter *h;
 198     ASSERT_EQ(0, fs.open_for_write("dir", "file_sync", &h, false));
 199     for (unsigned i = 0; i < 1000; ++i) {
 200       h->append("abcdeabcdeabcdeabcdeabcdeabc", 23);
 201       ASSERT_EQ(0, fs.fsync(h));
 202     }
 203     fs.close_writer(h);
 204   }
 205   fs.umount();
 206 }
 207
 208 TEST(BlueFS, very_large_write) {
 209   // we'll write a ~3G file, so allocate more than that for the whole fs
 210   uint64_t size = 1048576 * 1024 * 8ull;
 211   TempBdev bdev{size};
 212   BlueFS fs(g_ceph_context);
 213
 214   bool old = g_ceph_context->_conf.get_val<bool>("bluefs_buffered_io");
 215   g_ceph_context->_conf.set_val("bluefs_buffered_io", "false");
 216
 217   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 218   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 219   uuid_d fsid;
 220   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 221   ASSERT_EQ(0, fs.mount());
 222   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 223   char buf[1048571]; // this is biggish, but intentionally not evenly aligned
 224   for (unsigned i = 0; i < sizeof(buf); ++i) {
 225     buf[i] = i;
 226   }
 227   {
 228     BlueFS::FileWriter *h;
 229     ASSERT_EQ(0, fs.mkdir("dir"));
 230     ASSERT_EQ(0, fs.open_for_write("dir", "bigfile", &h, false));
 231     for (unsigned i = 0; i < 3*1024*1048576ull / sizeof(buf); ++i) {
 232       h->append(buf, sizeof(buf));
 233     }
 234     fs.fsync(h);
 235     fs.close_writer(h);
 236   }
 237   {
 238     BlueFS::FileReader *h;
 239     ASSERT_EQ(0, fs.open_for_read("dir", "bigfile", &h));
 240     bufferlist bl;
 241     BlueFS::FileReaderBuffer readbuf(10485760);
 242     for (unsigned i = 0; i < 3*1024*1048576ull / sizeof(buf); ++i) {
 243       bl.clear();
 244       fs.read(h, &readbuf, i * sizeof(buf), sizeof(buf), &bl, NULL);
 245       int r = memcmp(buf, bl.c_str(), sizeof(buf));
 246       if (r) {
 247         cerr << "read got mismatch at offset " << i*sizeof(buf) << " r " << r
 248              << std::endl;
 249       }
 250       ASSERT_EQ(0, r);
 251     }
 252     delete h;
 253   }
 254   fs.umount();
 255
 256   g_ceph_context->_conf.set_val("bluefs_buffered_io", stringify((int)old));
 257 }
 258
 259 #define ALLOC_SIZE 4096
 260
 261 void write_data(BlueFS &fs, uint64_t rationed_bytes)
 262 {
 263     int j=0, r=0;
 264     uint64_t written_bytes = 0;
 265     rationed_bytes -= ALLOC_SIZE;
 266     stringstream ss;
 267     string dir = "dir.";
 268     ss << std::this_thread::get_id();
 269     dir.append(ss.str());
 270     dir.append(".");
 271     dir.append(to_string(j));
 272     ASSERT_EQ(0, fs.mkdir(dir));
 273     while (1) {
 274       string file = "file.";
 275       file.append(to_string(j));
 276       BlueFS::FileWriter *h;
 277       ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false));
 278       ASSERT_NE(nullptr, h);
 279       auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); });
 280       bufferlist bl;
 281       std::unique_ptr<char[]> buf = gen_buffer(ALLOC_SIZE);
 282       bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get());
 283       bl.push_back(bp);
 284       h->append(bl.c_str(), bl.length());
 285       r = fs.fsync(h);
 286       if (r < 0) {
 287          break;
 288       }
 289       written_bytes += g_conf()->bluefs_alloc_size;
 290       j++;
 291       if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) {
 292         break;
 293       }
 294     }
 295 }
 296
 297 void create_single_file(BlueFS &fs)
 298 {
 299     BlueFS::FileWriter *h;
 300     stringstream ss;
 301     string dir = "dir.test";
 302     ASSERT_EQ(0, fs.mkdir(dir));
 303     string file = "testfile";
 304     ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false));
 305     bufferlist bl;
 306     std::unique_ptr<char[]> buf = gen_buffer(ALLOC_SIZE);
 307     bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get());
 308     bl.push_back(bp);
 309     h->append(bl.c_str(), bl.length());
 310     fs.fsync(h);
 311     fs.close_writer(h);
 312 }
 313
 314 void write_single_file(BlueFS &fs, uint64_t rationed_bytes)
 315 {
 316     stringstream ss;
 317     const string dir = "dir.test";
 318     const string file = "testfile";
 319     uint64_t written_bytes = 0;
 320     rationed_bytes -= ALLOC_SIZE;
 321     while (1) {
 322       BlueFS::FileWriter *h;
 323       ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false));
 324       ASSERT_NE(nullptr, h);
 325       auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); });
 326       bufferlist bl;
 327       std::unique_ptr<char[]> buf = gen_buffer(ALLOC_SIZE);
 328       bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get());
 329       bl.push_back(bp);
 330       h->append(bl.c_str(), bl.length());
 331       int r = fs.fsync(h);
 332       if (r < 0) {
 333          break;
 334       }
 335       written_bytes += g_conf()->bluefs_alloc_size;
 336       if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) {
 337         break;
 338       }
 339     }
 340 }
 341
 342 bool writes_done = false;
 343
 344 void sync_fs(BlueFS &fs)
 345 {
 346     while (1) {
 347       if (writes_done == true)
 348         break;
 349       fs.sync_metadata(false);
 350       sleep(1);
 351     }
 352 }
 353
 354
 355 void do_join(std::thread& t)
 356 {
 357     t.join();
 358 }
 359
 360 void join_all(std::vector<std::thread>& v)
 361 {
 362     std::for_each(v.begin(),v.end(),do_join);
 363 }
 364
 365 #define NUM_WRITERS 3
 366 #define NUM_SYNC_THREADS 1
 367
 368 #define NUM_SINGLE_FILE_WRITERS 1
 369 #define NUM_MULTIPLE_FILE_WRITERS 2
 370
 371 TEST(BlueFS, test_flush_1) {
 372   uint64_t size = 1048576 * 128;
 373   TempBdev bdev{size};
 374   g_ceph_context->_conf.set_val(
 375     "bluefs_alloc_size",
 376     "65536");
 377   g_ceph_context->_conf.apply_changes(nullptr);
 378
 379   BlueFS fs(g_ceph_context);
 380   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 381   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 382   uuid_d fsid;
 383   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 384   ASSERT_EQ(0, fs.mount());
 385   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 386   {
 387     std::vector<std::thread> write_thread_multiple;
 388     uint64_t effective_size = size - (32 * 1048576); // leaving the last 32 MB for log compaction
 389     uint64_t per_thread_bytes = (effective_size/(NUM_MULTIPLE_FILE_WRITERS + NUM_SINGLE_FILE_WRITERS));
 390     for (int i=0; i<NUM_MULTIPLE_FILE_WRITERS ; i++) {
 391       write_thread_multiple.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 392     }
 393
 394     create_single_file(fs);
 395     std::vector<std::thread> write_thread_single;
 396     for (int i=0; i<NUM_SINGLE_FILE_WRITERS; i++) {
 397       write_thread_single.push_back(std::thread(write_single_file, std::ref(fs), per_thread_bytes));
 398     }
 399
 400     join_all(write_thread_single);
 401     join_all(write_thread_multiple);
 402   }
 403   fs.umount();
 404 }
 405
 406 TEST(BlueFS, test_flush_2) {
 407   uint64_t size = 1048576 * 256;
 408   TempBdev bdev{size};
 409   g_ceph_context->_conf.set_val(
 410     "bluefs_alloc_size",
 411     "65536");
 412   g_ceph_context->_conf.apply_changes(nullptr);
 413
 414   BlueFS fs(g_ceph_context);
 415   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 416   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 417   uuid_d fsid;
 418   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 419   ASSERT_EQ(0, fs.mount());
 420   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 421   {
 422     uint64_t effective_size = size - (128 * 1048576); // leaving the last 32 MB for log compaction
 423     uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS));
 424     std::vector<std::thread> write_thread_multiple;
 425     for (int i=0; i<NUM_WRITERS; i++) {
 426       write_thread_multiple.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 427     }
 428
 429     join_all(write_thread_multiple);
 430   }
 431   fs.umount();
 432 }
 433
 434 TEST(BlueFS, test_flush_3) {
 435   uint64_t size = 1048576 * 256;
 436   TempBdev bdev{size};
 437   g_ceph_context->_conf.set_val(
 438     "bluefs_alloc_size",
 439     "65536");
 440   g_ceph_context->_conf.apply_changes(nullptr);
 441
 442   BlueFS fs(g_ceph_context);
 443   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 444   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 445   uuid_d fsid;
 446   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 447   ASSERT_EQ(0, fs.mount());
 448   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 449   {
 450     std::vector<std::thread> write_threads;
 451     uint64_t effective_size = size - (64 * 1048576); // leaving the last 11 MB for log compaction
 452     uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS));
 453     for (int i=0; i<NUM_WRITERS; i++) {
 454       write_threads.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 455     }
 456
 457     std::vector<std::thread> sync_threads;
 458     for (int i=0; i<NUM_SYNC_THREADS; i++) {
 459       sync_threads.push_back(std::thread(sync_fs, std::ref(fs)));
 460     }
 461
 462     join_all(write_threads);
 463     writes_done = true;
 464     join_all(sync_threads);
 465   }
 466   fs.umount();
 467 }
 468
 469 TEST(BlueFS, test_simple_compaction_sync) {
 470   g_ceph_context->_conf.set_val(
 471     "bluefs_compact_log_sync",
 472     "true");
 473   uint64_t size = 1048576 * 128;
 474   TempBdev bdev{size};
 475
 476   BlueFS fs(g_ceph_context);
 477   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 478   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 479   uuid_d fsid;
 480   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 481   ASSERT_EQ(0, fs.mount());
 482   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 483   {
 484     for (int i=0; i<10; i++) {
 485        string dir = "dir.";
 486        dir.append(to_string(i));
 487        ASSERT_EQ(0, fs.mkdir(dir));
 488        for (int j=0; j<10; j++) {
 489           string file = "file.";
 490           file.append(to_string(j));
 491           BlueFS::FileWriter *h;
 492           ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false));
 493           ASSERT_NE(nullptr, h);
 494           auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); });
 495           bufferlist bl;
 496           std::unique_ptr<char[]> buf = gen_buffer(4096);
 497           bufferptr bp = buffer::claim_char(4096, buf.get());
 498           bl.push_back(bp);
 499           h->append(bl.c_str(), bl.length());
 500           fs.fsync(h);
 501        }
 502     }
 503   }
 504   {
 505     for (int i=0; i<10; i+=2) {
 506        string dir = "dir.";
 507        dir.append(to_string(i));
 508        for (int j=0; j<10; j++) {
 509           string file = "file.";
 510           file.append(to_string(j));
 511           fs.unlink(dir, file);
 512           fs.sync_metadata(false);
 513        }
 514        ASSERT_EQ(0, fs.rmdir(dir));
 515        fs.sync_metadata(false);
 516     }
 517   }
 518   fs.compact_log();
 519   fs.umount();
 520 }
 521
 522 TEST(BlueFS, test_simple_compaction_async) {
 523   g_ceph_context->_conf.set_val(
 524     "bluefs_compact_log_sync",
 525     "false");
 526   uint64_t size = 1048576 * 128;
 527   TempBdev bdev{size};
 528
 529   BlueFS fs(g_ceph_context);
 530   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 531   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 532   uuid_d fsid;
 533   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 534   ASSERT_EQ(0, fs.mount());
 535   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 536   {
 537     for (int i=0; i<10; i++) {
 538        string dir = "dir.";
 539        dir.append(to_string(i));
 540        ASSERT_EQ(0, fs.mkdir(dir));
 541        for (int j=0; j<10; j++) {
 542           string file = "file.";
 543           file.append(to_string(j));
 544           BlueFS::FileWriter *h;
 545           ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false));
 546           ASSERT_NE(nullptr, h);
 547           auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); });
 548           bufferlist bl;
 549           std::unique_ptr<char[]> buf = gen_buffer(4096);
 550           bufferptr bp = buffer::claim_char(4096, buf.get());
 551           bl.push_back(bp);
 552           h->append(bl.c_str(), bl.length());
 553           fs.fsync(h);
 554        }
 555     }
 556   }
 557   {
 558     for (int i=0; i<10; i+=2) {
 559        string dir = "dir.";
 560        dir.append(to_string(i));
 561        for (int j=0; j<10; j++) {
 562           string file = "file.";
 563           file.append(to_string(j));
 564           fs.unlink(dir, file);
 565           fs.sync_metadata(false);
 566        }
 567        ASSERT_EQ(0, fs.rmdir(dir));
 568        fs.sync_metadata(false);
 569     }
 570   }
 571   fs.compact_log();
 572   fs.umount();
 573 }
 574
 575 TEST(BlueFS, test_compaction_sync) {
 576   uint64_t size = 1048576 * 128;
 577   TempBdev bdev{size};
 578   g_ceph_context->_conf.set_val(
 579     "bluefs_alloc_size",
 580     "65536");
 581   g_ceph_context->_conf.set_val(
 582     "bluefs_compact_log_sync",
 583     "true");
 584
 585   BlueFS fs(g_ceph_context);
 586   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 587   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 588   uuid_d fsid;
 589   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 590   ASSERT_EQ(0, fs.mount());
 591   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 592   {
 593     std::vector<std::thread> write_threads;
 594     uint64_t effective_size = size - (32 * 1048576); // leaving the last 32 MB for log compaction
 595     uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS));
 596     for (int i=0; i<NUM_WRITERS; i++) {
 597       write_threads.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 598     }
 599
 600     std::vector<std::thread> sync_threads;
 601     for (int i=0; i<NUM_SYNC_THREADS; i++) {
 602       sync_threads.push_back(std::thread(sync_fs, std::ref(fs)));
 603     }
 604
 605     join_all(write_threads);
 606     writes_done = true;
 607     join_all(sync_threads);
 608     fs.compact_log();
 609   }
 610   fs.umount();
 611 }
 612
 613 TEST(BlueFS, test_compaction_async) {
 614   uint64_t size = 1048576 * 128;
 615   TempBdev bdev{size};
 616   g_ceph_context->_conf.set_val(
 617     "bluefs_alloc_size",
 618     "65536");
 619   g_ceph_context->_conf.set_val(
 620     "bluefs_compact_log_sync",
 621     "false");
 622
 623   BlueFS fs(g_ceph_context);
 624   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 625   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 626   uuid_d fsid;
 627   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 628   ASSERT_EQ(0, fs.mount());
 629   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 630   {
 631     std::vector<std::thread> write_threads;
 632     uint64_t effective_size = size - (32 * 1048576); // leaving the last 32 MB for log compaction
 633     uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS));
 634     for (int i=0; i<NUM_WRITERS; i++) {
 635       write_threads.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 636     }
 637
 638     std::vector<std::thread> sync_threads;
 639     for (int i=0; i<NUM_SYNC_THREADS; i++) {
 640       sync_threads.push_back(std::thread(sync_fs, std::ref(fs)));
 641     }
 642
 643     join_all(write_threads);
 644     writes_done = true;
 645     join_all(sync_threads);
 646     fs.compact_log();
 647   }
 648   fs.umount();
 649 }
 650
 651 TEST(BlueFS, test_replay) {
 652   uint64_t size = 1048576 * 128;
 653   TempBdev bdev{size};
 654   g_ceph_context->_conf.set_val(
 655     "bluefs_alloc_size",
 656     "65536");
 657   g_ceph_context->_conf.set_val(
 658     "bluefs_compact_log_sync",
 659     "false");
 660
 661   BlueFS fs(g_ceph_context);
 662   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 663   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 664   uuid_d fsid;
 665   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 666   ASSERT_EQ(0, fs.mount());
 667   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 668   {
 669     std::vector<std::thread> write_threads;
 670     uint64_t effective_size = size - (32 * 1048576); // leaving the last 32 MB for log compaction
 671     uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS));
 672     for (int i=0; i<NUM_WRITERS; i++) {
 673       write_threads.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 674     }
 675
 676     std::vector<std::thread> sync_threads;
 677     for (int i=0; i<NUM_SYNC_THREADS; i++) {
 678       sync_threads.push_back(std::thread(sync_fs, std::ref(fs)));
 679     }
 680
 681     join_all(write_threads);
 682     writes_done = true;
 683     join_all(sync_threads);
 684     fs.compact_log();
 685   }
 686   fs.umount();
 687   // remount and check log can replay safe?
 688   ASSERT_EQ(0, fs.mount());
 689   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 690   fs.umount();
 691 }
 692
 693 int main(int argc, char **argv) {
 694   vector<const char*> args;
 695   argv_to_vec(argc, (const char **)argv, args);
 696
 697   map<string,string> defaults = {
 698     { "debug_bluefs", "1/20" },
 699     { "debug_bdev", "1/20" }
 700   };
 701
 702   auto cct = global_init(&defaults, args, CEPH_ENTITY_TYPE_CLIENT,
 703                          CODE_ENVIRONMENT_UTILITY,
 704                          CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
 705   common_init_finish(g_ceph_context);
 706   g_ceph_context->_conf.set_val(
 707     "enable_experimental_unrecoverable_data_corrupting_features",
 708     "*");
 709   g_ceph_context->_conf.apply_changes(nullptr);
 710
 711   ::testing::InitGoogleTest(&argc, argv);
 712   return RUN_ALL_TESTS();
 713 }