ceph/src/test/objectstore/test_bluefs.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include <stdio.h>
   5 #include <string.h>
   6 #include <iostream>
   7 #include <time.h>
   8 #include <fcntl.h>
   9 #include <unistd.h>
  10 #include <random>
  11 #include <thread>
  12 #include <stack>
  13 #include "global/global_init.h"
  14 #include "common/ceph_argparse.h"
  15 #include "include/stringify.h"
  16 #include "include/scope_guard.h"
  17 #include "common/errno.h"
  18 #include <gtest/gtest.h>
  19
  20 #include "os/bluestore/BlueFS.h"
  21
  22 std::unique_ptr<char[]> gen_buffer(uint64_t size)
  23 {
  24     std::unique_ptr<char[]> buffer = std::make_unique<char[]>(size);
  25     std::independent_bits_engine<std::default_random_engine, CHAR_BIT, unsigned char> e;
  26     std::generate(buffer.get(), buffer.get()+size, std::ref(e));
  27     return buffer;
  28 }
  29
  30 class TempBdev {
  31 public:
  32   TempBdev(uint64_t size)
  33     : path{get_temp_bdev(size)}
  34   {}
  35   ~TempBdev() {
  36     rm_temp_bdev(path);
  37   }
  38   const std::string path;
  39 private:
  40   static string get_temp_bdev(uint64_t size)
  41   {
  42     static int n = 0;
  43     string fn = "ceph_test_bluefs.tmp.block." + stringify(getpid())
  44       + "." + stringify(++n);
  45     int fd = ::open(fn.c_str(), O_CREAT|O_RDWR|O_TRUNC, 0644);
  46     ceph_assert(fd >= 0);
  47     int r = ::ftruncate(fd, size);
  48     ceph_assert(r >= 0);
  49     ::close(fd);
  50     return fn;
  51   }
  52   static void rm_temp_bdev(string f)
  53   {
  54     ::unlink(f.c_str());
  55   }
  56 };
  57
  58 class ConfSaver {
  59   std::stack<std::pair<std::string, std::string>> saved_settings;
  60   ConfigProxy& conf;
  61 public:
  62   ConfSaver(ConfigProxy& conf) : conf(conf) {
  63     conf._clear_safe_to_start_threads();
  64   };
  65   ~ConfSaver() {
  66     conf._clear_safe_to_start_threads();
  67     while(saved_settings.size() > 0) {
  68       auto& e = saved_settings.top();
  69       conf.set_val_or_die(e.first, e.second);
  70       saved_settings.pop();
  71     }
  72     conf.set_safe_to_start_threads();
  73     conf.apply_changes(nullptr);
  74   }
  75   void SetVal(const char* key, const char* val) {
  76     std::string skey(key);
  77     std::string prev_val;
  78     conf.get_val(skey, &prev_val);
  79     conf.set_val_or_die(skey, val);
  80     saved_settings.emplace(skey, prev_val);
  81   }
  82   void ApplyChanges() {
  83     conf.set_safe_to_start_threads();
  84     conf.apply_changes(nullptr);
  85   }
  86 };
  87
  88 TEST(BlueFS, mkfs) {
  89   uint64_t size = 1048576 * 128;
  90   TempBdev bdev{size};
  91   uuid_d fsid;
  92   BlueFS fs(g_ceph_context);
  93   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
  94   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
  95   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
  96 }
  97
  98 TEST(BlueFS, mkfs_mount) {
  99   uint64_t size = 1048576 * 128;
 100   TempBdev bdev{size};
 101   BlueFS fs(g_ceph_context);
 102   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 103   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 104   uuid_d fsid;
 105   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 106   ASSERT_EQ(0, fs.mount());
 107   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 108   ASSERT_EQ(fs.get_total(BlueFS::BDEV_DB), size - 1048576);
 109   ASSERT_LT(fs.get_free(BlueFS::BDEV_DB), size - 1048576);
 110   fs.umount();
 111 }
 112
 113 TEST(BlueFS, mkfs_mount_duplicate_gift) {
 114   uint64_t size = 1048576 * 128;
 115   TempBdev bdev{ size };
 116   bluefs_extent_t dup_ext;
 117   {
 118     BlueFS fs(g_ceph_context);
 119     ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 120     fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 121     uuid_d fsid;
 122     ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 123     ASSERT_EQ(0, fs.mount());
 124
 125     {
 126       BlueFS::FileWriter *h;
 127       ASSERT_EQ(0, fs.mkdir("dir"));
 128       ASSERT_EQ(0, fs.open_for_write("dir", "file1", &h, false));
 129       h->append("foo", 3);
 130       h->append("bar", 3);
 131       h->append("baz", 3);
 132       fs.fsync(h);
 133       ceph_assert(h->file->fnode.extents.size() > 0);
 134       dup_ext = h->file->fnode.extents[0];
 135       ceph_assert(dup_ext.bdev == BlueFS::BDEV_DB);
 136       fs.close_writer(h);
 137     }
 138
 139     fs.umount();
 140   }
 141
 142   {
 143     BlueFS fs(g_ceph_context);
 144     ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 145     ASSERT_EQ(0, fs.mount());
 146     // free allocation presumably allocated for file1
 147     std::cout << "duplicate extent: " << std::hex
 148       << dup_ext.offset << "~" << dup_ext.length
 149       << std::dec << std::endl;
 150     fs.debug_inject_duplicate_gift(BlueFS::BDEV_DB, dup_ext.offset, dup_ext.length);
 151     {
 152       // overwrite file1 with file2
 153       BlueFS::FileWriter *h;
 154       ASSERT_EQ(0, fs.open_for_write("dir", "file2", &h, false));
 155       h->append("foo", 3);
 156       h->append("bar", 3);
 157       h->append("baz", 3);
 158       fs.fsync(h);
 159       fs.close_writer(h);
 160     }
 161     fs.umount();
 162   }
 163
 164   g_ceph_context->_conf.set_val_or_die("bluefs_log_replay_check_allocations", "true");
 165   g_ceph_context->_conf.apply_changes(nullptr);
 166
 167   {
 168     // this should fail
 169     BlueFS fs(g_ceph_context);
 170     ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 171     ASSERT_NE(0, fs.mount());
 172   }
 173 }
 174
 175 TEST(BlueFS, write_read) {
 176   uint64_t size = 1048576 * 128;
 177   TempBdev bdev{size};
 178   BlueFS fs(g_ceph_context);
 179   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 180   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 181   uuid_d fsid;
 182   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 183   ASSERT_EQ(0, fs.mount());
 184   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 185   {
 186     BlueFS::FileWriter *h;
 187     ASSERT_EQ(0, fs.mkdir("dir"));
 188     ASSERT_EQ(0, fs.open_for_write("dir", "file", &h, false));
 189     h->append("foo", 3);
 190     h->append("bar", 3);
 191     h->append("baz", 3);
 192     fs.fsync(h);
 193     fs.close_writer(h);
 194   }
 195   {
 196     BlueFS::FileReader *h;
 197     ASSERT_EQ(0, fs.open_for_read("dir", "file", &h));
 198     bufferlist bl;
 199     BlueFS::FileReaderBuffer buf(4096);
 200     ASSERT_EQ(9, fs.read(h, &buf, 0, 1024, &bl, NULL));
 201     ASSERT_EQ(0, strncmp("foobarbaz", bl.c_str(), 9));
 202     delete h;
 203   }
 204   fs.umount();
 205 }
 206
 207 TEST(BlueFS, small_appends) {
 208   uint64_t size = 1048576 * 128;
 209   TempBdev bdev{size};
 210   BlueFS fs(g_ceph_context);
 211   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 212   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 213   uuid_d fsid;
 214   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 215   ASSERT_EQ(0, fs.mount());
 216   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 217   {
 218     BlueFS::FileWriter *h;
 219     ASSERT_EQ(0, fs.mkdir("dir"));
 220     ASSERT_EQ(0, fs.open_for_write("dir", "file", &h, false));
 221     for (unsigned i = 0; i < 10000; ++i) {
 222       h->append("abcdeabcdeabcdeabcdeabcdeabc", 23);
 223     }
 224     fs.fsync(h);
 225     fs.close_writer(h);
 226   }
 227   {
 228     BlueFS::FileWriter *h;
 229     ASSERT_EQ(0, fs.open_for_write("dir", "file_sync", &h, false));
 230     for (unsigned i = 0; i < 1000; ++i) {
 231       h->append("abcdeabcdeabcdeabcdeabcdeabc", 23);
 232       ASSERT_EQ(0, fs.fsync(h));
 233     }
 234     fs.close_writer(h);
 235   }
 236   fs.umount();
 237 }
 238
 239 TEST(BlueFS, very_large_write) {
 240   // we'll write a ~5G file, so allocate more than that for the whole fs
 241   uint64_t size = 1048576 * 1024 * 6ull;
 242   TempBdev bdev{size};
 243   BlueFS fs(g_ceph_context);
 244
 245   bool old = g_ceph_context->_conf.get_val<bool>("bluefs_buffered_io");
 246   g_ceph_context->_conf.set_val("bluefs_buffered_io", "false");
 247   uint64_t total_written = 0;
 248
 249   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 250   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 251   uuid_d fsid;
 252   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 253   ASSERT_EQ(0, fs.mount());
 254   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 255   char buf[1048571]; // this is biggish, but intentionally not evenly aligned
 256   for (unsigned i = 0; i < sizeof(buf); ++i) {
 257     buf[i] = i;
 258   }
 259   {
 260     BlueFS::FileWriter *h;
 261     ASSERT_EQ(0, fs.mkdir("dir"));
 262     ASSERT_EQ(0, fs.open_for_write("dir", "bigfile", &h, false));
 263     for (unsigned i = 0; i < 3*1024*1048576ull / sizeof(buf); ++i) {
 264       h->append(buf, sizeof(buf));
 265       total_written += sizeof(buf);
 266     }
 267     fs.fsync(h);
 268     for (unsigned i = 0; i < 2*1024*1048576ull / sizeof(buf); ++i) {
 269       h->append(buf, sizeof(buf));
 270       total_written += sizeof(buf);
 271     }
 272     fs.fsync(h);
 273     fs.close_writer(h);
 274   }
 275   {
 276     BlueFS::FileReader *h;
 277     ASSERT_EQ(0, fs.open_for_read("dir", "bigfile", &h));
 278     bufferlist bl;
 279     BlueFS::FileReaderBuffer readbuf(10485760);
 280     ASSERT_EQ(h->file->fnode.size, total_written);
 281     for (unsigned i = 0; i < 3*1024*1048576ull / sizeof(buf); ++i) {
 282       bl.clear();
 283       fs.read(h, &readbuf, i * sizeof(buf), sizeof(buf), &bl, NULL);
 284       int r = memcmp(buf, bl.c_str(), sizeof(buf));
 285       if (r) {
 286         cerr << "read got mismatch at offset " << i*sizeof(buf) << " r " << r
 287              << std::endl;
 288       }
 289       ASSERT_EQ(0, r);
 290     }
 291     for (unsigned i = 0; i < 2*1024*1048576ull / sizeof(buf); ++i) {
 292       bl.clear();
 293       fs.read(h, &readbuf, i * sizeof(buf), sizeof(buf), &bl, NULL);
 294       int r = memcmp(buf, bl.c_str(), sizeof(buf));
 295       if (r) {
 296         cerr << "read got mismatch at offset " << i*sizeof(buf) << " r " << r
 297              << std::endl;
 298       }
 299       ASSERT_EQ(0, r);
 300     }
 301     delete h;
 302     ASSERT_EQ(0, fs.open_for_read("dir", "bigfile", &h));
 303     ASSERT_EQ(h->file->fnode.size, total_written);
 304     unique_ptr<char> huge_buf(new char[h->file->fnode.size]);
 305     auto l = h->file->fnode.size;
 306     int64_t r = fs.read(h, &readbuf, 0, l, NULL, huge_buf.get());
 307     ASSERT_EQ(r, (int64_t)l);
 308     delete h;
 309   }
 310   fs.umount();
 311
 312   g_ceph_context->_conf.set_val("bluefs_buffered_io", stringify((int)old));
 313 }
 314
 315 TEST(BlueFS, very_large_write2) {
 316   // we'll write a ~5G file, so allocate more than that for the whole fs
 317   uint64_t size_full = 1048576 * 1024 * 6ull;
 318   uint64_t size = 1048576 * 1024 * 5ull;
 319   TempBdev bdev{ size_full };
 320   BlueFS fs(g_ceph_context);
 321
 322   bool old = g_ceph_context->_conf.get_val<bool>("bluefs_buffered_io");
 323   g_ceph_context->_conf.set_val("bluefs_buffered_io", "false");
 324   uint64_t total_written = 0;
 325
 326   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false, 1048576));
 327   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size_full - 1048576);
 328   uuid_d fsid;
 329   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 330   ASSERT_EQ(0, fs.mount());
 331   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 332
 333   char fill_arr[1 << 20]; // 1M
 334   for (size_t i = 0; i < sizeof(fill_arr); ++i) {
 335     fill_arr[i] = (char)i;
 336   }
 337   std::unique_ptr<char[]> buf;
 338   buf.reset(new char[size]);
 339   for (size_t i = 0; i < size; i += sizeof(fill_arr)) {
 340     memcpy(buf.get() + i, fill_arr, sizeof(fill_arr));
 341   }
 342   {
 343     BlueFS::FileWriter* h;
 344     ASSERT_EQ(0, fs.mkdir("dir"));
 345     ASSERT_EQ(0, fs.open_for_write("dir", "bigfile", &h, false));
 346     fs.append_try_flush(h, buf.get(), size);
 347     total_written = size;
 348     fs.fsync(h);
 349     fs.close_writer(h);
 350   }
 351   memset(buf.get(), 0, size);
 352   {
 353     BlueFS::FileReader* h;
 354     ASSERT_EQ(0, fs.open_for_read("dir", "bigfile", &h));
 355     ASSERT_EQ(h->file->fnode.size, total_written);
 356     auto l = h->file->fnode.size;
 357     BlueFS::FileReaderBuffer readbuf(10485760);
 358     int64_t r = fs.read(h, &readbuf, 0, l, NULL, buf.get());
 359     ASSERT_EQ(r, (int64_t)l);
 360     for (size_t i = 0; i < size; i += sizeof(fill_arr)) {
 361       ceph_assert(memcmp(buf.get() + i, fill_arr, sizeof(fill_arr)) == 0);
 362     }
 363     delete h;
 364   }
 365   fs.umount();
 366
 367   g_ceph_context->_conf.set_val("bluefs_buffered_io", stringify((int)old));
 368 }
 369
 370 #define ALLOC_SIZE 4096
 371
 372 void write_data(BlueFS &fs, uint64_t rationed_bytes)
 373 {
 374     int j=0, r=0;
 375     uint64_t written_bytes = 0;
 376     rationed_bytes -= ALLOC_SIZE;
 377     stringstream ss;
 378     string dir = "dir.";
 379     ss << std::this_thread::get_id();
 380     dir.append(ss.str());
 381     dir.append(".");
 382     dir.append(to_string(j));
 383     ASSERT_EQ(0, fs.mkdir(dir));
 384     while (1) {
 385       string file = "file.";
 386       file.append(to_string(j));
 387       BlueFS::FileWriter *h;
 388       ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false));
 389       ASSERT_NE(nullptr, h);
 390       auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); });
 391       bufferlist bl;
 392       std::unique_ptr<char[]> buf = gen_buffer(ALLOC_SIZE);
 393       bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get());
 394       bl.push_back(bp);
 395       h->append(bl.c_str(), bl.length());
 396       r = fs.fsync(h);
 397       if (r < 0) {
 398          break;
 399       }
 400       written_bytes += g_conf()->bluefs_alloc_size;
 401       j++;
 402       if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) {
 403         break;
 404       }
 405     }
 406 }
 407
 408 void create_single_file(BlueFS &fs)
 409 {
 410     BlueFS::FileWriter *h;
 411     stringstream ss;
 412     string dir = "dir.test";
 413     ASSERT_EQ(0, fs.mkdir(dir));
 414     string file = "testfile";
 415     ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false));
 416     bufferlist bl;
 417     std::unique_ptr<char[]> buf = gen_buffer(ALLOC_SIZE);
 418     bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get());
 419     bl.push_back(bp);
 420     h->append(bl.c_str(), bl.length());
 421     fs.fsync(h);
 422     fs.close_writer(h);
 423 }
 424
 425 void write_single_file(BlueFS &fs, uint64_t rationed_bytes)
 426 {
 427     stringstream ss;
 428     const string dir = "dir.test";
 429     const string file = "testfile";
 430     uint64_t written_bytes = 0;
 431     rationed_bytes -= ALLOC_SIZE;
 432     while (1) {
 433       BlueFS::FileWriter *h;
 434       ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false));
 435       ASSERT_NE(nullptr, h);
 436       auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); });
 437       bufferlist bl;
 438       std::unique_ptr<char[]> buf = gen_buffer(ALLOC_SIZE);
 439       bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf.get());
 440       bl.push_back(bp);
 441       h->append(bl.c_str(), bl.length());
 442       int r = fs.fsync(h);
 443       if (r < 0) {
 444          break;
 445       }
 446       written_bytes += g_conf()->bluefs_alloc_size;
 447       if ((rationed_bytes - written_bytes) <= g_conf()->bluefs_alloc_size) {
 448         break;
 449       }
 450     }
 451 }
 452
 453 bool writes_done = false;
 454
 455 void sync_fs(BlueFS &fs)
 456 {
 457     while (1) {
 458       if (writes_done == true)
 459         break;
 460       fs.sync_metadata(false);
 461       sleep(1);
 462     }
 463 }
 464
 465
 466 void do_join(std::thread& t)
 467 {
 468     t.join();
 469 }
 470
 471 void join_all(std::vector<std::thread>& v)
 472 {
 473     std::for_each(v.begin(),v.end(),do_join);
 474 }
 475
 476 #define NUM_WRITERS 3
 477 #define NUM_SYNC_THREADS 1
 478
 479 #define NUM_SINGLE_FILE_WRITERS 1
 480 #define NUM_MULTIPLE_FILE_WRITERS 2
 481
 482 TEST(BlueFS, test_flush_1) {
 483   uint64_t size = 1048576 * 128;
 484   TempBdev bdev{size};
 485   g_ceph_context->_conf.set_val(
 486     "bluefs_alloc_size",
 487     "65536");
 488   g_ceph_context->_conf.apply_changes(nullptr);
 489
 490   BlueFS fs(g_ceph_context);
 491   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 492   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 493   uuid_d fsid;
 494   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 495   ASSERT_EQ(0, fs.mount());
 496   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 497   {
 498     std::vector<std::thread> write_thread_multiple;
 499     uint64_t effective_size = size - (32 * 1048576); // leaving the last 32 MB for log compaction
 500     uint64_t per_thread_bytes = (effective_size/(NUM_MULTIPLE_FILE_WRITERS + NUM_SINGLE_FILE_WRITERS));
 501     for (int i=0; i<NUM_MULTIPLE_FILE_WRITERS ; i++) {
 502       write_thread_multiple.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 503     }
 504
 505     create_single_file(fs);
 506     std::vector<std::thread> write_thread_single;
 507     for (int i=0; i<NUM_SINGLE_FILE_WRITERS; i++) {
 508       write_thread_single.push_back(std::thread(write_single_file, std::ref(fs), per_thread_bytes));
 509     }
 510
 511     join_all(write_thread_single);
 512     join_all(write_thread_multiple);
 513   }
 514   fs.umount();
 515 }
 516
 517 TEST(BlueFS, test_flush_2) {
 518   uint64_t size = 1048576 * 256;
 519   TempBdev bdev{size};
 520   g_ceph_context->_conf.set_val(
 521     "bluefs_alloc_size",
 522     "65536");
 523   g_ceph_context->_conf.apply_changes(nullptr);
 524
 525   BlueFS fs(g_ceph_context);
 526   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 527   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 528   uuid_d fsid;
 529   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 530   ASSERT_EQ(0, fs.mount());
 531   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 532   {
 533     uint64_t effective_size = size - (128 * 1048576); // leaving the last 32 MB for log compaction
 534     uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS));
 535     std::vector<std::thread> write_thread_multiple;
 536     for (int i=0; i<NUM_WRITERS; i++) {
 537       write_thread_multiple.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 538     }
 539
 540     join_all(write_thread_multiple);
 541   }
 542   fs.umount();
 543 }
 544
 545 TEST(BlueFS, test_flush_3) {
 546   uint64_t size = 1048576 * 256;
 547   TempBdev bdev{size};
 548   g_ceph_context->_conf.set_val(
 549     "bluefs_alloc_size",
 550     "65536");
 551   g_ceph_context->_conf.apply_changes(nullptr);
 552
 553   BlueFS fs(g_ceph_context);
 554   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 555   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 556   uuid_d fsid;
 557   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 558   ASSERT_EQ(0, fs.mount());
 559   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 560   {
 561     std::vector<std::thread> write_threads;
 562     uint64_t effective_size = size - (64 * 1048576); // leaving the last 11 MB for log compaction
 563     uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS));
 564     for (int i=0; i<NUM_WRITERS; i++) {
 565       write_threads.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 566     }
 567
 568     std::vector<std::thread> sync_threads;
 569     for (int i=0; i<NUM_SYNC_THREADS; i++) {
 570       sync_threads.push_back(std::thread(sync_fs, std::ref(fs)));
 571     }
 572
 573     join_all(write_threads);
 574     writes_done = true;
 575     join_all(sync_threads);
 576   }
 577   fs.umount();
 578 }
 579
 580 TEST(BlueFS, test_simple_compaction_sync) {
 581   g_ceph_context->_conf.set_val(
 582     "bluefs_compact_log_sync",
 583     "true");
 584   uint64_t size = 1048576 * 128;
 585   TempBdev bdev{size};
 586
 587   BlueFS fs(g_ceph_context);
 588   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 589   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 590   uuid_d fsid;
 591   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 592   ASSERT_EQ(0, fs.mount());
 593   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 594   {
 595     for (int i=0; i<10; i++) {
 596        string dir = "dir.";
 597        dir.append(to_string(i));
 598        ASSERT_EQ(0, fs.mkdir(dir));
 599        for (int j=0; j<10; j++) {
 600           string file = "file.";
 601           file.append(to_string(j));
 602           BlueFS::FileWriter *h;
 603           ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false));
 604           ASSERT_NE(nullptr, h);
 605           auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); });
 606           bufferlist bl;
 607           std::unique_ptr<char[]> buf = gen_buffer(4096);
 608           bufferptr bp = buffer::claim_char(4096, buf.get());
 609           bl.push_back(bp);
 610           h->append(bl.c_str(), bl.length());
 611           fs.fsync(h);
 612        }
 613     }
 614   }
 615   {
 616     for (int i=0; i<10; i+=2) {
 617        string dir = "dir.";
 618        dir.append(to_string(i));
 619        for (int j=0; j<10; j++) {
 620           string file = "file.";
 621           file.append(to_string(j));
 622           fs.unlink(dir, file);
 623           fs.sync_metadata(false);
 624        }
 625        ASSERT_EQ(0, fs.rmdir(dir));
 626        fs.sync_metadata(false);
 627     }
 628   }
 629   fs.compact_log();
 630   fs.umount();
 631 }
 632
 633 TEST(BlueFS, test_simple_compaction_async) {
 634   g_ceph_context->_conf.set_val(
 635     "bluefs_compact_log_sync",
 636     "false");
 637   uint64_t size = 1048576 * 128;
 638   TempBdev bdev{size};
 639
 640   BlueFS fs(g_ceph_context);
 641   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 642   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 643   uuid_d fsid;
 644   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 645   ASSERT_EQ(0, fs.mount());
 646   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 647   {
 648     for (int i=0; i<10; i++) {
 649        string dir = "dir.";
 650        dir.append(to_string(i));
 651        ASSERT_EQ(0, fs.mkdir(dir));
 652        for (int j=0; j<10; j++) {
 653           string file = "file.";
 654           file.append(to_string(j));
 655           BlueFS::FileWriter *h;
 656           ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false));
 657           ASSERT_NE(nullptr, h);
 658           auto sg = make_scope_guard([&fs, h] { fs.close_writer(h); });
 659           bufferlist bl;
 660           std::unique_ptr<char[]> buf = gen_buffer(4096);
 661           bufferptr bp = buffer::claim_char(4096, buf.get());
 662           bl.push_back(bp);
 663           h->append(bl.c_str(), bl.length());
 664           fs.fsync(h);
 665        }
 666     }
 667   }
 668   {
 669     for (int i=0; i<10; i+=2) {
 670        string dir = "dir.";
 671        dir.append(to_string(i));
 672        for (int j=0; j<10; j++) {
 673           string file = "file.";
 674           file.append(to_string(j));
 675           fs.unlink(dir, file);
 676           fs.sync_metadata(false);
 677        }
 678        ASSERT_EQ(0, fs.rmdir(dir));
 679        fs.sync_metadata(false);
 680     }
 681   }
 682   fs.compact_log();
 683   fs.umount();
 684 }
 685
 686 TEST(BlueFS, test_compaction_sync) {
 687   uint64_t size = 1048576 * 128;
 688   TempBdev bdev{size};
 689   g_ceph_context->_conf.set_val(
 690     "bluefs_alloc_size",
 691     "65536");
 692   g_ceph_context->_conf.set_val(
 693     "bluefs_compact_log_sync",
 694     "true");
 695
 696   BlueFS fs(g_ceph_context);
 697   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 698   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 699   uuid_d fsid;
 700   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 701   ASSERT_EQ(0, fs.mount());
 702   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 703   {
 704     std::vector<std::thread> write_threads;
 705     uint64_t effective_size = size - (32 * 1048576); // leaving the last 32 MB for log compaction
 706     uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS));
 707     for (int i=0; i<NUM_WRITERS; i++) {
 708       write_threads.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 709     }
 710
 711     std::vector<std::thread> sync_threads;
 712     for (int i=0; i<NUM_SYNC_THREADS; i++) {
 713       sync_threads.push_back(std::thread(sync_fs, std::ref(fs)));
 714     }
 715
 716     join_all(write_threads);
 717     writes_done = true;
 718     join_all(sync_threads);
 719     fs.compact_log();
 720   }
 721   fs.umount();
 722 }
 723
 724 TEST(BlueFS, test_compaction_async) {
 725   uint64_t size = 1048576 * 128;
 726   TempBdev bdev{size};
 727   g_ceph_context->_conf.set_val(
 728     "bluefs_alloc_size",
 729     "65536");
 730   g_ceph_context->_conf.set_val(
 731     "bluefs_compact_log_sync",
 732     "false");
 733
 734   BlueFS fs(g_ceph_context);
 735   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 736   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 737   uuid_d fsid;
 738   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 739   ASSERT_EQ(0, fs.mount());
 740   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 741   {
 742     std::vector<std::thread> write_threads;
 743     uint64_t effective_size = size - (32 * 1048576); // leaving the last 32 MB for log compaction
 744     uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS));
 745     for (int i=0; i<NUM_WRITERS; i++) {
 746       write_threads.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 747     }
 748
 749     std::vector<std::thread> sync_threads;
 750     for (int i=0; i<NUM_SYNC_THREADS; i++) {
 751       sync_threads.push_back(std::thread(sync_fs, std::ref(fs)));
 752     }
 753
 754     join_all(write_threads);
 755     writes_done = true;
 756     join_all(sync_threads);
 757     fs.compact_log();
 758   }
 759   fs.umount();
 760 }
 761
 762 TEST(BlueFS, test_replay) {
 763   uint64_t size = 1048576 * 128;
 764   TempBdev bdev{size};
 765   g_ceph_context->_conf.set_val(
 766     "bluefs_alloc_size",
 767     "65536");
 768   g_ceph_context->_conf.set_val(
 769     "bluefs_compact_log_sync",
 770     "false");
 771
 772   BlueFS fs(g_ceph_context);
 773   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 774   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 775   uuid_d fsid;
 776   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 777   ASSERT_EQ(0, fs.mount());
 778   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 779   {
 780     std::vector<std::thread> write_threads;
 781     uint64_t effective_size = size - (32 * 1048576); // leaving the last 32 MB for log compaction
 782     uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS));
 783     for (int i=0; i<NUM_WRITERS; i++) {
 784       write_threads.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 785     }
 786
 787     std::vector<std::thread> sync_threads;
 788     for (int i=0; i<NUM_SYNC_THREADS; i++) {
 789       sync_threads.push_back(std::thread(sync_fs, std::ref(fs)));
 790     }
 791
 792     join_all(write_threads);
 793     writes_done = true;
 794     join_all(sync_threads);
 795     fs.compact_log();
 796   }
 797   fs.umount();
 798   // remount and check log can replay safe?
 799   ASSERT_EQ(0, fs.mount());
 800   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 801   fs.umount();
 802 }
 803
 804 TEST(BlueFS, test_replay_growth) {
 805   uint64_t size = 1048576LL * (2 * 1024 + 128);
 806   TempBdev bdev{size};
 807
 808   ConfSaver conf(g_ceph_context->_conf);
 809   conf.SetVal("bluefs_alloc_size", "4096");
 810   conf.SetVal("bluefs_shared_alloc_size", "4096");
 811   conf.SetVal("bluefs_compact_log_sync", "false");
 812   conf.SetVal("bluefs_min_log_runway", "32768");
 813   conf.SetVal("bluefs_max_log_runway", "65536");
 814   conf.SetVal("bluefs_allocator", "stupid");
 815   conf.SetVal("bluefs_sync_write", "true");
 816   conf.ApplyChanges();
 817
 818   BlueFS fs(g_ceph_context);
 819   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, bdev.path, false));
 820   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 821   uuid_d fsid;
 822   ASSERT_EQ(0, fs.mkfs(fsid, { BlueFS::BDEV_DB, false, false }));
 823   ASSERT_EQ(0, fs.mount());
 824   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 825   ASSERT_EQ(0, fs.mkdir("dir"));
 826
 827   char data[2000];
 828   BlueFS::FileWriter *h;
 829   ASSERT_EQ(0, fs.open_for_write("dir", "file", &h, false));
 830   for (size_t i = 0; i < 10000; i++) {
 831     h->append(data, 2000);
 832     fs.fsync(h);
 833   }
 834   fs.close_writer(h);
 835   fs.umount(true); //do not compact on exit!
 836
 837   // remount and check log can replay safe?
 838   ASSERT_EQ(0, fs.mount());
 839   ASSERT_EQ(0, fs.maybe_verify_layout({ BlueFS::BDEV_DB, false, false }));
 840   fs.umount();
 841 }
 842
 843 int main(int argc, char **argv) {
 844   vector<const char*> args;
 845   argv_to_vec(argc, (const char **)argv, args);
 846
 847   map<string,string> defaults = {
 848     { "debug_bluefs", "1/20" },
 849     { "debug_bdev", "1/20" }
 850   };
 851
 852   auto cct = global_init(&defaults, args, CEPH_ENTITY_TYPE_CLIENT,
 853                          CODE_ENVIRONMENT_UTILITY,
 854                          CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
 855   common_init_finish(g_ceph_context);
 856   g_ceph_context->_conf.set_val(
 857     "enable_experimental_unrecoverable_data_corrupting_features",
 858     "*");
 859   g_ceph_context->_conf.apply_changes(nullptr);
 860
 861   ::testing::InitGoogleTest(&argc, argv);
 862   return RUN_ALL_TESTS();
 863 }