ceph/src/test/objectstore/test_bluefs.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include <stdio.h>
   5 #include <string.h>
   6 #include <iostream>
   7 #include <time.h>
   8 #include <fcntl.h>
   9 #include <unistd.h>
  10 #include <random>
  11 #include <thread>
  12 #include "global/global_init.h"
  13 #include "common/ceph_argparse.h"
  14 #include "include/stringify.h"
  15 #include "common/errno.h"
  16 #include <gtest/gtest.h>
  17
  18 #include <boost/random/random_device.hpp>
  19
  20 #include "os/bluestore/BlueFS.h"
  21
  22 string get_temp_bdev(uint64_t size)
  23 {
  24   static int n = 0;
  25   string fn = "ceph_test_bluefs.tmp.block." + stringify(getpid())
  26     + "." + stringify(++n);
  27   int fd = ::open(fn.c_str(), O_CREAT|O_RDWR|O_TRUNC, 0644);
  28   assert(fd >= 0);
  29   int r = ::ftruncate(fd, size);
  30   assert(r >= 0);
  31   ::close(fd);
  32   return fn;
  33 }
  34
  35 char* gen_buffer(uint64_t size)
  36 {
  37     char *buffer = new char[size];
  38     boost::random::random_device rand;
  39     rand.generate(buffer, buffer + size);
  40     return buffer;
  41 }
  42
  43
  44 void rm_temp_bdev(string f)
  45 {
  46   ::unlink(f.c_str());
  47 }
  48
  49 TEST(BlueFS, mkfs) {
  50   uint64_t size = 1048576 * 128;
  51   string fn = get_temp_bdev(size);
  52   uuid_d fsid;
  53   BlueFS fs(g_ceph_context);
  54   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, fn));
  55   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
  56   ASSERT_EQ(0, fs.mkfs(fsid));
  57   rm_temp_bdev(fn);
  58 }
  59
  60 TEST(BlueFS, mkfs_mount) {
  61   uint64_t size = 1048576 * 128;
  62   string fn = get_temp_bdev(size);
  63   BlueFS fs(g_ceph_context);
  64   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, fn));
  65   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
  66   uuid_d fsid;
  67   ASSERT_EQ(0, fs.mkfs(fsid));
  68   ASSERT_EQ(0, fs.mount());
  69   ASSERT_EQ(fs.get_total(BlueFS::BDEV_DB), size - 1048576);
  70   ASSERT_LT(fs.get_free(BlueFS::BDEV_DB), size - 1048576);
  71   fs.umount();
  72   rm_temp_bdev(fn);
  73 }
  74
  75 TEST(BlueFS, write_read) {
  76   uint64_t size = 1048576 * 128;
  77   string fn = get_temp_bdev(size);
  78   BlueFS fs(g_ceph_context);
  79   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, fn));
  80   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
  81   uuid_d fsid;
  82   ASSERT_EQ(0, fs.mkfs(fsid));
  83   ASSERT_EQ(0, fs.mount());
  84   {
  85     BlueFS::FileWriter *h;
  86     ASSERT_EQ(0, fs.mkdir("dir"));
  87     ASSERT_EQ(0, fs.open_for_write("dir", "file", &h, false));
  88     h->append("foo", 3);
  89     h->append("bar", 3);
  90     h->append("baz", 3);
  91     fs.fsync(h);
  92     fs.close_writer(h);
  93   }
  94   {
  95     BlueFS::FileReader *h;
  96     ASSERT_EQ(0, fs.open_for_read("dir", "file", &h));
  97     bufferlist bl;
  98     BlueFS::FileReaderBuffer buf(4096);
  99     ASSERT_EQ(9, fs.read(h, &buf, 0, 1024, &bl, NULL));
 100     ASSERT_EQ(0, strncmp("foobarbaz", bl.c_str(), 9));
 101     delete h;
 102   }
 103   fs.umount();
 104   rm_temp_bdev(fn);
 105 }
 106
 107 TEST(BlueFS, small_appends) {
 108   uint64_t size = 1048576 * 128;
 109   string fn = get_temp_bdev(size);
 110   BlueFS fs(g_ceph_context);
 111   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, fn));
 112   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 113   uuid_d fsid;
 114   ASSERT_EQ(0, fs.mkfs(fsid));
 115   ASSERT_EQ(0, fs.mount());
 116   {
 117     BlueFS::FileWriter *h;
 118     ASSERT_EQ(0, fs.mkdir("dir"));
 119     ASSERT_EQ(0, fs.open_for_write("dir", "file", &h, false));
 120     for (unsigned i = 0; i < 10000; ++i) {
 121       h->append("abcdeabcdeabcdeabcdeabcdeabc", 23);
 122     }
 123     fs.fsync(h);
 124     fs.close_writer(h);
 125   }
 126   {
 127     BlueFS::FileWriter *h;
 128     ASSERT_EQ(0, fs.open_for_write("dir", "file_sync", &h, false));
 129     for (unsigned i = 0; i < 1000; ++i) {
 130       h->append("abcdeabcdeabcdeabcdeabcdeabc", 23);
 131       ASSERT_EQ(0, fs.fsync(h));
 132     }
 133     fs.close_writer(h);
 134   }
 135   fs.umount();
 136   rm_temp_bdev(fn);
 137 }
 138
 139 #define ALLOC_SIZE 4096
 140
 141 void write_data(BlueFS &fs, uint64_t rationed_bytes)
 142 {
 143     BlueFS::FileWriter *h;
 144     int j=0, r=0;
 145     uint64_t written_bytes = 0;
 146     rationed_bytes -= ALLOC_SIZE;
 147     stringstream ss;
 148     string dir = "dir.";
 149     ss << std::this_thread::get_id();
 150     dir.append(ss.str());
 151     dir.append(".");
 152     dir.append(to_string(j));
 153     ASSERT_EQ(0, fs.mkdir(dir));
 154     while (1) {
 155       string file = "file.";
 156       file.append(to_string(j));
 157       ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false));
 158       bufferlist bl;
 159       char *buf = gen_buffer(ALLOC_SIZE);
 160       bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf);
 161       bl.push_back(bp);
 162       h->append(bl.c_str(), bl.length());
 163       r = fs.fsync(h);
 164       if (r < 0) {
 165          fs.close_writer(h);
 166          break;
 167       }
 168       written_bytes += g_conf->bluefs_alloc_size;
 169       fs.close_writer(h);
 170       j++;
 171       if ((rationed_bytes - written_bytes) <= g_conf->bluefs_alloc_size) {
 172         break;
 173       }
 174     }
 175 }
 176
 177 void create_single_file(BlueFS &fs)
 178 {
 179     BlueFS::FileWriter *h;
 180     stringstream ss;
 181     string dir = "dir.test";
 182     ASSERT_EQ(0, fs.mkdir(dir));
 183     string file = "testfile";
 184     ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false));
 185     bufferlist bl;
 186     char *buf = gen_buffer(ALLOC_SIZE);
 187     bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf);
 188     bl.push_back(bp);
 189     h->append(bl.c_str(), bl.length());
 190     fs.fsync(h);
 191     fs.close_writer(h);
 192 }
 193
 194 void write_single_file(BlueFS &fs, uint64_t rationed_bytes)
 195 {
 196     BlueFS::FileWriter *h;
 197     stringstream ss;
 198     string dir = "dir.test";
 199     string file = "testfile";
 200     int r=0;
 201     uint64_t written_bytes = 0;
 202     rationed_bytes -= ALLOC_SIZE;
 203     while (1) {
 204       ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false));
 205       bufferlist bl;
 206       char *buf = gen_buffer(ALLOC_SIZE);
 207       bufferptr bp = buffer::claim_char(ALLOC_SIZE, buf);
 208       bl.push_back(bp);
 209       h->append(bl.c_str(), bl.length());
 210       r = fs.fsync(h);
 211       if (r < 0) {
 212          fs.close_writer(h);
 213          break;
 214       }
 215       written_bytes += g_conf->bluefs_alloc_size;
 216       fs.close_writer(h);
 217       if ((rationed_bytes - written_bytes) <= g_conf->bluefs_alloc_size) {
 218         break;
 219       }
 220     }
 221 }
 222
 223 bool writes_done = false;
 224
 225 void sync_fs(BlueFS &fs)
 226 {
 227     while (1) {
 228       if (writes_done == true)
 229         break;
 230       fs.sync_metadata();
 231       sleep(1);
 232     }
 233 }
 234
 235
 236 void do_join(std::thread& t)
 237 {
 238     t.join();
 239 }
 240
 241 void join_all(std::vector<std::thread>& v)
 242 {
 243     std::for_each(v.begin(),v.end(),do_join);
 244 }
 245
 246 #define NUM_WRITERS 3
 247 #define NUM_SYNC_THREADS 1
 248
 249 #define NUM_SINGLE_FILE_WRITERS 1
 250 #define NUM_MULTIPLE_FILE_WRITERS 2
 251
 252 TEST(BlueFS, test_flush_1) {
 253   uint64_t size = 1048576 * 128;
 254   string fn = get_temp_bdev(size);
 255   g_ceph_context->_conf->set_val(
 256     "bluefs_alloc_size",
 257     "65536");
 258   g_ceph_context->_conf->apply_changes(NULL);
 259
 260   BlueFS fs(g_ceph_context);
 261   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, fn));
 262   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 263   uuid_d fsid;
 264   ASSERT_EQ(0, fs.mkfs(fsid));
 265   ASSERT_EQ(0, fs.mount());
 266   {
 267     std::vector<std::thread> write_thread_multiple;
 268     uint64_t effective_size = size - (32 * 1048576); // leaving the last 32 MB for log compaction
 269     uint64_t per_thread_bytes = (effective_size/(NUM_MULTIPLE_FILE_WRITERS + NUM_SINGLE_FILE_WRITERS));
 270     for (int i=0; i<NUM_MULTIPLE_FILE_WRITERS ; i++) {
 271       write_thread_multiple.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 272     }
 273
 274     create_single_file(fs);
 275     std::vector<std::thread> write_thread_single;
 276     for (int i=0; i<NUM_SINGLE_FILE_WRITERS; i++) {
 277       write_thread_single.push_back(std::thread(write_single_file, std::ref(fs), per_thread_bytes));
 278     }
 279
 280     join_all(write_thread_single);
 281     join_all(write_thread_multiple);
 282   }
 283   fs.umount();
 284   rm_temp_bdev(fn);
 285 }
 286
 287 TEST(BlueFS, test_flush_2) {
 288   uint64_t size = 1048576 * 256;
 289   string fn = get_temp_bdev(size);
 290   g_ceph_context->_conf->set_val(
 291     "bluefs_alloc_size",
 292     "65536");
 293   g_ceph_context->_conf->apply_changes(NULL);
 294
 295   BlueFS fs(g_ceph_context);
 296   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, fn));
 297   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 298   uuid_d fsid;
 299   ASSERT_EQ(0, fs.mkfs(fsid));
 300   ASSERT_EQ(0, fs.mount());
 301   {
 302     uint64_t effective_size = size - (128 * 1048576); // leaving the last 32 MB for log compaction
 303     uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS));
 304     std::vector<std::thread> write_thread_multiple;
 305     for (int i=0; i<NUM_WRITERS; i++) {
 306       write_thread_multiple.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 307     }
 308
 309     join_all(write_thread_multiple);
 310   }
 311   fs.umount();
 312   rm_temp_bdev(fn);
 313 }
 314
 315 TEST(BlueFS, test_flush_3) {
 316   uint64_t size = 1048576 * 256;
 317   string fn = get_temp_bdev(size);
 318   g_ceph_context->_conf->set_val(
 319     "bluefs_alloc_size",
 320     "65536");
 321   g_ceph_context->_conf->apply_changes(NULL);
 322
 323   BlueFS fs(g_ceph_context);
 324   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, fn));
 325   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 326   uuid_d fsid;
 327   ASSERT_EQ(0, fs.mkfs(fsid));
 328   ASSERT_EQ(0, fs.mount());
 329   {
 330     std::vector<std::thread> write_threads;
 331     uint64_t effective_size = size - (64 * 1048576); // leaving the last 11 MB for log compaction
 332     uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS));
 333     for (int i=0; i<NUM_WRITERS; i++) {
 334       write_threads.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 335     }
 336
 337     std::vector<std::thread> sync_threads;
 338     for (int i=0; i<NUM_SYNC_THREADS; i++) {
 339       sync_threads.push_back(std::thread(sync_fs, std::ref(fs)));
 340     }
 341
 342     join_all(write_threads);
 343     writes_done = true;
 344     join_all(sync_threads);
 345   }
 346   fs.umount();
 347   rm_temp_bdev(fn);
 348 }
 349
 350 TEST(BlueFS, test_simple_compaction_sync) {
 351   g_ceph_context->_conf->set_val(
 352     "bluefs_compact_log_sync",
 353     "true");
 354   uint64_t size = 1048576 * 128;
 355   string fn = get_temp_bdev(size);
 356
 357   BlueFS fs(g_ceph_context);
 358   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, fn));
 359   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 360   uuid_d fsid;
 361   ASSERT_EQ(0, fs.mkfs(fsid));
 362   ASSERT_EQ(0, fs.mount());
 363   {
 364     BlueFS::FileWriter *h;
 365     for (int i=0; i<10; i++) {
 366        string dir = "dir.";
 367        dir.append(to_string(i));
 368        ASSERT_EQ(0, fs.mkdir(dir));
 369        for (int j=0; j<10; j++) {
 370           string file = "file.";
 371           file.append(to_string(j));
 372           ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false));
 373           bufferlist bl;
 374           char *buf = gen_buffer(4096);
 375           bufferptr bp = buffer::claim_char(4096, buf);
 376           bl.push_back(bp);
 377           h->append(bl.c_str(), bl.length());
 378           fs.fsync(h);
 379           fs.close_writer(h);
 380        }
 381     }
 382   }
 383   // Don't remove all
 384   {
 385     for (int i=0; i<10; i+=2) {
 386        string dir = "dir.";
 387        dir.append(to_string(i));
 388        for (int j=0; j<10; j+=2) {
 389           string file = "file.";
 390           file.append(to_string(j));
 391           fs.unlink(dir, file);
 392           fs.flush_log();
 393        }
 394        fs.rmdir(dir);
 395        fs.flush_log();
 396     }
 397   }
 398   fs.compact_log();
 399   fs.umount();
 400   rm_temp_bdev(fn);
 401 }
 402
 403 TEST(BlueFS, test_simple_compaction_async) {
 404   g_ceph_context->_conf->set_val(
 405     "bluefs_compact_log_sync",
 406     "false");
 407   uint64_t size = 1048576 * 128;
 408   string fn = get_temp_bdev(size);
 409
 410   BlueFS fs(g_ceph_context);
 411   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, fn));
 412   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 413   uuid_d fsid;
 414   ASSERT_EQ(0, fs.mkfs(fsid));
 415   ASSERT_EQ(0, fs.mount());
 416   {
 417     BlueFS::FileWriter *h;
 418     for (int i=0; i<10; i++) {
 419        string dir = "dir.";
 420        dir.append(to_string(i));
 421        ASSERT_EQ(0, fs.mkdir(dir));
 422        for (int j=0; j<10; j++) {
 423           string file = "file.";
 424           file.append(to_string(j));
 425           ASSERT_EQ(0, fs.open_for_write(dir, file, &h, false));
 426           bufferlist bl;
 427           char *buf = gen_buffer(4096);
 428           bufferptr bp = buffer::claim_char(4096, buf);
 429           bl.push_back(bp);
 430           h->append(bl.c_str(), bl.length());
 431           fs.fsync(h);
 432           fs.close_writer(h);
 433        }
 434     }
 435   }
 436   // Don't remove all
 437   {
 438     for (int i=0; i<10; i+=2) {
 439        string dir = "dir.";
 440        dir.append(to_string(i));
 441        for (int j=0; j<10; j+=2) {
 442           string file = "file.";
 443           file.append(to_string(j));
 444           fs.unlink(dir, file);
 445           fs.flush_log();
 446        }
 447        fs.rmdir(dir);
 448        fs.flush_log();
 449     }
 450   }
 451   fs.compact_log();
 452   fs.umount();
 453   rm_temp_bdev(fn);
 454 }
 455
 456 TEST(BlueFS, test_compaction_sync) {
 457   uint64_t size = 1048576 * 128;
 458   string fn = get_temp_bdev(size);
 459   g_ceph_context->_conf->set_val(
 460     "bluefs_alloc_size",
 461     "65536");
 462   g_ceph_context->_conf->set_val(
 463     "bluefs_compact_log_sync",
 464     "true");
 465
 466   BlueFS fs(g_ceph_context);
 467   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, fn));
 468   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 469   uuid_d fsid;
 470   ASSERT_EQ(0, fs.mkfs(fsid));
 471   ASSERT_EQ(0, fs.mount());
 472   {
 473     std::vector<std::thread> write_threads;
 474     uint64_t effective_size = size - (32 * 1048576); // leaving the last 32 MB for log compaction
 475     uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS));
 476     for (int i=0; i<NUM_WRITERS; i++) {
 477       write_threads.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 478     }
 479
 480     std::vector<std::thread> sync_threads;
 481     for (int i=0; i<NUM_SYNC_THREADS; i++) {
 482       sync_threads.push_back(std::thread(sync_fs, std::ref(fs)));
 483     }
 484
 485     join_all(write_threads);
 486     writes_done = true;
 487     join_all(sync_threads);
 488     fs.compact_log();
 489   }
 490   fs.umount();
 491   rm_temp_bdev(fn);
 492 }
 493
 494 TEST(BlueFS, test_compaction_async) {
 495   uint64_t size = 1048576 * 128;
 496   string fn = get_temp_bdev(size);
 497   g_ceph_context->_conf->set_val(
 498     "bluefs_alloc_size",
 499     "65536");
 500   g_ceph_context->_conf->set_val(
 501     "bluefs_compact_log_sync",
 502     "false");
 503
 504   BlueFS fs(g_ceph_context);
 505   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, fn));
 506   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 507   uuid_d fsid;
 508   ASSERT_EQ(0, fs.mkfs(fsid));
 509   ASSERT_EQ(0, fs.mount());
 510   {
 511     std::vector<std::thread> write_threads;
 512     uint64_t effective_size = size - (32 * 1048576); // leaving the last 32 MB for log compaction
 513     uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS));
 514     for (int i=0; i<NUM_WRITERS; i++) {
 515       write_threads.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 516     }
 517
 518     std::vector<std::thread> sync_threads;
 519     for (int i=0; i<NUM_SYNC_THREADS; i++) {
 520       sync_threads.push_back(std::thread(sync_fs, std::ref(fs)));
 521     }
 522
 523     join_all(write_threads);
 524     writes_done = true;
 525     join_all(sync_threads);
 526     fs.compact_log();
 527   }
 528   fs.umount();
 529   rm_temp_bdev(fn);
 530 }
 531
 532 TEST(BlueFS, test_replay) {
 533   uint64_t size = 1048576 * 128;
 534   string fn = get_temp_bdev(size);
 535   g_ceph_context->_conf->set_val(
 536     "bluefs_alloc_size",
 537     "65536");
 538   g_ceph_context->_conf->set_val(
 539     "bluefs_compact_log_sync",
 540     "false");
 541
 542   BlueFS fs(g_ceph_context);
 543   ASSERT_EQ(0, fs.add_block_device(BlueFS::BDEV_DB, fn));
 544   fs.add_block_extent(BlueFS::BDEV_DB, 1048576, size - 1048576);
 545   uuid_d fsid;
 546   ASSERT_EQ(0, fs.mkfs(fsid));
 547   ASSERT_EQ(0, fs.mount());
 548   {
 549     std::vector<std::thread> write_threads;
 550     uint64_t effective_size = size - (32 * 1048576); // leaving the last 32 MB for log compaction
 551     uint64_t per_thread_bytes = (effective_size/(NUM_WRITERS));
 552     for (int i=0; i<NUM_WRITERS; i++) {
 553       write_threads.push_back(std::thread(write_data, std::ref(fs), per_thread_bytes));
 554     }
 555
 556     std::vector<std::thread> sync_threads;
 557     for (int i=0; i<NUM_SYNC_THREADS; i++) {
 558       sync_threads.push_back(std::thread(sync_fs, std::ref(fs)));
 559     }
 560
 561     join_all(write_threads);
 562     writes_done = true;
 563     join_all(sync_threads);
 564     fs.compact_log();
 565   }
 566   fs.umount();
 567   // remount and check log can replay safe?
 568   ASSERT_EQ(0, fs.mount());
 569   fs.umount();
 570   rm_temp_bdev(fn);
 571 }
 572
 573 int main(int argc, char **argv) {
 574   vector<const char*> args;
 575   argv_to_vec(argc, (const char **)argv, args);
 576   env_to_vec(args);
 577
 578   vector<const char *> def_args;
 579   def_args.push_back("--debug-bluefs=1/20");
 580   def_args.push_back("--debug-bdev=1/20");
 581
 582   auto cct = global_init(&def_args, args, CEPH_ENTITY_TYPE_CLIENT,
 583                          CODE_ENVIRONMENT_UTILITY,
 584                          0);
 585   common_init_finish(g_ceph_context);
 586   g_ceph_context->_conf->set_val(
 587     "enable_experimental_unrecoverable_data_corrupting_features",
 588     "*");
 589   g_ceph_context->_conf->apply_changes(NULL);
 590
 591   ::testing::InitGoogleTest(&argc, argv);
 592   return RUN_ALL_TESTS();
 593 }