ceph/src/test/crimson/seastore/test_omap_manager.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "test/crimson/gtest_seastar.h"
   5
   6 #include "test/crimson/seastore/transaction_manager_test_state.h"
   7
   8 #include "crimson/os/seastore/cache.h"
   9 #include "crimson/os/seastore/transaction_manager.h"
  10 #include "crimson/os/seastore/segment_manager.h"
  11 #include "crimson/os/seastore/omap_manager.h"
  12
  13 #include "test/crimson/seastore/test_block.h"
  14
  15 using namespace crimson;
  16 using namespace crimson::os;
  17 using namespace crimson::os::seastore;
  18 using namespace std;
  19
  20 namespace {
  21   [[maybe_unused]] seastar::logger& logger() {
  22     return crimson::get_logger(ceph_subsys_test);
  23   }
  24 }
  25
  26 const int STR_LEN = 50;
  27
  28 std::string rand_name(const int len)
  29 {
  30   std::string ret;
  31   ret.reserve(len);
  32   for (int i = 0; i < len; ++i) {
  33     ret.append(1, (char)(rand() % ('z' - '0')) + '0');
  34   }
  35   return ret;
  36 }
  37
  38 bufferlist rand_buffer(const int len) {
  39   bufferptr ptr(len);
  40   for (auto i = ptr.c_str(); i < ptr.c_str() + len; ++i) {
  41     *i = (char)rand();
  42   }
  43   bufferlist bl;
  44   bl.append(ptr);
  45   return bl;
  46 }
  47
  48 struct omap_manager_test_t :
  49   public seastar_test_suite_t,
  50   TMTestState {
  51
  52   OMapManagerRef omap_manager;
  53
  54   omap_manager_test_t() {}
  55
  56   seastar::future<> set_up_fut() final {
  57     return tm_setup().then([this] {
  58       omap_manager = omap_manager::create_omap_manager(*tm);
  59       return seastar::now();
  60     });
  61   }
  62
  63   seastar::future<> tear_down_fut() final {
  64     return tm_teardown().then([this] {
  65       omap_manager.reset();
  66       return seastar::now();
  67     });
  68   }
  69
  70   using test_omap_t = std::map<std::string, ceph::bufferlist>;
  71   test_omap_t test_omap_mappings;
  72
  73   void set_key(
  74     omap_root_t &omap_root,
  75     Transaction &t,
  76     const string &key,
  77     const bufferlist &val) {
  78     with_trans_intr(
  79       t,
  80       [&, this](auto &t) {
  81         return omap_manager->omap_set_key(omap_root, t, key, val);
  82       }).unsafe_get0();
  83     test_omap_mappings[key] = val;
  84   }
  85
  86   void set_key(
  87     omap_root_t &omap_root,
  88     Transaction &t,
  89     const string &key,
  90     const string &val) {
  91     bufferlist bl;
  92     bl.append(val);
  93     set_key(omap_root, t, key, bl);
  94   }
  95
  96   std::string set_random_key(
  97     omap_root_t &omap_root,
  98     Transaction &t) {
  99     auto key = rand_name(STR_LEN);
 100     set_key(
 101       omap_root,
 102       t,
 103       key,
 104       rand_buffer(STR_LEN));
 105     return key;
 106   }
 107
 108   void get_value(
 109     omap_root_t &omap_root,
 110     Transaction &t,
 111     const string &key) {
 112     auto ret = with_trans_intr(
 113       t,
 114       [&, this](auto &t) {
 115         return omap_manager->omap_get_value(omap_root, t, key);
 116       }).unsafe_get0();
 117     auto iter = test_omap_mappings.find(key);
 118     if (iter == test_omap_mappings.end()) {
 119       EXPECT_FALSE(ret);
 120     } else {
 121       EXPECT_TRUE(ret);
 122       if (ret) {
 123         EXPECT_TRUE(*ret == iter->second);
 124       }
 125     }
 126   }
 127
 128   void rm_key(
 129     omap_root_t &omap_root,
 130     Transaction &t,
 131     const string &key) {
 132     with_trans_intr(
 133       t,
 134       [&, this](auto &t) {
 135         return omap_manager->omap_rm_key(omap_root, t, key);
 136       }).unsafe_get0();
 137     test_omap_mappings.erase(test_omap_mappings.find(key));
 138   }
 139
 140   void list(
 141     const omap_root_t &omap_root,
 142     Transaction &t,
 143     const std::optional<std::string> &start,
 144     size_t max = 128) {
 145
 146     if (start) {
 147       logger().debug("list on {}", *start);
 148     } else {
 149       logger().debug("list on start");
 150     }
 151
 152     auto config = OMapManager::omap_list_config_t::with_max(max);
 153     config.max_result_size = max;
 154
 155     auto [complete, results] = with_trans_intr(
 156       t,
 157       [&, this](auto &t) {
 158         return omap_manager->omap_list(omap_root, t, start, config);
 159       }).unsafe_get0();
 160
 161     auto it = start ?
 162       test_omap_mappings.upper_bound(*start) :
 163       test_omap_mappings.begin();
 164     for (auto &&[k, v]: results) {
 165       EXPECT_NE(it, test_omap_mappings.end());
 166       if (it == test_omap_mappings.end())
 167         return;
 168       EXPECT_EQ(k, it->first);
 169       EXPECT_EQ(v, it->second);
 170       it++;
 171     }
 172     if (it == test_omap_mappings.end()) {
 173       EXPECT_TRUE(complete);
 174     } else {
 175       EXPECT_EQ(results.size(), max);
 176     }
 177   }
 178
 179   void clear(
 180     omap_root_t &omap_root,
 181     Transaction &t) {
 182     with_trans_intr(
 183       t,
 184       [&, this](auto &t) {
 185         return omap_manager->omap_clear(omap_root, t);
 186       }).unsafe_get0();
 187     EXPECT_EQ(omap_root.get_location(), L_ADDR_NULL);
 188   }
 189
 190   void check_mappings(omap_root_t &omap_root, Transaction &t) {
 191     for (const auto &i: test_omap_mappings){
 192       get_value(omap_root, t, i.first);
 193     }
 194   }
 195
 196   void check_mappings(omap_root_t &omap_root) {
 197     auto t = create_read_transaction();
 198     check_mappings(omap_root, *t);
 199   }
 200
 201   std::vector<std::string> get_mapped_keys() {
 202     std::vector<std::string> mkeys;
 203     mkeys.reserve(test_omap_mappings.size());
 204     for (auto &k: test_omap_mappings) {
 205       mkeys.push_back(k.first);
 206     }
 207     return mkeys;
 208   }
 209
 210   void replay() {
 211     logger().debug("{}: begin", __func__);
 212     restart();
 213     omap_manager = omap_manager::create_omap_manager(*tm);
 214     logger().debug("{}: end", __func__);
 215   }
 216
 217   auto initialize() {
 218     auto t = create_mutate_transaction();
 219     omap_root_t omap_root = with_trans_intr(
 220       *t,
 221       [this](auto &t) {
 222         return omap_manager->initialize_omap(t, L_ADDR_MIN);
 223       }).unsafe_get0();
 224     submit_transaction(std::move(t));
 225     return omap_root;
 226   }
 227 };
 228
 229 TEST_F(omap_manager_test_t, basic)
 230 {
 231   run_async([this] {
 232     omap_root_t omap_root = initialize();
 233
 234     string key = "owner";
 235     string val = "test";
 236
 237     {
 238       auto t = create_mutate_transaction();
 239       logger().debug("first transaction");
 240       set_key(omap_root, *t, key, val);
 241       get_value(omap_root, *t, key);
 242       submit_transaction(std::move(t));
 243     }
 244     {
 245       auto t = create_mutate_transaction();
 246       logger().debug("second transaction");
 247       get_value(omap_root, *t, key);
 248       rm_key(omap_root, *t, key);
 249       get_value(omap_root, *t, key);
 250       submit_transaction(std::move(t));
 251     }
 252     {
 253       auto t = create_mutate_transaction();
 254       logger().debug("third transaction");
 255       get_value(omap_root, *t, key);
 256       submit_transaction(std::move(t));
 257     }
 258   });
 259 }
 260
 261 TEST_F(omap_manager_test_t, force_leafnode_split)
 262 {
 263   run_async([this] {
 264     omap_root_t omap_root = initialize();
 265
 266     for (unsigned i = 0; i < 40; i++) {
 267       auto t = create_mutate_transaction();
 268       logger().debug("opened transaction");
 269       for (unsigned j = 0; j < 10; ++j) {
 270         set_random_key(omap_root, *t);
 271         if ((i % 20 == 0) && (j == 5)) {
 272           check_mappings(omap_root, *t);
 273         }
 274       }
 275       logger().debug("force split submit transaction i = {}", i);
 276       submit_transaction(std::move(t));
 277       check_mappings(omap_root);
 278     }
 279   });
 280 }
 281
 282 TEST_F(omap_manager_test_t, force_leafnode_split_merge)
 283 {
 284   run_async([this] {
 285     omap_root_t omap_root = initialize();
 286
 287     for (unsigned i = 0; i < 80; i++) {
 288       auto t = create_mutate_transaction();
 289       logger().debug("opened split_merge transaction");
 290       for (unsigned j = 0; j < 5; ++j) {
 291         set_random_key(omap_root, *t);
 292         if ((i % 10 == 0) && (j == 3)) {
 293           check_mappings(omap_root, *t);
 294         }
 295       }
 296       logger().debug("submitting transaction");
 297       submit_transaction(std::move(t));
 298       if (i % 50 == 0) {
 299         check_mappings(omap_root);
 300       }
 301     }
 302     auto mkeys = get_mapped_keys();
 303     auto t = create_mutate_transaction();
 304     for (unsigned i = 0; i < mkeys.size(); i++) {
 305       if (i % 3 != 0) {
 306         rm_key(omap_root, *t, mkeys[i]);
 307       }
 308
 309       if (i % 10 == 0) {
 310         logger().debug("submitting transaction i= {}", i);
 311         submit_transaction(std::move(t));
 312         t = create_mutate_transaction();
 313       }
 314       if (i % 100 == 0) {
 315         logger().debug("check_mappings  i= {}", i);
 316         check_mappings(omap_root, *t);
 317         check_mappings(omap_root);
 318       }
 319     }
 320     logger().debug("finally submitting transaction ");
 321     submit_transaction(std::move(t));
 322   });
 323 }
 324
 325 TEST_F(omap_manager_test_t, force_leafnode_split_merge_fullandbalanced)
 326 {
 327   run_async([this] {
 328     omap_root_t omap_root = initialize();
 329
 330     for (unsigned i = 0; i < 50; i++) {
 331       auto t = create_mutate_transaction();
 332       logger().debug("opened split_merge transaction");
 333       for (unsigned j = 0; j < 5; ++j) {
 334         set_random_key(omap_root, *t);
 335         if ((i % 10 == 0) && (j == 3)) {
 336           check_mappings(omap_root, *t);
 337         }
 338       }
 339       logger().debug("submitting transaction");
 340       submit_transaction(std::move(t));
 341       if (i % 50 == 0) {
 342         check_mappings(omap_root);
 343       }
 344     }
 345     auto mkeys = get_mapped_keys();
 346     auto t = create_mutate_transaction();
 347     for (unsigned i = 0; i < mkeys.size(); i++) {
 348       if (30 < i && i < 100) {
 349         rm_key(omap_root, *t, mkeys[i]);
 350       }
 351
 352       if (i % 10 == 0) {
 353         logger().debug("submitting transaction i= {}", i);
 354         submit_transaction(std::move(t));
 355         t = create_mutate_transaction();
 356       }
 357       if (i % 50 == 0) {
 358         logger().debug("check_mappings  i= {}", i);
 359         check_mappings(omap_root, *t);
 360         check_mappings(omap_root);
 361       }
 362       if (i == 100)
 363         break;
 364     }
 365     logger().debug("finally submitting transaction ");
 366     submit_transaction(std::move(t));
 367     check_mappings(omap_root);
 368   });
 369 }
 370
 371 TEST_F(omap_manager_test_t, force_split_listkeys_list_clear)
 372 {
 373   run_async([this] {
 374     omap_root_t omap_root = initialize();
 375
 376     string temp;
 377     for (unsigned i = 0; i < 40; i++) {
 378       auto t = create_mutate_transaction();
 379       logger().debug("opened transaction");
 380       for (unsigned j = 0; j < 10; ++j) {
 381         auto key = set_random_key(omap_root, *t);
 382         if (i == 10)
 383           temp = key;
 384         if ((i % 20 == 0) && (j == 5)) {
 385           check_mappings(omap_root, *t);
 386         }
 387       }
 388       logger().debug("force split submit transaction i = {}", i);
 389       submit_transaction(std::move(t));
 390       check_mappings(omap_root);
 391     }
 392
 393     {
 394       auto t = create_read_transaction();
 395       list(omap_root, *t, std::nullopt);
 396     }
 397
 398     {
 399       auto t = create_read_transaction();
 400       list(omap_root, *t, temp, 100);
 401     }
 402
 403     {
 404       auto t = create_mutate_transaction();
 405       clear(omap_root, *t);
 406       submit_transaction(std::move(t));
 407     }
 408   });
 409 }
 410
 411 TEST_F(omap_manager_test_t, internal_force_split)
 412 {
 413   run_async([this] {
 414     omap_root_t omap_root = initialize();
 415
 416     for (unsigned i = 0; i < 10; i++) {
 417       logger().debug("opened split transaction");
 418       auto t = create_mutate_transaction();
 419
 420       for (unsigned j = 0; j < 80; ++j) {
 421         set_random_key(omap_root, *t);
 422         if ((i % 2 == 0) && (j % 50 == 0)) {
 423           check_mappings(omap_root, *t);
 424         }
 425       }
 426       logger().debug("submitting transaction i = {}", i);
 427       submit_transaction(std::move(t));
 428     }
 429     check_mappings(omap_root);
 430   });
 431 }
 432
 433 TEST_F(omap_manager_test_t, internal_force_merge_fullandbalanced)
 434 {
 435   run_async([this] {
 436     omap_root_t omap_root = initialize();
 437
 438     for (unsigned i = 0; i < 8; i++) {
 439       logger().debug("opened split transaction");
 440       auto t = create_mutate_transaction();
 441
 442       for (unsigned j = 0; j < 80; ++j) {
 443         set_random_key(omap_root, *t);
 444         if ((i % 2 == 0) && (j % 50 == 0)) {
 445           check_mappings(omap_root, *t);
 446         }
 447       }
 448       logger().debug("submitting transaction");
 449       submit_transaction(std::move(t));
 450     }
 451     auto mkeys = get_mapped_keys();
 452     auto t = create_mutate_transaction();
 453     for (unsigned i = 0; i < mkeys.size(); i++) {
 454       rm_key(omap_root, *t, mkeys[i]);
 455
 456       if (i % 10 == 0) {
 457         logger().debug("submitting transaction i= {}", i);
 458         submit_transaction(std::move(t));
 459         t = create_mutate_transaction();
 460       }
 461       if (i % 50 == 0) {
 462         logger().debug("check_mappings  i= {}", i);
 463         check_mappings(omap_root, *t);
 464         check_mappings(omap_root);
 465       }
 466     }
 467     logger().debug("finally submitting transaction ");
 468     submit_transaction(std::move(t));
 469     check_mappings(omap_root);
 470   });
 471 }
 472
 473 TEST_F(omap_manager_test_t, replay)
 474 {
 475   run_async([this] {
 476     omap_root_t omap_root = initialize();
 477
 478     for (unsigned i = 0; i < 8; i++) {
 479       logger().debug("opened split transaction");
 480       auto t = create_mutate_transaction();
 481
 482       for (unsigned j = 0; j < 80; ++j) {
 483         set_random_key(omap_root, *t);
 484         if ((i % 2 == 0) && (j % 50 == 0)) {
 485           check_mappings(omap_root, *t);
 486         }
 487       }
 488       logger().debug("submitting transaction i = {}", i);
 489       submit_transaction(std::move(t));
 490     }
 491     replay();
 492     check_mappings(omap_root);
 493
 494     auto mkeys = get_mapped_keys();
 495     auto t = create_mutate_transaction();
 496     for (unsigned i = 0; i < mkeys.size(); i++) {
 497       rm_key(omap_root, *t, mkeys[i]);
 498
 499       if (i % 10 == 0) {
 500         logger().debug("submitting transaction i= {}", i);
 501         submit_transaction(std::move(t));
 502         replay();
 503         t = create_mutate_transaction();
 504       }
 505       if (i % 50 == 0) {
 506         logger().debug("check_mappings  i= {}", i);
 507         check_mappings(omap_root, *t);
 508         check_mappings(omap_root);
 509       }
 510     }
 511     logger().debug("finally submitting transaction ");
 512     submit_transaction(std::move(t));
 513     replay();
 514     check_mappings(omap_root);
 515   });
 516 }
 517
 518
 519 TEST_F(omap_manager_test_t, internal_force_split_to_root)
 520 {
 521   run_async([this] {
 522     omap_root_t omap_root = initialize();
 523
 524     logger().debug("set big keys");
 525     for (unsigned i = 0; i < 53; i++) {
 526       auto t = create_mutate_transaction();
 527
 528       for (unsigned j = 0; j < 8; ++j) {
 529         set_random_key(omap_root, *t);
 530       }
 531       logger().debug("submitting transaction i = {}", i);
 532       submit_transaction(std::move(t));
 533     }
 534      logger().debug("set small keys");
 535      for (unsigned i = 0; i < 100; i++) {
 536        auto t = create_mutate_transaction();
 537        for (unsigned j = 0; j < 8; ++j) {
 538          set_random_key(omap_root, *t);
 539        }
 540       logger().debug("submitting transaction last");
 541       submit_transaction(std::move(t));
 542      }
 543     check_mappings(omap_root);
 544   });
 545 }