ceph/src/test/crimson/seastore/test_seastore.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include <string>
   5 #include <iostream>
   6 #include <sstream>
   7
   8 #include "test/crimson/gtest_seastar.h"
   9
  10 #include "test/crimson/seastore/transaction_manager_test_state.h"
  11
  12 #include "crimson/os/futurized_collection.h"
  13 #include "crimson/os/seastore/seastore.h"
  14
  15 using namespace crimson;
  16 using namespace crimson::os;
  17 using namespace crimson::os::seastore;
  18 using CTransaction = ceph::os::Transaction;
  19 using namespace std;
  20
  21 namespace {
  22   [[maybe_unused]] seastar::logger& logger() {
  23     return crimson::get_logger(ceph_subsys_test);
  24   }
  25 }
  26
  27
  28 struct seastore_test_t :
  29   public seastar_test_suite_t,
  30   SeaStoreTestState {
  31
  32   coll_t coll_name{spg_t{pg_t{0, 0}}};
  33   CollectionRef coll;
  34
  35   seastore_test_t() {}
  36
  37   seastar::future<> set_up_fut() final {
  38     return tm_setup(
  39     ).then([this] {
  40       return seastore->create_new_collection(coll_name);
  41     }).then([this](auto coll_ref) {
  42       coll = coll_ref;
  43       CTransaction t;
  44       t.create_collection(coll_name, 4);
  45       return seastore->do_transaction(
  46         coll,
  47         std::move(t));
  48     });
  49   }
  50
  51   seastar::future<> tear_down_fut() final {
  52     coll.reset();
  53     return tm_teardown();
  54   }
  55
  56   void do_transaction(CTransaction &&t) {
  57     return seastore->do_transaction(
  58       coll,
  59       std::move(t)).get0();
  60   }
  61
  62   void set_meta(
  63     const std::string& key,
  64     const std::string& value) {
  65     return seastore->write_meta(key, value).get0();
  66   }
  67
  68   std::tuple<int, std::string> get_meta(
  69     const std::string& key) {
  70     return seastore->read_meta(key).get();
  71   }
  72
  73   struct object_state_t {
  74     const coll_t cid;
  75     const CollectionRef coll;
  76     const ghobject_t oid;
  77
  78     std::map<string, bufferlist> omap;
  79     bufferlist contents;
  80
  81     void touch(
  82       CTransaction &t) {
  83       t.touch(cid, oid);
  84     }
  85
  86     void touch(
  87       SeaStore &seastore) {
  88       CTransaction t;
  89       touch(t);
  90       seastore.do_transaction(
  91         coll,
  92         std::move(t)).get0();
  93     }
  94
  95     void remove(
  96       CTransaction &t) {
  97       t.remove(cid, oid);
  98     }
  99
 100     void remove(
 101       SeaStore &seastore) {
 102       CTransaction t;
 103       remove(t);
 104       seastore.do_transaction(
 105         coll,
 106         std::move(t)).get0();
 107     }
 108
 109     void set_omap(
 110       CTransaction &t,
 111       const string &key,
 112       const bufferlist &val) {
 113       omap[key] = val;
 114       std::map<string, bufferlist> arg;
 115       arg[key] = val;
 116       t.omap_setkeys(
 117         cid,
 118         oid,
 119         arg);
 120     }
 121
 122     void set_omap(
 123       SeaStore &seastore,
 124       const string &key,
 125       const bufferlist &val) {
 126       CTransaction t;
 127       set_omap(t, key, val);
 128       seastore.do_transaction(
 129         coll,
 130         std::move(t)).get0();
 131     }
 132
 133     void write(
 134       SeaStore &seastore,
 135       CTransaction &t,
 136       uint64_t offset,
 137       bufferlist bl)  {
 138       bufferlist new_contents;
 139       if (offset > 0 && contents.length()) {
 140         new_contents.substr_of(
 141           contents,
 142           0,
 143           std::min<size_t>(offset, contents.length())
 144         );
 145       }
 146       new_contents.append_zero(offset - new_contents.length());
 147       new_contents.append(bl);
 148
 149       auto tail_offset = offset + bl.length();
 150       if (contents.length() > tail_offset) {
 151         bufferlist tail;
 152         tail.substr_of(
 153           contents,
 154           tail_offset,
 155           contents.length() - tail_offset);
 156         new_contents.append(tail);
 157       }
 158       contents.swap(new_contents);
 159
 160       t.write(
 161         cid,
 162         oid,
 163         offset,
 164         bl.length(),
 165         bl);
 166     }
 167     void write(
 168       SeaStore &seastore,
 169       uint64_t offset,
 170       bufferlist bl)  {
 171       CTransaction t;
 172       write(seastore, t, offset, bl);
 173       seastore.do_transaction(
 174         coll,
 175         std::move(t)).get0();
 176     }
 177     void write(
 178       SeaStore &seastore,
 179       uint64_t offset,
 180       size_t len,
 181       char fill)  {
 182       auto buffer = bufferptr(buffer::create(len));
 183       ::memset(buffer.c_str(), fill, len);
 184       bufferlist bl;
 185       bl.append(buffer);
 186       write(seastore, offset, bl);
 187     }
 188
 189     void read(
 190       SeaStore &seastore,
 191       uint64_t offset,
 192       uint64_t len) {
 193       bufferlist to_check;
 194       to_check.substr_of(
 195         contents,
 196         offset,
 197         len);
 198       auto ret = seastore.read(
 199         coll,
 200         oid,
 201         offset,
 202         len).unsafe_get0();
 203       EXPECT_EQ(ret.length(), to_check.length());
 204       EXPECT_EQ(ret, to_check);
 205     }
 206
 207     void check_size(SeaStore &seastore) {
 208       auto st = seastore.stat(
 209         coll,
 210         oid).get0();
 211       EXPECT_EQ(contents.length(), st.st_size);
 212     }
 213
 214     void set_attr(
 215       SeaStore &seastore,
 216       std::string key,
 217       bufferlist& val) {
 218       CTransaction t;
 219       t.setattr(cid, oid, key, val);
 220       seastore.do_transaction(
 221         coll,
 222         std::move(t)).get0();
 223     }
 224
 225     SeaStore::attrs_t get_attrs(
 226       SeaStore &seastore) {
 227       return seastore.get_attrs(coll, oid)
 228                      .handle_error(SeaStore::get_attrs_ertr::discard_all{})
 229                      .get();
 230     }
 231
 232     ceph::bufferlist get_attr(
 233       SeaStore& seastore,
 234       std::string_view name) {
 235       return seastore.get_attr(coll, oid, name)
 236                       .handle_error(
 237                         SeaStore::get_attr_errorator::discard_all{})
 238                       .get();
 239     }
 240
 241     void check_omap_key(
 242       SeaStore &seastore,
 243       const string &key) {
 244       std::set<string> to_check;
 245       to_check.insert(key);
 246       auto result = seastore.omap_get_values(
 247         coll,
 248         oid,
 249         to_check).unsafe_get0();
 250       if (result.empty()) {
 251         EXPECT_EQ(omap.find(key), omap.end());
 252       } else {
 253         auto iter = omap.find(key);
 254         EXPECT_NE(iter, omap.end());
 255         if (iter != omap.end()) {
 256           EXPECT_EQ(result.size(), 1);
 257           EXPECT_EQ(iter->second, result.begin()->second);
 258         }
 259       }
 260     }
 261
 262     void check_omap(SeaStore &seastore) {
 263       auto iter = seastore.get_omap_iterator(coll, oid).get0();
 264       iter->seek_to_first().get0();
 265       auto refiter = omap.begin();
 266       while (true) {
 267         if (!iter->valid() && refiter == omap.end())
 268           break;
 269
 270         if (!iter->valid() || refiter->first < iter->key()) {
 271           logger().debug(
 272             "check_omap: missing omap key {}",
 273             refiter->first);
 274           GTEST_FAIL() << "missing omap key " << refiter->first;
 275           ++refiter;
 276         } else if (refiter == omap.end() || refiter->first > iter->key()) {
 277           logger().debug(
 278             "check_omap: extra omap key {}",
 279             iter->key());
 280           GTEST_FAIL() << "extra omap key" << iter->key();
 281           iter->next().get0();
 282         } else {
 283           EXPECT_EQ(iter->value(), refiter->second);
 284           iter->next().get0();
 285           ++refiter;
 286         }
 287       }
 288     }
 289   };
 290
 291   map<ghobject_t, object_state_t> test_objects;
 292   object_state_t &get_object(
 293     const ghobject_t &oid) {
 294     return test_objects.emplace(
 295       std::make_pair(
 296         oid,
 297         object_state_t{coll_name, coll, oid})).first->second;
 298   }
 299
 300   void remove_object(
 301     object_state_t &sobj) {
 302
 303     sobj.remove(*seastore);
 304     auto erased = test_objects.erase(sobj.oid);
 305     ceph_assert(erased == 1);
 306   }
 307
 308   void validate_objects() const {
 309     std::vector<ghobject_t> oids;
 310     for (auto& [oid, obj] : test_objects) {
 311       oids.emplace_back(oid);
 312     }
 313     auto ret = seastore->list_objects(
 314         coll,
 315         ghobject_t(),
 316         ghobject_t::get_max(),
 317         std::numeric_limits<uint64_t>::max()).get0();
 318     EXPECT_EQ(std::get<1>(ret), ghobject_t::get_max());
 319     EXPECT_EQ(std::get<0>(ret), oids);
 320   }
 321 };
 322
 323 ghobject_t make_oid(int i) {
 324   stringstream ss;
 325   ss << "object_" << i;
 326   auto ret = ghobject_t(
 327     hobject_t(
 328       sobject_t(ss.str(), CEPH_NOSNAP)));
 329   ret.set_shard(shard_id_t(0));
 330   ret.hobj.nspace = "asdf";
 331   return ret;
 332 }
 333
 334 template <typename T, typename V>
 335 auto contains(const T &t, const V &v) {
 336   return std::find(
 337     t.begin(),
 338     t.end(),
 339     v) != t.end();
 340 }
 341
 342 TEST_F(seastore_test_t, collection_create_list_remove)
 343 {
 344   run_async([this] {
 345     coll_t test_coll{spg_t{pg_t{1, 0}}};
 346     {
 347       seastore->create_new_collection(test_coll).get0();
 348       {
 349         CTransaction t;
 350         t.create_collection(test_coll, 4);
 351         do_transaction(std::move(t));
 352       }
 353       auto collections = seastore->list_collections().get0();
 354       EXPECT_EQ(collections.size(), 2);
 355       EXPECT_TRUE(contains(collections, coll_name));
 356       EXPECT_TRUE(contains(collections,  test_coll));
 357     }
 358
 359     {
 360       {
 361         CTransaction t;
 362         t.remove_collection(test_coll);
 363         do_transaction(std::move(t));
 364       }
 365       auto collections = seastore->list_collections().get0();
 366       EXPECT_EQ(collections.size(), 1);
 367       EXPECT_TRUE(contains(collections, coll_name));
 368     }
 369   });
 370 }
 371
 372 TEST_F(seastore_test_t, meta) {
 373   run_async([this] {
 374     set_meta("key1", "value1");
 375     set_meta("key2", "value2");
 376
 377     const auto [ret1, value1] = get_meta("key1");
 378     const auto [ret2, value2] = get_meta("key2");
 379     EXPECT_EQ(ret1, 0);
 380     EXPECT_EQ(ret2, 0);
 381     EXPECT_EQ(value1, "value1");
 382     EXPECT_EQ(value2, "value2");
 383   });
 384 }
 385
 386 TEST_F(seastore_test_t, touch_stat_list_remove)
 387 {
 388   run_async([this] {
 389     auto &test_obj = get_object(make_oid(0));
 390     test_obj.touch(*seastore);
 391     test_obj.check_size(*seastore);
 392     validate_objects();
 393
 394     remove_object(test_obj);
 395     validate_objects();
 396   });
 397 }
 398
 399 bufferlist make_bufferlist(size_t len) {
 400   bufferptr ptr(len);
 401   bufferlist bl;
 402   bl.append(ptr);
 403   return bl;
 404 }
 405
 406 TEST_F(seastore_test_t, omap_test_simple)
 407 {
 408   run_async([this] {
 409     auto &test_obj = get_object(make_oid(0));
 410     test_obj.set_omap(
 411       *seastore,
 412       "asdf",
 413       make_bufferlist(128));
 414     test_obj.check_omap_key(
 415       *seastore,
 416       "asdf");
 417   });
 418 }
 419
 420 TEST_F(seastore_test_t, attr)
 421 {
 422   run_async([this] {
 423     auto& test_obj = get_object(make_oid(0));
 424
 425     std::string oi("asdfasdfasdf");
 426     bufferlist bl;
 427     encode(oi, bl);
 428     test_obj.set_attr(*seastore, OI_ATTR, bl);
 429
 430     std::string ss("fdsfdsfs");
 431     bl.clear();
 432     encode(ss, bl);
 433     test_obj.set_attr(*seastore, SS_ATTR, bl);
 434
 435     std::string test_val("ssssssssssss");
 436     bl.clear();
 437     encode(test_val, bl);
 438     test_obj.set_attr(*seastore, "test_key", bl);
 439
 440     auto attrs = test_obj.get_attrs(*seastore);
 441     std::string oi2;
 442     bufferlist bl2 = attrs[OI_ATTR];
 443     decode(oi2, bl2);
 444     bl2.clear();
 445     bl2 = attrs[SS_ATTR];
 446     std::string ss2;
 447     decode(ss2, bl2);
 448     std::string test_val2;
 449     bl2.clear();
 450     bl2 = attrs["test_key"];
 451     decode(test_val2, bl2);
 452     EXPECT_EQ(ss, ss2);
 453     EXPECT_EQ(oi, oi2);
 454     EXPECT_EQ(test_val, test_val2);
 455
 456     bl2.clear();
 457     bl2 = test_obj.get_attr(*seastore, "test_key");
 458     test_val2.clear();
 459     decode(test_val2, bl2);
 460     EXPECT_EQ(test_val, test_val2);
 461
 462     std::cout << "test_key passed" << std::endl;
 463     char ss_array[256] = {0};
 464     std::string ss_str(&ss_array[0], 256);
 465     bl.clear();
 466     encode(ss_str, bl);
 467     test_obj.set_attr(*seastore, SS_ATTR, bl);
 468
 469     attrs = test_obj.get_attrs(*seastore);
 470     std::cout << "got attr" << std::endl;
 471     bl2.clear();
 472     bl2 = attrs[SS_ATTR];
 473     std::string ss_str2;
 474     decode(ss_str2, bl2);
 475     EXPECT_EQ(ss_str, ss_str2);
 476
 477     bl2.clear();
 478     ss_str2.clear();
 479     bl2 = test_obj.get_attr(*seastore, SS_ATTR);
 480     decode(ss_str2, bl2);
 481     EXPECT_EQ(ss_str, ss_str2);
 482   });
 483 }
 484
 485 TEST_F(seastore_test_t, omap_test_iterator)
 486 {
 487   run_async([this] {
 488     auto make_key = [](unsigned i) {
 489       std::stringstream ss;
 490       ss << "key" << i;
 491       return ss.str();
 492     };
 493     auto &test_obj = get_object(make_oid(0));
 494     for (unsigned i = 0; i < 20; ++i) {
 495       test_obj.set_omap(
 496         *seastore,
 497         make_key(i),
 498         make_bufferlist(128));
 499     }
 500     test_obj.check_omap(*seastore);
 501   });
 502 }
 503
 504
 505 TEST_F(seastore_test_t, simple_extent_test)
 506 {
 507   run_async([this] {
 508     auto &test_obj = get_object(make_oid(0));
 509     test_obj.write(
 510       *seastore,
 511       1024,
 512       1024,
 513       'a');
 514     test_obj.read(
 515       *seastore,
 516       1024,
 517       1024);
 518     test_obj.check_size(*seastore);
 519   });
 520 }