ceph/src/test/os/TestLFNIndex.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph - scalable distributed file system
   5  *
   6  * Copyright (C) 2013 Cloudwatt <libre.licensing@cloudwatt.com>
   7  *
   8  * Author: Loic Dachary <loic@dachary.org>
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU Library Public License as published by
  12  * the Free Software Foundation; either version 2, or (at your option)
  13  * any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU Library Public License for more details.
  19  *
  20  */
  21
  22 #include <stdio.h>
  23 #include <signal.h>
  24 #include "os/filestore/LFNIndex.h"
  25 #include "os/filestore/chain_xattr.h"
  26 #include "common/ceph_argparse.h"
  27 #include "global/global_init.h"
  28 #include <gtest/gtest.h>
  29
  30 using namespace std;
  31
  32 class TestWrapLFNIndex : public LFNIndex {
  33 public:
  34   TestWrapLFNIndex(CephContext* cct,
  35                    coll_t collection,
  36                    const char *base_path,
  37                    uint32_t index_version)
  38     : LFNIndex(cct, collection, base_path, index_version) {}
  39
  40   uint32_t collection_version() override {
  41     return index_version;
  42   }
  43
  44   int cleanup() override { return 0; }
  45
  46   int _split(
  47                      uint32_t match,
  48                      uint32_t bits,
  49                      CollectionIndex* dest
  50                      ) override { return 0; }
  51   int _merge(
  52                      uint32_t bits,
  53                      CollectionIndex* dest
  54                      ) override { return 0; }
  55
  56   void test_generate_and_parse(const ghobject_t &hoid, const std::string &mangled_expected) {
  57     const std::string mangled_name = lfn_generate_object_name(hoid);
  58     EXPECT_EQ(mangled_expected, mangled_name);
  59     ghobject_t hoid_parsed;
  60     EXPECT_EQ(0, lfn_parse_object_name(mangled_name, &hoid_parsed));
  61     EXPECT_EQ(hoid, hoid_parsed);
  62   }
  63
  64 protected:
  65   int _init() override { return 0; }
  66
  67   int _created(
  68                        const vector<string> &path,
  69                        const ghobject_t &hoid,
  70                        const string &mangled_name
  71                        ) override { return 0; }
  72
  73   int _remove(
  74                       const vector<string> &path,
  75                       const ghobject_t &hoid,
  76                       const string &mangled_name
  77                       ) override { return 0; }
  78
  79   int _lookup(
  80                       const ghobject_t &hoid,
  81                       vector<string> *path,
  82                       string *mangled_name,
  83                       int *exists
  84                       ) override { return 0; }
  85
  86   int _collection_list_partial(
  87                                        const ghobject_t &start,
  88                                        const ghobject_t &end,
  89                                        int max_count,
  90                                        vector<ghobject_t> *ls,
  91                                        ghobject_t *next
  92                                        ) override { return 0; }
  93   int _pre_hash_collection(
  94                                    uint32_t pg_num,
  95                                    uint64_t expected_num_objs
  96                                   ) override { return 0; }
  97
  98 };
  99
 100 class TestHASH_INDEX_TAG : public TestWrapLFNIndex, public ::testing::Test {
 101 public:
 102   TestHASH_INDEX_TAG()
 103     : TestWrapLFNIndex(g_ceph_context, coll_t(), "PATH_1",
 104                        CollectionIndex::HASH_INDEX_TAG) {
 105   }
 106 };
 107
 108 TEST_F(TestHASH_INDEX_TAG, generate_and_parse_name) {
 109   const vector<string> path;
 110   const std::string key;
 111   uint64_t hash = 0xABABABAB;
 112   uint64_t pool = -1;
 113
 114   test_generate_and_parse(ghobject_t(hobject_t(object_t(".A/B_\\C.D"), key, CEPH_NOSNAP, hash, pool, "")),
 115                           "\\.A\\sB_\\\\C.D_head_ABABABAB");
 116   test_generate_and_parse(ghobject_t(hobject_t(object_t("DIR_A"), key, CEPH_NOSNAP, hash, pool, "")),
 117                           "\\dA_head_ABABABAB");
 118 }
 119
 120 class TestHASH_INDEX_TAG_2 : public TestWrapLFNIndex, public ::testing::Test {
 121 public:
 122   TestHASH_INDEX_TAG_2()
 123     : TestWrapLFNIndex(g_ceph_context,
 124                        coll_t(), "PATH_1", CollectionIndex::HASH_INDEX_TAG_2) {
 125   }
 126 };
 127
 128 TEST_F(TestHASH_INDEX_TAG_2, generate_and_parse_name) {
 129   const vector<string> path;
 130   const std::string key("KEY");
 131   uint64_t hash = 0xABABABAB;
 132   uint64_t pool = -1;
 133
 134   {
 135     std::string name(".XA/B_\\C.D");
 136     name[1] = '\0';
 137     ghobject_t hoid(hobject_t(object_t(name), key, CEPH_NOSNAP, hash, pool, ""));
 138
 139     test_generate_and_parse(hoid, "\\.\\nA\\sB\\u\\\\C.D_KEY_head_ABABABAB");
 140   }
 141   test_generate_and_parse(ghobject_t(hobject_t(object_t("DIR_A"), key, CEPH_NOSNAP, hash, pool, "")),
 142                           "\\dA_KEY_head_ABABABAB");
 143 }
 144
 145 class TestHOBJECT_WITH_POOL : public TestWrapLFNIndex, public ::testing::Test {
 146 public:
 147   TestHOBJECT_WITH_POOL()
 148     : TestWrapLFNIndex(g_ceph_context, coll_t(),
 149                        "PATH_1", CollectionIndex::HOBJECT_WITH_POOL) {
 150   }
 151 };
 152
 153 TEST_F(TestHOBJECT_WITH_POOL, generate_and_parse_name) {
 154   const vector<string> path;
 155   const std::string key("KEY");
 156   uint64_t hash = 0xABABABAB;
 157   uint64_t pool = 0xCDCDCDCD;
 158   int64_t gen = 0xefefefefef;
 159   shard_id_t shard_id(0xb);
 160
 161   {
 162     std::string name(".XA/B_\\C.D");
 163     name[1] = '\0';
 164     ghobject_t hoid(hobject_t(object_t(name), key, CEPH_NOSNAP, hash, pool, ""));
 165     hoid.hobj.nspace = "NSPACE";
 166
 167     test_generate_and_parse(hoid, "\\.\\nA\\sB\\u\\\\C.D_KEY_head_ABABABAB_NSPACE_cdcdcdcd");
 168   }
 169   {
 170     ghobject_t hoid(hobject_t(object_t("DIR_A"), key, CEPH_NOSNAP, hash, pool, ""));
 171     hoid.hobj.nspace = "NSPACE";
 172
 173     test_generate_and_parse(hoid, "\\dA_KEY_head_ABABABAB_NSPACE_cdcdcdcd");
 174   }
 175   {
 176     std::string name(".XA/B_\\C.D");
 177     name[1] = '\0';
 178     ghobject_t hoid(hobject_t(object_t(name), key, CEPH_NOSNAP, hash, pool, ""), gen, shard_id);
 179     hoid.hobj.nspace = "NSPACE";
 180
 181     test_generate_and_parse(hoid, "\\.\\nA\\sB\\u\\\\C.D_KEY_head_ABABABAB_NSPACE_cdcdcdcd_efefefefef_b");
 182   }
 183   {
 184     ghobject_t hoid(hobject_t(object_t("DIR_A"), key, CEPH_NOSNAP, hash, pool, ""), gen, shard_id);
 185     hoid.hobj.nspace = "NSPACE";
 186
 187     test_generate_and_parse(hoid, "\\dA_KEY_head_ABABABAB_NSPACE_cdcdcdcd_efefefefef_b");
 188   }
 189 }
 190
 191 class TestLFNIndex : public TestWrapLFNIndex, public ::testing::Test {
 192 public:
 193   TestLFNIndex()
 194     : TestWrapLFNIndex(g_ceph_context, coll_t(), "PATH_1",
 195                        CollectionIndex::HOBJECT_WITH_POOL) {
 196   }
 197
 198   void SetUp() override {
 199     ::chmod("PATH_1", 0700);
 200     ASSERT_EQ(0, ::system("rm -fr PATH_1"));
 201     ASSERT_EQ(0, ::mkdir("PATH_1", 0700));
 202   }
 203
 204   void TearDown() override {
 205     ASSERT_EQ(0, ::system("rm -fr PATH_1"));
 206   }
 207 };
 208
 209 TEST_F(TestLFNIndex, remove_object) {
 210   const vector<string> path;
 211
 212   //
 213   // small object name removal
 214   //
 215   {
 216     std::string mangled_name;
 217     int exists = 666;
 218     ghobject_t hoid(hobject_t(sobject_t("ABC", CEPH_NOSNAP)));
 219
 220     EXPECT_EQ(0, ::chmod("PATH_1", 0000));
 221     if (getuid() != 0) {
 222       EXPECT_EQ(-EACCES, remove_object(path, hoid));
 223     }
 224     EXPECT_EQ(0, ::chmod("PATH_1", 0700));
 225     EXPECT_EQ(-ENOENT, remove_object(path, hoid));
 226     EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
 227     const std::string pathname("PATH_1/" + mangled_name);
 228     EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
 229     EXPECT_EQ(0, remove_object(path, hoid));
 230     EXPECT_EQ(-1, ::access(pathname.c_str(), 0));
 231     EXPECT_EQ(ENOENT, errno);
 232   }
 233   //
 234   // long object name removal of a single file
 235   //
 236   {
 237     std::string mangled_name;
 238     int exists;
 239     const std::string object_name(1024, 'A');
 240     ghobject_t hoid(hobject_t(sobject_t(object_name, CEPH_NOSNAP)));
 241
 242     EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
 243     EXPECT_EQ(0, exists);
 244     EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
 245     std::string pathname("PATH_1/" + mangled_name);
 246     EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
 247     EXPECT_EQ(0, created(hoid, pathname.c_str()));
 248
 249     EXPECT_EQ(0, remove_object(path, hoid));
 250     EXPECT_EQ(-1, ::access(pathname.c_str(), 0));
 251     EXPECT_EQ(ENOENT, errno);
 252   }
 253
 254   //
 255   // long object name removal of the last file
 256   //
 257   {
 258     std::string mangled_name;
 259     int exists;
 260     const std::string object_name(1024, 'A');
 261     ghobject_t hoid(hobject_t(sobject_t(object_name, CEPH_NOSNAP)));
 262
 263     //
 264     //   PATH_1/AAA..._0_long => does not match long object name
 265     //
 266     EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
 267     EXPECT_EQ(0, exists);
 268     EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
 269     std::string pathname("PATH_1/" + mangled_name);
 270     EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
 271     EXPECT_EQ(0, created(hoid, pathname.c_str()));
 272     string LFN_ATTR = "user.cephos.lfn";
 273     if (index_version != HASH_INDEX_TAG) {
 274       char buf[100];
 275       snprintf(buf, sizeof(buf), "%d", index_version);
 276       LFN_ATTR += string(buf);
 277     }
 278     const std::string object_name_1 = object_name + "SUFFIX";
 279     EXPECT_EQ(object_name_1.size(), (unsigned)chain_setxattr(pathname.c_str(), LFN_ATTR.c_str(), object_name_1.c_str(), object_name_1.size()));
 280
 281     //
 282     //   PATH_1/AAA..._1_long => matches long object name
 283     //
 284     std::string mangled_name_1;
 285     exists = 666;
 286     EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name_1, &exists));
 287     EXPECT_NE(std::string::npos, mangled_name_1.find("1_long"));
 288     EXPECT_EQ(0, exists);
 289     std::string pathname_1("PATH_1/" + mangled_name_1);
 290     auto retvalue = ::creat(pathname_1.c_str(), 0600);
 291     ceph_assert(retvalue > 2);
 292     EXPECT_EQ(0, ::close(retvalue));
 293     EXPECT_EQ(0, created(hoid, pathname_1.c_str()));
 294
 295     //
 296     // remove_object skips PATH_1/AAA..._0_long and removes PATH_1/AAA..._1_long
 297     //
 298     EXPECT_EQ(0, remove_object(path, hoid));
 299     EXPECT_EQ(0, ::access(pathname.c_str(), 0));
 300     EXPECT_EQ(-1, ::access(pathname_1.c_str(), 0));
 301     EXPECT_EQ(ENOENT, errno);
 302     EXPECT_EQ(0, ::unlink(pathname.c_str()));
 303   }
 304
 305   //
 306   // long object name removal of a file in the middle of the list
 307   //
 308   {
 309     std::string mangled_name;
 310     int exists;
 311     const std::string object_name(1024, 'A');
 312     ghobject_t hoid(hobject_t(sobject_t(object_name, CEPH_NOSNAP)));
 313
 314     //
 315     //   PATH_1/AAA..._0_long => matches long object name
 316     //
 317     EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
 318     EXPECT_EQ(0, exists);
 319     EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
 320     std::string pathname("PATH_1/" + mangled_name);
 321     EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
 322     EXPECT_EQ(0, created(hoid, pathname.c_str()));
 323     //
 324     //   PATH_1/AAA..._1_long => matches long object name
 325     //
 326     std::string mangled_name_1 = mangled_name;
 327     mangled_name_1.replace(mangled_name_1.find("0_long"), 6, "1_long");
 328     const std::string pathname_1("PATH_1/" + mangled_name_1);
 329     const std::string cmd("cp -a " + pathname + " " + pathname_1);
 330     EXPECT_EQ(0, ::system(cmd.c_str()));
 331     const string ATTR = "user.MARK";
 332     EXPECT_EQ((unsigned)1, (unsigned)chain_setxattr(pathname_1.c_str(), ATTR.c_str(), "Y", 1));
 333
 334     //
 335     // remove_object replaces the file to be removed with the last from the
 336     // collision list. In this case it replaces
 337     //    PATH_1/AAA..._0_long
 338     // with
 339     //    PATH_1/AAA..._1_long
 340     //
 341     EXPECT_EQ(0, remove_object(path, hoid));
 342     EXPECT_EQ(0, ::access(pathname.c_str(), 0));
 343     char buffer[1] = { 0, };
 344     EXPECT_EQ((unsigned)1, (unsigned)chain_getxattr(pathname.c_str(), ATTR.c_str(), buffer, 1));
 345     EXPECT_EQ('Y', buffer[0]);
 346     EXPECT_EQ(-1, ::access(pathname_1.c_str(), 0));
 347     EXPECT_EQ(ENOENT, errno);
 348   }
 349 }
 350
 351 TEST_F(TestLFNIndex, get_mangled_name) {
 352   const vector<string> path;
 353
 354   //
 355   // small object name
 356   //
 357   {
 358     std::string mangled_name;
 359     int exists = 666;
 360     ghobject_t hoid(hobject_t(sobject_t("ABC", CEPH_NOSNAP)));
 361
 362     EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
 363     EXPECT_NE(std::string::npos, mangled_name.find("ABC__head"));
 364     EXPECT_EQ(std::string::npos, mangled_name.find("0_long"));
 365     EXPECT_EQ(0, exists);
 366     const std::string pathname("PATH_1/" + mangled_name);
 367     EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
 368     EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
 369     EXPECT_NE(std::string::npos, mangled_name.find("ABC__head"));
 370     EXPECT_EQ(1, exists);
 371     EXPECT_EQ(0, ::unlink(pathname.c_str()));
 372   }
 373   //
 374   // long object name
 375   //
 376   {
 377     std::string mangled_name;
 378     int exists;
 379     const std::string object_name(1024, 'A');
 380     ghobject_t hoid(hobject_t(sobject_t(object_name, CEPH_NOSNAP)));
 381
 382     //
 383     // long version of the mangled name and no matching
 384     // file exists
 385     //
 386     mangled_name.clear();
 387     exists = 666;
 388     EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
 389     EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
 390     EXPECT_EQ(0, exists);
 391
 392     const std::string pathname("PATH_1/" + mangled_name);
 393
 394     //
 395     // if a file by the same name exists but does not have the
 396     // expected extended attribute, it is silently removed
 397     //
 398     mangled_name.clear();
 399     exists = 666;
 400     EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
 401     EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
 402     EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
 403     EXPECT_EQ(0, exists);
 404     EXPECT_EQ(-1, ::access(pathname.c_str(), 0));
 405     EXPECT_EQ(ENOENT, errno);
 406
 407     //
 408     // if a file by the same name exists but does not have the
 409     // expected extended attribute, and cannot be removed,
 410     // return on error
 411     //
 412     mangled_name.clear();
 413     exists = 666;
 414     EXPECT_EQ(0, ::close(::creat(pathname.c_str(), 0600)));
 415     EXPECT_EQ(0, ::chmod("PATH_1", 0500));
 416     if (getuid() != 0) {
 417       EXPECT_EQ(-EACCES, get_mangled_name(path, hoid, &mangled_name, &exists));
 418     }
 419     EXPECT_EQ("", mangled_name);
 420     EXPECT_EQ(666, exists);
 421     EXPECT_EQ(0, ::chmod("PATH_1", 0700));
 422     EXPECT_EQ(0, ::unlink(pathname.c_str()));
 423
 424     //
 425     // long version of the mangled name and a file
 426     // exists by that name and contains the long object name
 427     //
 428     mangled_name.clear();
 429     exists = 666;
 430     auto retvalue = ::creat(pathname.c_str(), 0600);
 431     ceph_assert(retvalue > 2);
 432     EXPECT_EQ(0, ::close(retvalue));
 433     EXPECT_EQ(0, created(hoid, pathname.c_str()));
 434     EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name, &exists));
 435     EXPECT_NE(std::string::npos, mangled_name.find("0_long"));
 436     EXPECT_EQ(1, exists);
 437     EXPECT_EQ(0, ::access(pathname.c_str(), 0));
 438
 439     //
 440     // long version of the mangled name and a file exists by that name
 441     // and contains a long object name with the same prefix but they
 442     // are not identical and it so happens that their SHA1 is
 443     // identical : a collision number is used to differentiate them
 444     //
 445     string LFN_ATTR = "user.cephos.lfn";
 446     if (index_version != HASH_INDEX_TAG) {
 447       char buf[100];
 448       snprintf(buf, sizeof(buf), "%d", index_version);
 449       LFN_ATTR += string(buf);
 450     }
 451     const std::string object_name_same_prefix = object_name + "SUFFIX";
 452     EXPECT_EQ(object_name_same_prefix.size(), (unsigned)chain_setxattr(pathname.c_str(), LFN_ATTR.c_str(), object_name_same_prefix.c_str(), object_name_same_prefix.size()));
 453     std::string mangled_name_same_prefix;
 454     exists = 666;
 455     EXPECT_EQ(0, get_mangled_name(path, hoid, &mangled_name_same_prefix, &exists));
 456     EXPECT_NE(std::string::npos, mangled_name_same_prefix.find("1_long"));
 457     EXPECT_EQ(0, exists);
 458
 459     EXPECT_EQ(0, ::unlink(pathname.c_str()));
 460   }
 461 }
 462
 463 int main(int argc, char **argv) {
 464   int fd = ::creat("detect", 0600);
 465   if (fd < 0){
 466     cerr << "failed to create file detect" << std::endl;
 467     return EXIT_FAILURE;
 468   }
 469   int ret = chain_fsetxattr(fd, "user.test", "A", 1);
 470   ::close(fd);
 471   ::unlink("detect");
 472   if (ret < 0) {
 473     cerr << "SKIP LFNIndex because unable to test for xattr" << std::endl;
 474   } else {
 475   auto args = argv_to_vec(argc, argv);
 476
 477     auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
 478                            CODE_ENVIRONMENT_UTILITY,
 479                            CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
 480     common_init_finish(g_ceph_context);
 481
 482     ::testing::InitGoogleTest(&argc, argv);
 483     return RUN_ALL_TESTS();
 484   }
 485 }
 486
 487 /*
 488  * Local Variables:
 489  * compile-command: "cd ../.. ;
 490  *   make unittest_lfnindex &&
 491  *   valgrind --tool=memcheck ./unittest_lfnindex \
 492  *   # --gtest_filter=TestLFNIndex.* --log-to-stderr=true --debug-filestore=20"
 493  * End:
 494  */