ceph/src/test/osd/TestOSDMap.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 #include "gtest/gtest.h"
   3 #include "osd/OSDMap.h"
   4 #include "osd/OSDMapMapping.h"
   5
   6 #include "global/global_context.h"
   7 #include "global/global_init.h"
   8 #include "common/common_init.h"
   9
  10 #include <iostream>
  11
  12 using namespace std;
  13
  14 int main(int argc, char **argv) {
  15   std::vector<const char*> args(argv, argv+argc);
  16   auto cct = global_init(nullptr, args, CEPH_ENTITY_TYPE_CLIENT,
  17                          CODE_ENVIRONMENT_UTILITY,
  18                          CINIT_FLAG_NO_DEFAULT_CONFIG_FILE);
  19   common_init_finish(g_ceph_context);
  20   // make sure we have 3 copies, or some tests won't work
  21   g_ceph_context->_conf->set_val("osd_pool_default_size", "3", false);
  22   // our map is flat, so just try and split across OSDs, not hosts or whatever
  23   g_ceph_context->_conf->set_val("osd_crush_chooseleaf_type", "0", false);
  24   ::testing::InitGoogleTest(&argc, argv);
  25   return RUN_ALL_TESTS();
  26 }
  27
  28 class OSDMapTest : public testing::Test {
  29   const static int num_osds = 6;
  30 public:
  31   OSDMap osdmap;
  32   OSDMapMapping mapping;
  33
  34   OSDMapTest() {}
  35
  36   void set_up_map() {
  37     uuid_d fsid;
  38     osdmap.build_simple(g_ceph_context, 0, fsid, num_osds, 6, 6);
  39     OSDMap::Incremental pending_inc(osdmap.get_epoch() + 1);
  40     pending_inc.fsid = osdmap.get_fsid();
  41     entity_addr_t sample_addr;
  42     uuid_d sample_uuid;
  43     for (int i = 0; i < num_osds; ++i) {
  44       sample_uuid.generate_random();
  45       sample_addr.nonce = i;
  46       pending_inc.new_state[i] = CEPH_OSD_EXISTS | CEPH_OSD_NEW;
  47       pending_inc.new_up_client[i] = sample_addr;
  48       pending_inc.new_up_cluster[i] = sample_addr;
  49       pending_inc.new_hb_back_up[i] = sample_addr;
  50       pending_inc.new_hb_front_up[i] = sample_addr;
  51       pending_inc.new_weight[i] = CEPH_OSD_IN;
  52       pending_inc.new_uuid[i] = sample_uuid;
  53     }
  54     osdmap.apply_incremental(pending_inc);
  55
  56     // Create an EC ruleset and a pool using it
  57     int r = osdmap.crush->add_simple_ruleset("erasure", "default", "osd",
  58                                              "indep", pg_pool_t::TYPE_ERASURE,
  59                                              &cerr);
  60
  61     OSDMap::Incremental new_pool_inc(osdmap.get_epoch() + 1);
  62     new_pool_inc.new_pool_max = osdmap.get_pool_max();
  63     new_pool_inc.fsid = osdmap.get_fsid();
  64     pg_pool_t empty;
  65     uint64_t pool_id = ++new_pool_inc.new_pool_max;
  66     pg_pool_t *p = new_pool_inc.get_new_pool(pool_id, &empty);
  67     p->size = 3;
  68     p->set_pg_num(64);
  69     p->set_pgp_num(64);
  70     p->type = pg_pool_t::TYPE_ERASURE;
  71     p->crush_ruleset = r;
  72     new_pool_inc.new_pool_names[pool_id] = "ec";
  73     osdmap.apply_incremental(new_pool_inc);
  74   }
  75   unsigned int get_num_osds() { return num_osds; }
  76
  77   void test_mappings(int pool,
  78                      int num,
  79                      vector<int> *any,
  80                      vector<int> *first,
  81                      vector<int> *primary) {
  82     mapping.update(osdmap);
  83     for (int i=0; i<num; ++i) {
  84       vector<int> up, acting;
  85       int up_primary, acting_primary;
  86       pg_t pgid(i, pool);
  87       osdmap.pg_to_up_acting_osds(pgid,
  88                                   &up, &up_primary, &acting, &acting_primary);
  89       for (unsigned j=0; j<acting.size(); ++j)
  90         (*any)[acting[j]]++;
  91       if (!acting.empty())
  92         (*first)[acting[0]]++;
  93       if (acting_primary >= 0)
  94         (*primary)[acting_primary]++;
  95
  96       // compare to precalc mapping
  97       vector<int> up2, acting2;
  98       int up_primary2, acting_primary2;
  99       pgid = osdmap.raw_pg_to_pg(pgid);
 100       mapping.get(pgid, &up2, &up_primary2, &acting2, &acting_primary2);
 101       ASSERT_EQ(up, up2);
 102       ASSERT_EQ(up_primary, up_primary2);
 103       ASSERT_EQ(acting, acting2);
 104       ASSERT_EQ(acting_primary, acting_primary2);
 105     }
 106   }
 107 };
 108
 109 TEST_F(OSDMapTest, Create) {
 110   set_up_map();
 111   ASSERT_EQ(get_num_osds(), (unsigned)osdmap.get_max_osd());
 112   ASSERT_EQ(get_num_osds(), osdmap.get_num_in_osds());
 113 }
 114
 115 TEST_F(OSDMapTest, Features) {
 116   // with EC pool
 117   set_up_map();
 118   uint64_t features = osdmap.get_features(CEPH_ENTITY_TYPE_OSD, NULL);
 119   ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES);
 120   ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES2);
 121   ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES3);
 122   ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_V2);
 123   ASSERT_TRUE(features & CEPH_FEATURE_OSD_ERASURE_CODES);
 124   ASSERT_TRUE(features & CEPH_FEATURE_OSDHASHPSPOOL);
 125   ASSERT_TRUE(features & CEPH_FEATURE_OSD_PRIMARY_AFFINITY);
 126
 127   // clients have a slightly different view
 128   features = osdmap.get_features(CEPH_ENTITY_TYPE_CLIENT, NULL);
 129   ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES);
 130   ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES2);
 131   ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES3);
 132   ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_V2);
 133   ASSERT_FALSE(features & CEPH_FEATURE_OSD_ERASURE_CODES);  // dont' need this
 134   ASSERT_TRUE(features & CEPH_FEATURE_OSDHASHPSPOOL);
 135   ASSERT_TRUE(features & CEPH_FEATURE_OSD_PRIMARY_AFFINITY);
 136
 137   // remove teh EC pool, but leave the rule.  add primary affinity.
 138   {
 139     OSDMap::Incremental new_pool_inc(osdmap.get_epoch() + 1);
 140     new_pool_inc.old_pools.insert(osdmap.lookup_pg_pool_name("ec"));
 141     new_pool_inc.new_primary_affinity[0] = 0x8000;
 142     osdmap.apply_incremental(new_pool_inc);
 143   }
 144
 145   features = osdmap.get_features(CEPH_ENTITY_TYPE_MON, NULL);
 146   ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES);
 147   ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES2);
 148   ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES3); // shared bit with primary affinity
 149   ASSERT_FALSE(features & CEPH_FEATURE_CRUSH_V2);
 150   ASSERT_FALSE(features & CEPH_FEATURE_OSD_ERASURE_CODES);
 151   ASSERT_TRUE(features & CEPH_FEATURE_OSDHASHPSPOOL);
 152   ASSERT_TRUE(features & CEPH_FEATURE_OSD_PRIMARY_AFFINITY);
 153
 154   // FIXME: test tiering feature bits
 155 }
 156
 157 TEST_F(OSDMapTest, MapPG) {
 158   set_up_map();
 159
 160   pg_t rawpg(0, 0, -1);
 161   pg_t pgid = osdmap.raw_pg_to_pg(rawpg);
 162   vector<int> up_osds, acting_osds;
 163   int up_primary, acting_primary;
 164
 165   osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
 166                               &acting_osds, &acting_primary);
 167
 168   vector<int> old_up_osds, old_acting_osds;
 169   osdmap.pg_to_up_acting_osds(pgid, old_up_osds, old_acting_osds);
 170   ASSERT_EQ(old_up_osds, up_osds);
 171   ASSERT_EQ(old_acting_osds, acting_osds);
 172
 173   ASSERT_EQ(osdmap.get_pg_pool(0)->get_size(), up_osds.size());
 174 }
 175
 176 TEST_F(OSDMapTest, MapFunctionsMatch) {
 177   // TODO: make sure pg_to_up_acting_osds and pg_to_acting_osds match
 178   set_up_map();
 179
 180   pg_t rawpg(0, 0, -1);
 181   pg_t pgid = osdmap.raw_pg_to_pg(rawpg);
 182   vector<int> up_osds, acting_osds;
 183   int up_primary, acting_primary;
 184
 185   osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
 186                               &acting_osds, &acting_primary);
 187
 188   vector<int> up_osds_two, acting_osds_two;
 189
 190   osdmap.pg_to_up_acting_osds(pgid, up_osds_two, acting_osds_two);
 191
 192   ASSERT_EQ(up_osds, up_osds_two);
 193   ASSERT_EQ(acting_osds, acting_osds_two);
 194
 195   int acting_primary_two;
 196   osdmap.pg_to_acting_osds(pgid, &acting_osds_two, &acting_primary_two);
 197   EXPECT_EQ(acting_osds, acting_osds_two);
 198   EXPECT_EQ(acting_primary, acting_primary_two);
 199   osdmap.pg_to_acting_osds(pgid, acting_osds_two);
 200   EXPECT_EQ(acting_osds, acting_osds_two);
 201 }
 202
 203 /** This test must be removed or modified appropriately when we allow
 204  * other ways to specify a primary. */
 205 TEST_F(OSDMapTest, PrimaryIsFirst) {
 206   set_up_map();
 207
 208   pg_t rawpg(0, 0, -1);
 209   pg_t pgid = osdmap.raw_pg_to_pg(rawpg);
 210   vector<int> up_osds, acting_osds;
 211   int up_primary, acting_primary;
 212
 213   osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
 214                               &acting_osds, &acting_primary);
 215   EXPECT_EQ(up_osds[0], up_primary);
 216   EXPECT_EQ(acting_osds[0], acting_primary);
 217 }
 218
 219 TEST_F(OSDMapTest, PGTempRespected) {
 220   set_up_map();
 221
 222   pg_t rawpg(0, 0, -1);
 223   pg_t pgid = osdmap.raw_pg_to_pg(rawpg);
 224   vector<int> up_osds, acting_osds;
 225   int up_primary, acting_primary;
 226
 227   osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
 228                               &acting_osds, &acting_primary);
 229
 230   // copy and swap first and last element in acting_osds
 231   vector<int> new_acting_osds(acting_osds);
 232   int first = new_acting_osds[0];
 233   new_acting_osds[0] = *new_acting_osds.rbegin();
 234   *new_acting_osds.rbegin() = first;
 235
 236   // apply pg_temp to osdmap
 237   OSDMap::Incremental pgtemp_map(osdmap.get_epoch() + 1);
 238   pgtemp_map.new_pg_temp[pgid] = mempool::osdmap::vector<int>(
 239     new_acting_osds.begin(), new_acting_osds.end());
 240   osdmap.apply_incremental(pgtemp_map);
 241
 242   osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
 243                               &acting_osds, &acting_primary);
 244   EXPECT_EQ(new_acting_osds, acting_osds);
 245 }
 246
 247 TEST_F(OSDMapTest, PrimaryTempRespected) {
 248   set_up_map();
 249
 250   pg_t rawpg(0, 0, -1);
 251   pg_t pgid = osdmap.raw_pg_to_pg(rawpg);
 252   vector<int> up_osds;
 253   vector<int> acting_osds;
 254   int up_primary, acting_primary;
 255
 256   osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
 257                               &acting_osds, &acting_primary);
 258
 259   // make second OSD primary via incremental
 260   OSDMap::Incremental pgtemp_map(osdmap.get_epoch() + 1);
 261   pgtemp_map.new_primary_temp[pgid] = acting_osds[1];
 262   osdmap.apply_incremental(pgtemp_map);
 263
 264   osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
 265                               &acting_osds, &acting_primary);
 266   EXPECT_EQ(acting_primary, acting_osds[1]);
 267 }
 268
 269 TEST_F(OSDMapTest, CleanTemps) {
 270   set_up_map();
 271
 272   OSDMap::Incremental pgtemp_map(osdmap.get_epoch() + 1);
 273   OSDMap::Incremental pending_inc(osdmap.get_epoch() + 2);
 274   pg_t pga = osdmap.raw_pg_to_pg(pg_t(0, 0));
 275   {
 276     vector<int> up_osds, acting_osds;
 277     int up_primary, acting_primary;
 278     osdmap.pg_to_up_acting_osds(pga, &up_osds, &up_primary,
 279                                 &acting_osds, &acting_primary);
 280     pgtemp_map.new_pg_temp[pga] = mempool::osdmap::vector<int>(
 281       up_osds.begin(), up_osds.end());
 282     pgtemp_map.new_primary_temp[pga] = up_primary;
 283   }
 284   pg_t pgb = osdmap.raw_pg_to_pg(pg_t(1, 0));
 285   {
 286     vector<int> up_osds, acting_osds;
 287     int up_primary, acting_primary;
 288     osdmap.pg_to_up_acting_osds(pgb, &up_osds, &up_primary,
 289                                 &acting_osds, &acting_primary);
 290     pending_inc.new_pg_temp[pgb] = mempool::osdmap::vector<int>(
 291       up_osds.begin(), up_osds.end());
 292     pending_inc.new_primary_temp[pgb] = up_primary;
 293   }
 294
 295   osdmap.apply_incremental(pgtemp_map);
 296
 297   OSDMap::clean_temps(g_ceph_context, osdmap, &pending_inc);
 298
 299   EXPECT_TRUE(pending_inc.new_pg_temp.count(pga) &&
 300               pending_inc.new_pg_temp[pga].size() == 0);
 301   EXPECT_EQ(-1, pending_inc.new_primary_temp[pga]);
 302
 303   EXPECT_TRUE(!pending_inc.new_pg_temp.count(pgb) &&
 304               !pending_inc.new_primary_temp.count(pgb));
 305 }
 306
 307 TEST_F(OSDMapTest, KeepsNecessaryTemps) {
 308   set_up_map();
 309
 310   pg_t rawpg(0, 0, -1);
 311   pg_t pgid = osdmap.raw_pg_to_pg(rawpg);
 312   vector<int> up_osds, acting_osds;
 313   int up_primary, acting_primary;
 314
 315   osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary,
 316                               &acting_osds, &acting_primary);
 317
 318   // find unused OSD and stick it in there
 319   OSDMap::Incremental pgtemp_map(osdmap.get_epoch() + 1);
 320   // find an unused osd and put it in place of the first one
 321   int i = 0;
 322   for(; i != (int)get_num_osds(); ++i) {
 323     bool in_use = false;
 324     for (vector<int>::iterator osd_it = up_osds.begin();
 325          osd_it != up_osds.end();
 326          ++osd_it) {
 327       if (i == *osd_it) {
 328         in_use = true;
 329         break;
 330       }
 331     }
 332     if (!in_use) {
 333       up_osds[1] = i;
 334       break;
 335     }
 336   }
 337   if (i == (int)get_num_osds())
 338     FAIL() << "did not find unused OSD for temp mapping";
 339
 340   pgtemp_map.new_pg_temp[pgid] = mempool::osdmap::vector<int>(
 341     up_osds.begin(), up_osds.end());
 342   pgtemp_map.new_primary_temp[pgid] = up_osds[1];
 343   osdmap.apply_incremental(pgtemp_map);
 344
 345   OSDMap::Incremental pending_inc(osdmap.get_epoch() + 1);
 346
 347   OSDMap::clean_temps(g_ceph_context, osdmap, &pending_inc);
 348   EXPECT_FALSE(pending_inc.new_pg_temp.count(pgid));
 349   EXPECT_FALSE(pending_inc.new_primary_temp.count(pgid));
 350 }
 351
 352 TEST_F(OSDMapTest, PrimaryAffinity) {
 353   set_up_map();
 354
 355   /*
 356   osdmap.print(cout);
 357   Formatter *f = Formatter::create("json-pretty");
 358   f->open_object_section("CRUSH");
 359   osdmap.crush->dump(f);
 360   f->close_section();
 361   f->flush(cout);
 362   delete f;
 363   */
 364
 365   int n = get_num_osds();
 366   for (map<int64_t,pg_pool_t>::const_iterator p = osdmap.get_pools().begin();
 367        p != osdmap.get_pools().end();
 368        ++p) {
 369     int pool = p->first;
 370     cout << "pool " << pool << std::endl;
 371     {
 372       vector<int> any(n, 0);
 373       vector<int> first(n, 0);
 374       vector<int> primary(n, 0);
 375       test_mappings(0, 10000, &any, &first, &primary);
 376       for (int i=0; i<n; ++i) {
 377         //cout << "osd." << i << " " << any[i] << " " << first[i] << " " << primary[i] << std::endl;
 378         ASSERT_LT(0, any[i]);
 379         ASSERT_LT(0, first[i]);
 380         ASSERT_LT(0, primary[i]);
 381       }
 382     }
 383
 384     osdmap.set_primary_affinity(0, 0);
 385     osdmap.set_primary_affinity(1, 0);
 386     {
 387       vector<int> any(n, 0);
 388       vector<int> first(n, 0);
 389       vector<int> primary(n, 0);
 390       test_mappings(pool, 10000, &any, &first, &primary);
 391       for (int i=0; i<n; ++i) {
 392         //cout << "osd." << i << " " << any[i] << " " << first[i] << " " << primary[i] << std::endl;
 393         ASSERT_LT(0, any[i]);
 394         if (i >= 2) {
 395           ASSERT_LT(0, first[i]);
 396           ASSERT_LT(0, primary[i]);
 397         } else {
 398           if (p->second.is_replicated()) {
 399             ASSERT_EQ(0, first[i]);
 400           }
 401           ASSERT_EQ(0, primary[i]);
 402         }
 403       }
 404     }
 405
 406     osdmap.set_primary_affinity(0, 0x8000);
 407     osdmap.set_primary_affinity(1, 0);
 408     {
 409       vector<int> any(n, 0);
 410       vector<int> first(n, 0);
 411       vector<int> primary(n, 0);
 412       test_mappings(pool, 10000, &any, &first, &primary);
 413       for (int i=0; i<n; ++i) {
 414         //cout << "osd." << i << " " << any[i] << " " << first[i] << " " << primary[i] << std::endl;
 415         ASSERT_LT(0, any[i]);
 416         if (i >= 2) {
 417           ASSERT_LT(0, first[i]);
 418           ASSERT_LT(0, primary[i]);
 419         } else if (i == 1) {
 420           if (p->second.is_replicated()) {
 421             ASSERT_EQ(0, first[i]);
 422           }
 423           ASSERT_EQ(0, primary[i]);
 424         } else {
 425           ASSERT_LT(10000/6/4, primary[0]);
 426           ASSERT_GT(10000/6/4*3, primary[0]);
 427         }
 428       }
 429     }
 430
 431     osdmap.set_primary_affinity(0, 0x10000);
 432     osdmap.set_primary_affinity(1, 0x10000);
 433   }
 434 }