}
class OSDMapTest : public testing::Test {
- const static int num_osds = 6;
+ int num_osds = 6;
public:
OSDMap osdmap;
OSDMapMapping mapping;
OSDMapTest() {}
- void set_up_map() {
+ void set_up_map(int new_num_osds = 6, bool no_default_pools = false) {
+ num_osds = new_num_osds;
uuid_d fsid;
osdmap.build_simple(g_ceph_context, 0, fsid, num_osds);
OSDMap::Incremental pending_inc(osdmap.get_epoch() + 1);
pending_inc.new_uuid[i] = sample_uuid;
}
osdmap.apply_incremental(pending_inc);
+ if (no_default_pools) // do not create any default pool(s)
+ return;
// Create an EC ruleset and a pool using it
int r = osdmap.crush->add_simple_rule(
osdmap.apply_incremental(new_pool_inc);
}
unsigned int get_num_osds() { return num_osds; }
- void get_crush(CrushWrapper& newcrush) {
+ void get_crush(const OSDMap& tmap, CrushWrapper& newcrush) {
bufferlist bl;
- osdmap.crush->encode(bl, CEPH_FEATURES_SUPPORTED_DEFAULT);
+ tmap.crush->encode(bl, CEPH_FEATURES_SUPPORTED_DEFAULT);
bufferlist::iterator p = bl.begin();
newcrush.decode(p);
}
- int crush_move(const string &name, const vector<string> &argvec) {
+ int crush_move(OSDMap& tmap, const string &name, const vector<string> &argvec) {
map<string,string> loc;
CrushWrapper::parse_loc_map(argvec, &loc);
CrushWrapper newcrush;
- get_crush(newcrush);
+ get_crush(tmap, newcrush);
if (!newcrush.name_exists(name)) {
return -ENOENT;
}
err = newcrush.move_bucket(g_ceph_context, id, loc);
}
if (err >= 0) {
- OSDMap::Incremental pending_inc(osdmap.get_epoch() + 1);
+ OSDMap::Incremental pending_inc(tmap.get_epoch() + 1);
pending_inc.crush.clear();
newcrush.encode(pending_inc.crush, CEPH_FEATURES_SUPPORTED_DEFAULT);
- osdmap.apply_incremental(pending_inc);
+ tmap.apply_incremental(pending_inc);
err = 0;
}
} else {
return osdmap.crush->get_rule_id(name);
}
CrushWrapper newcrush;
- get_crush(newcrush);
+ get_crush(osdmap, newcrush);
string device_class;
stringstream ss;
int ruleno = newcrush.add_simple_rule(
move_to.push_back("root=default");
string host_loc = "host=" + host_name.str();
move_to.push_back(host_loc);
- int r = crush_move(osd_name.str(), move_to);
+ int r = crush_move(osdmap, osd_name.str(), move_to);
ASSERT_EQ(0, r);
}
const string upmap_rule = "upmap";
}
}
+ {
+ // http://tracker.ceph.com/issues/37968
+
+ // build a temporary crush topology of 2 hosts, 3 osds per host
+ OSDMap tmp; // use a tmpmap here, so we do not dirty origin map..
+ tmp.deepish_copy_from(osdmap);
+ const int expected_host_num = 2;
+ int osd_per_host = get_num_osds() / expected_host_num;
+ ASSERT_GE(osd_per_host, 3);
+ int index = 0;
+ for (int i = 0; i < (int)get_num_osds(); i++) {
+ if (i && i % osd_per_host == 0) {
+ ++index;
+ }
+ stringstream osd_name;
+ stringstream host_name;
+ vector<string> move_to;
+ osd_name << "osd." << i;
+ host_name << "host-" << index;
+ move_to.push_back("root=default");
+ string host_loc = "host=" + host_name.str();
+ move_to.push_back(host_loc);
+ auto r = crush_move(tmp, osd_name.str(), move_to);
+ ASSERT_EQ(0, r);
+ }
+
+ // build crush rule
+ CrushWrapper crush;
+ get_crush(tmp, crush);
+ string rule_name = "rule_37968";
+ int rule_type = pg_pool_t::TYPE_ERASURE;
+ ASSERT_TRUE(!crush.rule_exists(rule_name));
+ int rno;
+ for (rno = 0; rno < crush.get_max_rules(); rno++) {
+ if (!crush.rule_exists(rno) && !crush.ruleset_exists(rno))
+ break;
+ }
+ string root_name = "default";
+ int root = crush.get_item_id(root_name);
+ int min_size = 3;
+ int max_size = 4;
+ int steps = 6;
+ crush_rule *rule = crush_make_rule(steps, rno, rule_type, min_size, max_size);
+ int step = 0;
+ crush_rule_set_step(rule, step++, CRUSH_RULE_SET_CHOOSELEAF_TRIES, 5, 0);
+ crush_rule_set_step(rule, step++, CRUSH_RULE_SET_CHOOSE_TRIES, 100, 0);
+ crush_rule_set_step(rule, step++, CRUSH_RULE_TAKE, root, 0);
+ crush_rule_set_step(rule, step++, CRUSH_RULE_CHOOSE_INDEP, 2, 1 /* host*/);
+ crush_rule_set_step(rule, step++, CRUSH_RULE_CHOOSE_INDEP, 2, 0 /* osd */);
+ crush_rule_set_step(rule, step++, CRUSH_RULE_EMIT, 0, 0);
+ ASSERT_TRUE(step == steps);
+ auto r = crush_add_rule(crush.get_crush_map(), rule, rno);
+ ASSERT_TRUE(r >= 0);
+ crush.set_rule_name(rno, rule_name);
+ {
+ OSDMap::Incremental pending_inc(tmp.get_epoch() + 1);
+ pending_inc.crush.clear();
+ crush.encode(pending_inc.crush, CEPH_FEATURES_SUPPORTED_DEFAULT);
+ tmp.apply_incremental(pending_inc);
+ }
+
+ // create a erasuce-coded pool referencing the above rule
+ int64_t pool_37968;
+ {
+ OSDMap::Incremental new_pool_inc(tmp.get_epoch() + 1);
+ new_pool_inc.new_pool_max = tmp.get_pool_max();
+ new_pool_inc.fsid = tmp.get_fsid();
+ pg_pool_t empty;
+ pool_37968 = ++new_pool_inc.new_pool_max;
+ pg_pool_t *p = new_pool_inc.get_new_pool(pool_37968, &empty);
+ p->size = 4;
+ p->set_pg_num(8);
+ p->set_pgp_num(8);
+ p->type = pg_pool_t::TYPE_ERASURE;
+ p->crush_rule = rno;
+ p->set_flag(pg_pool_t::FLAG_HASHPSPOOL);
+ new_pool_inc.new_pool_names[pool_37968] = "pool_37968";
+ tmp.apply_incremental(new_pool_inc);
+ }
+
+ pg_t ec_pg(0, pool_37968);
+ pg_t ec_pgid = tmp.raw_pg_to_pg(ec_pg);
+ int from = -1;
+ int to = -1;
+ {
+ // insert a valid pg_upmap_item
+ vector<int> ec_up;
+ int ec_up_primary;
+ tmp.pg_to_raw_up(ec_pgid, &ec_up, &ec_up_primary);
+ ASSERT_TRUE(ec_up.size() == 4);
+ from = *(ec_up.begin());
+ ASSERT_TRUE(from >= 0);
+ auto parent = tmp.crush->get_parent_of_type(from, 1 /* host */, rno);
+ ASSERT_TRUE(parent < 0);
+ // pick an osd of the same parent with *from*
+ for (int i = 0; i < (int)get_num_osds(); i++) {
+ if (std::find(ec_up.begin(), ec_up.end(), i) == ec_up.end()) {
+ auto p = tmp.crush->get_parent_of_type(i, 1 /* host */, rno);
+ if (p == parent) {
+ to = i;
+ break;
+ }
+ }
+ }
+ ASSERT_TRUE(to >= 0);
+ ASSERT_TRUE(from != to);
+ vector<pair<int32_t,int32_t>> new_pg_upmap_items;
+ new_pg_upmap_items.push_back(make_pair(from, to));
+ OSDMap::Incremental pending_inc(tmp.get_epoch() + 1);
+ pending_inc.new_pg_upmap_items[ec_pgid] =
+ mempool::osdmap::vector<pair<int32_t,int32_t>>(
+ new_pg_upmap_items.begin(), new_pg_upmap_items.end());
+ tmp.apply_incremental(pending_inc);
+ ASSERT_TRUE(tmp.have_pg_upmaps(ec_pgid));
+ }
+ {
+ // *maybe_remove_pg_upmaps* should not remove the above upmap_item
+ OSDMap::Incremental pending_inc(tmp.get_epoch() + 1);
+ OSDMap nextmap;
+ nextmap.deepish_copy_from(tmp);
+ nextmap.maybe_remove_pg_upmaps(g_ceph_context, nextmap, &pending_inc);
+ tmp.apply_incremental(pending_inc);
+ ASSERT_TRUE(tmp.have_pg_upmaps(ec_pgid));
+ }
+ }
+
{
// TEST pg_upmap
{
}
}
+TEST_F(OSDMapTest, BUG_38897) {
+ // http://tracker.ceph.com/issues/38897
+ // build a fresh map with 12 OSDs, without any default pools
+ set_up_map(12, true);
+ const string pool_1("pool1");
+ const string pool_2("pool2");
+ int64_t pool_1_id = -1;
+
+ {
+ // build customized crush rule for "pool1"
+ string host_name = "host_for_pool_1";
+ // build a customized host to capture osd.1~5
+ for (int i = 1; i < 5; i++) {
+ stringstream osd_name;
+ vector<string> move_to;
+ osd_name << "osd." << i;
+ move_to.push_back("root=default");
+ string host_loc = "host=" + host_name;
+ move_to.push_back(host_loc);
+ auto r = crush_move(osdmap, osd_name.str(), move_to);
+ ASSERT_EQ(0, r);
+ }
+ CrushWrapper crush;
+ get_crush(osdmap, crush);
+ auto host_id = crush.get_item_id(host_name);
+ ASSERT_TRUE(host_id < 0);
+ string rule_name = "rule_for_pool1";
+ int rule_type = pg_pool_t::TYPE_REPLICATED;
+ ASSERT_TRUE(!crush.rule_exists(rule_name));
+ int rno;
+ for (rno = 0; rno < crush.get_max_rules(); rno++) {
+ if (!crush.rule_exists(rno) && !crush.ruleset_exists(rno))
+ break;
+ }
+ int min_size = 3;
+ int max_size = 3;
+ int steps = 7;
+ crush_rule *rule = crush_make_rule(steps, rno, rule_type, min_size, max_size);
+ int step = 0;
+ crush_rule_set_step(rule, step++, CRUSH_RULE_SET_CHOOSELEAF_TRIES, 5, 0);
+ crush_rule_set_step(rule, step++, CRUSH_RULE_SET_CHOOSE_TRIES, 100, 0);
+ // always choose osd.0
+ crush_rule_set_step(rule, step++, CRUSH_RULE_TAKE, 0, 0);
+ crush_rule_set_step(rule, step++, CRUSH_RULE_EMIT, 0, 0);
+ // then pick any other random osds
+ crush_rule_set_step(rule, step++, CRUSH_RULE_TAKE, host_id, 0);
+ crush_rule_set_step(rule, step++, CRUSH_RULE_CHOOSELEAF_FIRSTN, 2, 0);
+ crush_rule_set_step(rule, step++, CRUSH_RULE_EMIT, 0, 0);
+ ASSERT_TRUE(step == steps);
+ auto r = crush_add_rule(crush.get_crush_map(), rule, rno);
+ ASSERT_TRUE(r >= 0);
+ crush.set_rule_name(rno, rule_name);
+ {
+ OSDMap::Incremental pending_inc(osdmap.get_epoch() + 1);
+ pending_inc.crush.clear();
+ crush.encode(pending_inc.crush, CEPH_FEATURES_SUPPORTED_DEFAULT);
+ osdmap.apply_incremental(pending_inc);
+ }
+
+ // create "pool1"
+ OSDMap::Incremental pending_inc(osdmap.get_epoch() + 1);
+ pending_inc.new_pool_max = osdmap.get_pool_max();
+ auto pool_id = ++pending_inc.new_pool_max;
+ pool_1_id = pool_id;
+ pg_pool_t empty;
+ auto p = pending_inc.get_new_pool(pool_id, &empty);
+ p->size = 3;
+ p->min_size = 1;
+ p->set_pg_num(3);
+ p->set_pgp_num(3);
+ p->type = pg_pool_t::TYPE_REPLICATED;
+ p->crush_rule = rno;
+ p->set_flag(pg_pool_t::FLAG_HASHPSPOOL);
+ pending_inc.new_pool_names[pool_id] = pool_1;
+ osdmap.apply_incremental(pending_inc);
+ ASSERT_TRUE(osdmap.have_pg_pool(pool_id));
+ ASSERT_TRUE(osdmap.get_pool_name(pool_id) == pool_1);
+ {
+ for (unsigned i = 0; i < 3; i++) {
+ // 1.x -> [1]
+ pg_t rawpg(i, pool_id);
+ pg_t pgid = osdmap.raw_pg_to_pg(rawpg);
+ vector<int> up;
+ int up_primary;
+ osdmap.pg_to_raw_up(pgid, &up, &up_primary);
+ ASSERT_TRUE(up.size() == 3);
+ ASSERT_TRUE(up[0] == 0);
+
+ // insert a new pg_upmap
+ vector<int32_t> new_up;
+ // and remap 1.x to osd.1 only
+ // this way osd.0 is deemed to be *underfull*
+ // and osd.1 is deemed to be *overfull*
+ new_up.push_back(1);
+ {
+ OSDMap::Incremental pending_inc(osdmap.get_epoch() + 1);
+ pending_inc.new_pg_upmap[pgid] = mempool::osdmap::vector<int32_t>(
+ new_up.begin(), new_up.end());
+ osdmap.apply_incremental(pending_inc);
+ }
+ osdmap.pg_to_raw_up(pgid, &up, &up_primary);
+ ASSERT_TRUE(up.size() == 1);
+ ASSERT_TRUE(up[0] == 1);
+ }
+ }
+ }
+
+ {
+ // build customized crush rule for "pool2"
+ string host_name = "host_for_pool_2";
+ // build a customized host to capture osd.6~11
+ for (int i = 6; i < (int)get_num_osds(); i++) {
+ stringstream osd_name;
+ vector<string> move_to;
+ osd_name << "osd." << i;
+ move_to.push_back("root=default");
+ string host_loc = "host=" + host_name;
+ move_to.push_back(host_loc);
+ auto r = crush_move(osdmap, osd_name.str(), move_to);
+ ASSERT_EQ(0, r);
+ }
+ CrushWrapper crush;
+ get_crush(osdmap, crush);
+ auto host_id = crush.get_item_id(host_name);
+ ASSERT_TRUE(host_id < 0);
+ string rule_name = "rule_for_pool2";
+ int rule_type = pg_pool_t::TYPE_REPLICATED;
+ ASSERT_TRUE(!crush.rule_exists(rule_name));
+ int rno;
+ for (rno = 0; rno < crush.get_max_rules(); rno++) {
+ if (!crush.rule_exists(rno) && !crush.ruleset_exists(rno))
+ break;
+ }
+ int min_size = 3;
+ int max_size = 3;
+ int steps = 7;
+ crush_rule *rule = crush_make_rule(steps, rno, rule_type, min_size, max_size);
+ int step = 0;
+ crush_rule_set_step(rule, step++, CRUSH_RULE_SET_CHOOSELEAF_TRIES, 5, 0);
+ crush_rule_set_step(rule, step++, CRUSH_RULE_SET_CHOOSE_TRIES, 100, 0);
+ // always choose osd.0
+ crush_rule_set_step(rule, step++, CRUSH_RULE_TAKE, 0, 0);
+ crush_rule_set_step(rule, step++, CRUSH_RULE_EMIT, 0, 0);
+ // then pick any other random osds
+ crush_rule_set_step(rule, step++, CRUSH_RULE_TAKE, host_id, 0);
+ crush_rule_set_step(rule, step++, CRUSH_RULE_CHOOSELEAF_FIRSTN, 2, 0);
+ crush_rule_set_step(rule, step++, CRUSH_RULE_EMIT, 0, 0);
+ ASSERT_TRUE(step == steps);
+ auto r = crush_add_rule(crush.get_crush_map(), rule, rno);
+ ASSERT_TRUE(r >= 0);
+ crush.set_rule_name(rno, rule_name);
+ {
+ OSDMap::Incremental pending_inc(osdmap.get_epoch() + 1);
+ pending_inc.crush.clear();
+ crush.encode(pending_inc.crush, CEPH_FEATURES_SUPPORTED_DEFAULT);
+ osdmap.apply_incremental(pending_inc);
+ }
+
+ // create "pool2"
+ OSDMap::Incremental pending_inc(osdmap.get_epoch() + 1);
+ pending_inc.new_pool_max = osdmap.get_pool_max();
+ auto pool_id = ++pending_inc.new_pool_max;
+ pg_pool_t empty;
+ auto p = pending_inc.get_new_pool(pool_id, &empty);
+ p->size = 3;
+ // include a single PG
+ p->set_pg_num(1);
+ p->set_pgp_num(1);
+ p->type = pg_pool_t::TYPE_REPLICATED;
+ p->crush_rule = rno;
+ p->set_flag(pg_pool_t::FLAG_HASHPSPOOL);
+ pending_inc.new_pool_names[pool_id] = pool_2;
+ osdmap.apply_incremental(pending_inc);
+ ASSERT_TRUE(osdmap.have_pg_pool(pool_id));
+ ASSERT_TRUE(osdmap.get_pool_name(pool_id) == pool_2);
+ pg_t rawpg(0, pool_id);
+ pg_t pgid = osdmap.raw_pg_to_pg(rawpg);
+ EXPECT_TRUE(!osdmap.have_pg_upmaps(pgid));
+ vector<int> up;
+ int up_primary;
+ osdmap.pg_to_raw_up(pgid, &up, &up_primary);
+ ASSERT_TRUE(up.size() == 3);
+ ASSERT_TRUE(up[0] == 0);
+
+ {
+ // build a pg_upmap_item that will
+ // remap pg out from *underfull* osd.0
+ vector<pair<int32_t,int32_t>> new_pg_upmap_items;
+ new_pg_upmap_items.push_back(make_pair(0, 10)); // osd.0 -> osd.10
+ OSDMap::Incremental pending_inc(osdmap.get_epoch() + 1);
+ pending_inc.new_pg_upmap_items[pgid] =
+ mempool::osdmap::vector<pair<int32_t,int32_t>>(
+ new_pg_upmap_items.begin(), new_pg_upmap_items.end());
+ osdmap.apply_incremental(pending_inc);
+ ASSERT_TRUE(osdmap.have_pg_upmaps(pgid));
+ vector<int> up;
+ int up_primary;
+ osdmap.pg_to_raw_up(pgid, &up, &up_primary);
+ ASSERT_TRUE(up.size() == 3);
+ ASSERT_TRUE(up[0] == 10);
+ }
+ }
+
+ // ready to go
+ {
+ // require perfect distribution!
+ auto ret = g_ceph_context->_conf->set_val(
+ "osd_calc_pg_upmaps_max_stddev", "0");
+ ASSERT_EQ(0, ret);
+ g_ceph_context->_conf->apply_changes(nullptr);
+ set<int64_t> only_pools;
+ ASSERT_TRUE(pool_1_id >= 0);
+ only_pools.insert(pool_1_id);
+ OSDMap::Incremental pending_inc(osdmap.get_epoch() + 1);
+ osdmap.calc_pg_upmaps(g_ceph_context,
+ 0, // so we can force optimizing
+ 100,
+ only_pools,
+ &pending_inc);
+ osdmap.apply_incremental(pending_inc);
+ }
+}
+
TEST(PGTempMap, basic)
{
PGTempMap m;
projects / ceph.git / blobdiff