]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | #include "gtest/gtest.h" | |
3 | #include "osd/OSDMap.h" | |
4 | #include "osd/OSDMapMapping.h" | |
5 | ||
6 | #include "global/global_context.h" | |
7 | #include "global/global_init.h" | |
8 | #include "common/common_init.h" | |
9 | ||
10 | #include <iostream> | |
11 | ||
12 | using namespace std; | |
13 | ||
14 | int main(int argc, char **argv) { | |
15 | std::vector<const char*> args(argv, argv+argc); | |
16 | auto cct = global_init(nullptr, args, CEPH_ENTITY_TYPE_CLIENT, | |
17 | CODE_ENVIRONMENT_UTILITY, | |
18 | CINIT_FLAG_NO_DEFAULT_CONFIG_FILE); | |
19 | common_init_finish(g_ceph_context); | |
20 | // make sure we have 3 copies, or some tests won't work | |
21 | g_ceph_context->_conf->set_val("osd_pool_default_size", "3", false); | |
22 | // our map is flat, so just try and split across OSDs, not hosts or whatever | |
23 | g_ceph_context->_conf->set_val("osd_crush_chooseleaf_type", "0", false); | |
24 | ::testing::InitGoogleTest(&argc, argv); | |
25 | return RUN_ALL_TESTS(); | |
26 | } | |
27 | ||
28 | class OSDMapTest : public testing::Test { | |
29 | const static int num_osds = 6; | |
30 | public: | |
31 | OSDMap osdmap; | |
32 | OSDMapMapping mapping; | |
33 | ||
34 | OSDMapTest() {} | |
35 | ||
36 | void set_up_map() { | |
37 | uuid_d fsid; | |
38 | osdmap.build_simple(g_ceph_context, 0, fsid, num_osds, 6, 6); | |
39 | OSDMap::Incremental pending_inc(osdmap.get_epoch() + 1); | |
40 | pending_inc.fsid = osdmap.get_fsid(); | |
41 | entity_addr_t sample_addr; | |
42 | uuid_d sample_uuid; | |
43 | for (int i = 0; i < num_osds; ++i) { | |
44 | sample_uuid.generate_random(); | |
45 | sample_addr.nonce = i; | |
46 | pending_inc.new_state[i] = CEPH_OSD_EXISTS | CEPH_OSD_NEW; | |
47 | pending_inc.new_up_client[i] = sample_addr; | |
48 | pending_inc.new_up_cluster[i] = sample_addr; | |
49 | pending_inc.new_hb_back_up[i] = sample_addr; | |
50 | pending_inc.new_hb_front_up[i] = sample_addr; | |
51 | pending_inc.new_weight[i] = CEPH_OSD_IN; | |
52 | pending_inc.new_uuid[i] = sample_uuid; | |
53 | } | |
54 | osdmap.apply_incremental(pending_inc); | |
55 | ||
56 | // Create an EC ruleset and a pool using it | |
57 | int r = osdmap.crush->add_simple_ruleset("erasure", "default", "osd", | |
58 | "indep", pg_pool_t::TYPE_ERASURE, | |
59 | &cerr); | |
60 | ||
61 | OSDMap::Incremental new_pool_inc(osdmap.get_epoch() + 1); | |
62 | new_pool_inc.new_pool_max = osdmap.get_pool_max(); | |
63 | new_pool_inc.fsid = osdmap.get_fsid(); | |
64 | pg_pool_t empty; | |
65 | uint64_t pool_id = ++new_pool_inc.new_pool_max; | |
66 | pg_pool_t *p = new_pool_inc.get_new_pool(pool_id, &empty); | |
67 | p->size = 3; | |
68 | p->set_pg_num(64); | |
69 | p->set_pgp_num(64); | |
70 | p->type = pg_pool_t::TYPE_ERASURE; | |
71 | p->crush_ruleset = r; | |
72 | new_pool_inc.new_pool_names[pool_id] = "ec"; | |
73 | osdmap.apply_incremental(new_pool_inc); | |
74 | } | |
75 | unsigned int get_num_osds() { return num_osds; } | |
76 | ||
77 | void test_mappings(int pool, | |
78 | int num, | |
79 | vector<int> *any, | |
80 | vector<int> *first, | |
81 | vector<int> *primary) { | |
82 | mapping.update(osdmap); | |
83 | for (int i=0; i<num; ++i) { | |
84 | vector<int> up, acting; | |
85 | int up_primary, acting_primary; | |
86 | pg_t pgid(i, pool); | |
87 | osdmap.pg_to_up_acting_osds(pgid, | |
88 | &up, &up_primary, &acting, &acting_primary); | |
89 | for (unsigned j=0; j<acting.size(); ++j) | |
90 | (*any)[acting[j]]++; | |
91 | if (!acting.empty()) | |
92 | (*first)[acting[0]]++; | |
93 | if (acting_primary >= 0) | |
94 | (*primary)[acting_primary]++; | |
95 | ||
96 | // compare to precalc mapping | |
97 | vector<int> up2, acting2; | |
98 | int up_primary2, acting_primary2; | |
99 | pgid = osdmap.raw_pg_to_pg(pgid); | |
100 | mapping.get(pgid, &up2, &up_primary2, &acting2, &acting_primary2); | |
101 | ASSERT_EQ(up, up2); | |
102 | ASSERT_EQ(up_primary, up_primary2); | |
103 | ASSERT_EQ(acting, acting2); | |
104 | ASSERT_EQ(acting_primary, acting_primary2); | |
105 | } | |
106 | } | |
107 | }; | |
108 | ||
109 | TEST_F(OSDMapTest, Create) { | |
110 | set_up_map(); | |
111 | ASSERT_EQ(get_num_osds(), (unsigned)osdmap.get_max_osd()); | |
112 | ASSERT_EQ(get_num_osds(), osdmap.get_num_in_osds()); | |
113 | } | |
114 | ||
115 | TEST_F(OSDMapTest, Features) { | |
116 | // with EC pool | |
117 | set_up_map(); | |
118 | uint64_t features = osdmap.get_features(CEPH_ENTITY_TYPE_OSD, NULL); | |
119 | ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES); | |
120 | ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES2); | |
121 | ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES3); | |
122 | ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_V2); | |
123 | ASSERT_TRUE(features & CEPH_FEATURE_OSD_ERASURE_CODES); | |
124 | ASSERT_TRUE(features & CEPH_FEATURE_OSDHASHPSPOOL); | |
125 | ASSERT_TRUE(features & CEPH_FEATURE_OSD_PRIMARY_AFFINITY); | |
126 | ||
127 | // clients have a slightly different view | |
128 | features = osdmap.get_features(CEPH_ENTITY_TYPE_CLIENT, NULL); | |
129 | ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES); | |
130 | ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES2); | |
131 | ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES3); | |
132 | ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_V2); | |
133 | ASSERT_FALSE(features & CEPH_FEATURE_OSD_ERASURE_CODES); // dont' need this | |
134 | ASSERT_TRUE(features & CEPH_FEATURE_OSDHASHPSPOOL); | |
135 | ASSERT_TRUE(features & CEPH_FEATURE_OSD_PRIMARY_AFFINITY); | |
136 | ||
137 | // remove teh EC pool, but leave the rule. add primary affinity. | |
138 | { | |
139 | OSDMap::Incremental new_pool_inc(osdmap.get_epoch() + 1); | |
140 | new_pool_inc.old_pools.insert(osdmap.lookup_pg_pool_name("ec")); | |
141 | new_pool_inc.new_primary_affinity[0] = 0x8000; | |
142 | osdmap.apply_incremental(new_pool_inc); | |
143 | } | |
144 | ||
145 | features = osdmap.get_features(CEPH_ENTITY_TYPE_MON, NULL); | |
146 | ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES); | |
147 | ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES2); | |
148 | ASSERT_TRUE(features & CEPH_FEATURE_CRUSH_TUNABLES3); // shared bit with primary affinity | |
149 | ASSERT_FALSE(features & CEPH_FEATURE_CRUSH_V2); | |
150 | ASSERT_FALSE(features & CEPH_FEATURE_OSD_ERASURE_CODES); | |
151 | ASSERT_TRUE(features & CEPH_FEATURE_OSDHASHPSPOOL); | |
152 | ASSERT_TRUE(features & CEPH_FEATURE_OSD_PRIMARY_AFFINITY); | |
153 | ||
154 | // FIXME: test tiering feature bits | |
155 | } | |
156 | ||
157 | TEST_F(OSDMapTest, MapPG) { | |
158 | set_up_map(); | |
159 | ||
160 | pg_t rawpg(0, 0, -1); | |
161 | pg_t pgid = osdmap.raw_pg_to_pg(rawpg); | |
162 | vector<int> up_osds, acting_osds; | |
163 | int up_primary, acting_primary; | |
164 | ||
165 | osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary, | |
166 | &acting_osds, &acting_primary); | |
167 | ||
168 | vector<int> old_up_osds, old_acting_osds; | |
169 | osdmap.pg_to_up_acting_osds(pgid, old_up_osds, old_acting_osds); | |
170 | ASSERT_EQ(old_up_osds, up_osds); | |
171 | ASSERT_EQ(old_acting_osds, acting_osds); | |
172 | ||
173 | ASSERT_EQ(osdmap.get_pg_pool(0)->get_size(), up_osds.size()); | |
174 | } | |
175 | ||
176 | TEST_F(OSDMapTest, MapFunctionsMatch) { | |
177 | // TODO: make sure pg_to_up_acting_osds and pg_to_acting_osds match | |
178 | set_up_map(); | |
179 | ||
180 | pg_t rawpg(0, 0, -1); | |
181 | pg_t pgid = osdmap.raw_pg_to_pg(rawpg); | |
182 | vector<int> up_osds, acting_osds; | |
183 | int up_primary, acting_primary; | |
184 | ||
185 | osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary, | |
186 | &acting_osds, &acting_primary); | |
187 | ||
188 | vector<int> up_osds_two, acting_osds_two; | |
189 | ||
190 | osdmap.pg_to_up_acting_osds(pgid, up_osds_two, acting_osds_two); | |
191 | ||
192 | ASSERT_EQ(up_osds, up_osds_two); | |
193 | ASSERT_EQ(acting_osds, acting_osds_two); | |
194 | ||
195 | int acting_primary_two; | |
196 | osdmap.pg_to_acting_osds(pgid, &acting_osds_two, &acting_primary_two); | |
197 | EXPECT_EQ(acting_osds, acting_osds_two); | |
198 | EXPECT_EQ(acting_primary, acting_primary_two); | |
199 | osdmap.pg_to_acting_osds(pgid, acting_osds_two); | |
200 | EXPECT_EQ(acting_osds, acting_osds_two); | |
201 | } | |
202 | ||
203 | /** This test must be removed or modified appropriately when we allow | |
204 | * other ways to specify a primary. */ | |
205 | TEST_F(OSDMapTest, PrimaryIsFirst) { | |
206 | set_up_map(); | |
207 | ||
208 | pg_t rawpg(0, 0, -1); | |
209 | pg_t pgid = osdmap.raw_pg_to_pg(rawpg); | |
210 | vector<int> up_osds, acting_osds; | |
211 | int up_primary, acting_primary; | |
212 | ||
213 | osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary, | |
214 | &acting_osds, &acting_primary); | |
215 | EXPECT_EQ(up_osds[0], up_primary); | |
216 | EXPECT_EQ(acting_osds[0], acting_primary); | |
217 | } | |
218 | ||
219 | TEST_F(OSDMapTest, PGTempRespected) { | |
220 | set_up_map(); | |
221 | ||
222 | pg_t rawpg(0, 0, -1); | |
223 | pg_t pgid = osdmap.raw_pg_to_pg(rawpg); | |
224 | vector<int> up_osds, acting_osds; | |
225 | int up_primary, acting_primary; | |
226 | ||
227 | osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary, | |
228 | &acting_osds, &acting_primary); | |
229 | ||
230 | // copy and swap first and last element in acting_osds | |
231 | vector<int> new_acting_osds(acting_osds); | |
232 | int first = new_acting_osds[0]; | |
233 | new_acting_osds[0] = *new_acting_osds.rbegin(); | |
234 | *new_acting_osds.rbegin() = first; | |
235 | ||
236 | // apply pg_temp to osdmap | |
237 | OSDMap::Incremental pgtemp_map(osdmap.get_epoch() + 1); | |
238 | pgtemp_map.new_pg_temp[pgid] = mempool::osdmap::vector<int>( | |
239 | new_acting_osds.begin(), new_acting_osds.end()); | |
240 | osdmap.apply_incremental(pgtemp_map); | |
241 | ||
242 | osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary, | |
243 | &acting_osds, &acting_primary); | |
244 | EXPECT_EQ(new_acting_osds, acting_osds); | |
245 | } | |
246 | ||
247 | TEST_F(OSDMapTest, PrimaryTempRespected) { | |
248 | set_up_map(); | |
249 | ||
250 | pg_t rawpg(0, 0, -1); | |
251 | pg_t pgid = osdmap.raw_pg_to_pg(rawpg); | |
252 | vector<int> up_osds; | |
253 | vector<int> acting_osds; | |
254 | int up_primary, acting_primary; | |
255 | ||
256 | osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary, | |
257 | &acting_osds, &acting_primary); | |
258 | ||
259 | // make second OSD primary via incremental | |
260 | OSDMap::Incremental pgtemp_map(osdmap.get_epoch() + 1); | |
261 | pgtemp_map.new_primary_temp[pgid] = acting_osds[1]; | |
262 | osdmap.apply_incremental(pgtemp_map); | |
263 | ||
264 | osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary, | |
265 | &acting_osds, &acting_primary); | |
266 | EXPECT_EQ(acting_primary, acting_osds[1]); | |
267 | } | |
268 | ||
269 | TEST_F(OSDMapTest, CleanTemps) { | |
270 | set_up_map(); | |
271 | ||
272 | OSDMap::Incremental pgtemp_map(osdmap.get_epoch() + 1); | |
273 | OSDMap::Incremental pending_inc(osdmap.get_epoch() + 2); | |
274 | pg_t pga = osdmap.raw_pg_to_pg(pg_t(0, 0)); | |
275 | { | |
276 | vector<int> up_osds, acting_osds; | |
277 | int up_primary, acting_primary; | |
278 | osdmap.pg_to_up_acting_osds(pga, &up_osds, &up_primary, | |
279 | &acting_osds, &acting_primary); | |
280 | pgtemp_map.new_pg_temp[pga] = mempool::osdmap::vector<int>( | |
281 | up_osds.begin(), up_osds.end()); | |
282 | pgtemp_map.new_primary_temp[pga] = up_primary; | |
283 | } | |
284 | pg_t pgb = osdmap.raw_pg_to_pg(pg_t(1, 0)); | |
285 | { | |
286 | vector<int> up_osds, acting_osds; | |
287 | int up_primary, acting_primary; | |
288 | osdmap.pg_to_up_acting_osds(pgb, &up_osds, &up_primary, | |
289 | &acting_osds, &acting_primary); | |
290 | pending_inc.new_pg_temp[pgb] = mempool::osdmap::vector<int>( | |
291 | up_osds.begin(), up_osds.end()); | |
292 | pending_inc.new_primary_temp[pgb] = up_primary; | |
293 | } | |
294 | ||
295 | osdmap.apply_incremental(pgtemp_map); | |
296 | ||
297 | OSDMap::clean_temps(g_ceph_context, osdmap, &pending_inc); | |
298 | ||
299 | EXPECT_TRUE(pending_inc.new_pg_temp.count(pga) && | |
300 | pending_inc.new_pg_temp[pga].size() == 0); | |
301 | EXPECT_EQ(-1, pending_inc.new_primary_temp[pga]); | |
302 | ||
303 | EXPECT_TRUE(!pending_inc.new_pg_temp.count(pgb) && | |
304 | !pending_inc.new_primary_temp.count(pgb)); | |
305 | } | |
306 | ||
307 | TEST_F(OSDMapTest, KeepsNecessaryTemps) { | |
308 | set_up_map(); | |
309 | ||
310 | pg_t rawpg(0, 0, -1); | |
311 | pg_t pgid = osdmap.raw_pg_to_pg(rawpg); | |
312 | vector<int> up_osds, acting_osds; | |
313 | int up_primary, acting_primary; | |
314 | ||
315 | osdmap.pg_to_up_acting_osds(pgid, &up_osds, &up_primary, | |
316 | &acting_osds, &acting_primary); | |
317 | ||
318 | // find unused OSD and stick it in there | |
319 | OSDMap::Incremental pgtemp_map(osdmap.get_epoch() + 1); | |
320 | // find an unused osd and put it in place of the first one | |
321 | int i = 0; | |
322 | for(; i != (int)get_num_osds(); ++i) { | |
323 | bool in_use = false; | |
324 | for (vector<int>::iterator osd_it = up_osds.begin(); | |
325 | osd_it != up_osds.end(); | |
326 | ++osd_it) { | |
327 | if (i == *osd_it) { | |
328 | in_use = true; | |
329 | break; | |
330 | } | |
331 | } | |
332 | if (!in_use) { | |
333 | up_osds[1] = i; | |
334 | break; | |
335 | } | |
336 | } | |
337 | if (i == (int)get_num_osds()) | |
338 | FAIL() << "did not find unused OSD for temp mapping"; | |
339 | ||
340 | pgtemp_map.new_pg_temp[pgid] = mempool::osdmap::vector<int>( | |
341 | up_osds.begin(), up_osds.end()); | |
342 | pgtemp_map.new_primary_temp[pgid] = up_osds[1]; | |
343 | osdmap.apply_incremental(pgtemp_map); | |
344 | ||
345 | OSDMap::Incremental pending_inc(osdmap.get_epoch() + 1); | |
346 | ||
347 | OSDMap::clean_temps(g_ceph_context, osdmap, &pending_inc); | |
348 | EXPECT_FALSE(pending_inc.new_pg_temp.count(pgid)); | |
349 | EXPECT_FALSE(pending_inc.new_primary_temp.count(pgid)); | |
350 | } | |
351 | ||
352 | TEST_F(OSDMapTest, PrimaryAffinity) { | |
353 | set_up_map(); | |
354 | ||
355 | /* | |
356 | osdmap.print(cout); | |
357 | Formatter *f = Formatter::create("json-pretty"); | |
358 | f->open_object_section("CRUSH"); | |
359 | osdmap.crush->dump(f); | |
360 | f->close_section(); | |
361 | f->flush(cout); | |
362 | delete f; | |
363 | */ | |
364 | ||
365 | int n = get_num_osds(); | |
366 | for (map<int64_t,pg_pool_t>::const_iterator p = osdmap.get_pools().begin(); | |
367 | p != osdmap.get_pools().end(); | |
368 | ++p) { | |
369 | int pool = p->first; | |
370 | cout << "pool " << pool << std::endl; | |
371 | { | |
372 | vector<int> any(n, 0); | |
373 | vector<int> first(n, 0); | |
374 | vector<int> primary(n, 0); | |
375 | test_mappings(0, 10000, &any, &first, &primary); | |
376 | for (int i=0; i<n; ++i) { | |
377 | //cout << "osd." << i << " " << any[i] << " " << first[i] << " " << primary[i] << std::endl; | |
378 | ASSERT_LT(0, any[i]); | |
379 | ASSERT_LT(0, first[i]); | |
380 | ASSERT_LT(0, primary[i]); | |
381 | } | |
382 | } | |
383 | ||
384 | osdmap.set_primary_affinity(0, 0); | |
385 | osdmap.set_primary_affinity(1, 0); | |
386 | { | |
387 | vector<int> any(n, 0); | |
388 | vector<int> first(n, 0); | |
389 | vector<int> primary(n, 0); | |
390 | test_mappings(pool, 10000, &any, &first, &primary); | |
391 | for (int i=0; i<n; ++i) { | |
392 | //cout << "osd." << i << " " << any[i] << " " << first[i] << " " << primary[i] << std::endl; | |
393 | ASSERT_LT(0, any[i]); | |
394 | if (i >= 2) { | |
395 | ASSERT_LT(0, first[i]); | |
396 | ASSERT_LT(0, primary[i]); | |
397 | } else { | |
398 | if (p->second.is_replicated()) { | |
399 | ASSERT_EQ(0, first[i]); | |
400 | } | |
401 | ASSERT_EQ(0, primary[i]); | |
402 | } | |
403 | } | |
404 | } | |
405 | ||
406 | osdmap.set_primary_affinity(0, 0x8000); | |
407 | osdmap.set_primary_affinity(1, 0); | |
408 | { | |
409 | vector<int> any(n, 0); | |
410 | vector<int> first(n, 0); | |
411 | vector<int> primary(n, 0); | |
412 | test_mappings(pool, 10000, &any, &first, &primary); | |
413 | for (int i=0; i<n; ++i) { | |
414 | //cout << "osd." << i << " " << any[i] << " " << first[i] << " " << primary[i] << std::endl; | |
415 | ASSERT_LT(0, any[i]); | |
416 | if (i >= 2) { | |
417 | ASSERT_LT(0, first[i]); | |
418 | ASSERT_LT(0, primary[i]); | |
419 | } else if (i == 1) { | |
420 | if (p->second.is_replicated()) { | |
421 | ASSERT_EQ(0, first[i]); | |
422 | } | |
423 | ASSERT_EQ(0, primary[i]); | |
424 | } else { | |
425 | ASSERT_LT(10000/6/4, primary[0]); | |
426 | ASSERT_GT(10000/6/4*3, primary[0]); | |
427 | } | |
428 | } | |
429 | } | |
430 | ||
431 | osdmap.set_primary_affinity(0, 0x10000); | |
432 | osdmap.set_primary_affinity(1, 0x10000); | |
433 | } | |
434 | } |