]>
Commit | Line | Data |
---|---|---|
522d829b TL |
1 | # python unit test |
2 | import unittest | |
3 | from tests import mock | |
4 | import pytest | |
5 | import json | |
6 | from pg_autoscaler import module | |
7 | ||
8 | ||
9 | class OSDMAP: | |
10 | def __init__(self, pools): | |
11 | self.pools = pools | |
12 | ||
13 | def get_pools(self): | |
14 | return self.pools | |
15 | ||
16 | def pool_raw_used_rate(pool_id): | |
17 | return 1 | |
18 | ||
19 | ||
20 | class CRUSH: | |
21 | def __init__(self, rules, osd_dic): | |
22 | self.rules = rules | |
23 | self.osd_dic = osd_dic | |
24 | ||
25 | def get_rule_by_id(self, rule_id): | |
26 | for rule in self.rules: | |
27 | if rule['rule_id'] == rule_id: | |
28 | return rule | |
29 | ||
30 | return None | |
31 | ||
32 | def get_rule_root(self, rule_name): | |
33 | for rule in self.rules: | |
34 | if rule['rule_name'] == rule_name: | |
35 | return rule['root_id'] | |
36 | ||
37 | return None | |
38 | ||
39 | def get_osds_under(self, root_id): | |
40 | return self.osd_dic[root_id] | |
41 | ||
42 | ||
43 | class TestPgAutoscaler(object): | |
44 | ||
39ae355f | 45 | def setup_method(self): |
522d829b TL |
46 | # a bunch of attributes for testing. |
47 | self.autoscaler = module.PgAutoscaler('module_name', 0, 0) | |
48 | ||
49 | def helper_test(self, osd_dic, rules, pools, expected_overlapped_roots): | |
50 | result = {} | |
51 | roots = [] | |
52 | overlapped_roots = set() | |
53 | osdmap = OSDMAP(pools) | |
54 | crush = CRUSH(rules, osd_dic) | |
55 | roots, overlapped_roots = self.autoscaler.identify_subtrees_and_overlaps(osdmap, | |
56 | crush, result, overlapped_roots, roots) | |
57 | assert overlapped_roots == expected_overlapped_roots | |
58 | ||
59 | def test_subtrees_and_overlaps(self): | |
60 | osd_dic = { | |
61 | -1: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], | |
62 | -40: [11, 12, 13, 14, 15], | |
63 | -5: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], | |
64 | } | |
65 | ||
66 | rules = [ | |
67 | { | |
68 | "rule_id": 0, | |
69 | "rule_name": "data", | |
70 | "ruleset": 0, | |
71 | "type": 1, | |
72 | "min_size": 1, | |
73 | "max_size": 10, | |
74 | "root_id": -1, | |
75 | }, | |
76 | { | |
77 | "rule_id": 1, | |
78 | "rule_name": "teuthology-data-ec", | |
79 | "ruleset": 1, | |
80 | "type": 3, | |
81 | "min_size": 3, | |
82 | "max_size": 6, | |
83 | "root_id": -5, | |
84 | }, | |
85 | { | |
86 | "rule_id": 4, | |
87 | "rule_name": "rep-ssd", | |
88 | "ruleset": 4, | |
89 | "type": 1, | |
90 | "min_size": 1, | |
91 | "max_size": 10, | |
92 | "root_id": -40, | |
93 | }, | |
94 | ] | |
95 | pools = { | |
96 | 0: { | |
97 | "pool_name": "data", | |
98 | "pg_num_target": 1024, | |
99 | "size": 3, | |
100 | "crush_rule": 0, | |
101 | "capacity_ratio": 0.1624, | |
102 | "options": { | |
103 | "pg_num_min": 1024, | |
104 | }, | |
105 | "expected_final_pg_target": 1024, | |
106 | }, | |
107 | 1: { | |
108 | "pool_name": "metadata", | |
109 | "pg_num_target": 64, | |
110 | "size": 3, | |
111 | "crush_rule": 4, | |
112 | "capacity_ratio": 0.0144, | |
113 | "options": { | |
114 | "pg_num_min": 64, | |
115 | }, | |
116 | "expected_final_pg_target": 64, | |
117 | }, | |
118 | 4: { | |
119 | "pool_name": "libvirt-pool", | |
120 | "pg_num_target": 32, | |
121 | "size": 3, | |
122 | "crush_rule": 0, | |
123 | "capacity_ratio": 0.0001, | |
124 | "options": {}, | |
125 | "expected_final_pg_target": 128, | |
126 | }, | |
127 | 93: { | |
128 | "pool_name": ".rgw.root", | |
129 | "pg_num_target": 32, | |
130 | "size": 3, | |
131 | "crush_rule": 4, | |
132 | "capacity_ratio": 0, | |
133 | "options": {}, | |
134 | "expected_final_pg_target": 32, | |
135 | }, | |
136 | 94: { | |
137 | "pool_name": "default.rgw.control", | |
138 | "pg_num_target": 32, | |
139 | "size": 3, | |
140 | "crush_rule": 4, | |
141 | "capacity_ratio": 0, | |
142 | "options": {}, | |
143 | "expected_final_pg_target": 32, | |
144 | }, | |
145 | 95: { | |
146 | "pool_name": "default.rgw.meta", | |
147 | "pg_num_target": 32, | |
148 | "size": 3, | |
149 | "crush_rule": 4, | |
150 | "capacity_ratio": 0, | |
151 | "options": {}, | |
152 | "expected_final_pg_target": 32, | |
153 | }, | |
154 | 96: { | |
155 | "pool_name": "default.rgw.log", | |
156 | "pg_num_target": 32, | |
157 | "size": 3, | |
158 | "crush_rule": 4, | |
159 | "capacity_ratio": 0, | |
160 | "options": {}, | |
161 | "expected_final_pg_target": 32, | |
162 | }, | |
163 | 97: { | |
164 | "pool_name": "default.rgw.buckets.index", | |
165 | "pg_num_target": 32, | |
166 | "size": 3, | |
167 | "crush_rule": 4, | |
168 | "capacity_ratio": 0.0002, | |
169 | "options": {}, | |
170 | "expected_final_pg_target": 32, | |
171 | }, | |
172 | 98: { | |
173 | "pool_name": "default.rgw.buckets.data", | |
174 | "pg_num_target": 32, | |
175 | "size": 3, | |
176 | "crush_rule": 0, | |
177 | "capacity_ratio": 0.0457, | |
178 | "options": {}, | |
179 | "expected_final_pg_target": 128, | |
180 | }, | |
181 | 99: { | |
182 | "pool_name": "default.rgw.buckets.non-ec", | |
183 | "pg_num_target": 32, | |
184 | "size": 3, | |
185 | "crush_rule": 0, | |
186 | "capacity_ratio": 0, | |
187 | "options": {}, | |
188 | "expected_final_pg_target": 32, | |
189 | }, | |
190 | 100: { | |
191 | "pool_name": "device_health_metrics", | |
192 | "pg_num_target": 1, | |
193 | "size": 3, | |
194 | "crush_rule": 0, | |
195 | "capacity_ratio": 0, | |
196 | "options": { | |
197 | "pg_num_min": 1 | |
198 | }, | |
199 | "expected_final_pg_target": 1, | |
200 | }, | |
201 | 113: { | |
202 | "pool_name": "cephfs.teuthology.meta", | |
203 | "pg_num_target": 64, | |
204 | "size": 3, | |
205 | "crush_rule": 4, | |
206 | "capacity_ratio": 0.1389, | |
207 | "options": { | |
208 | "pg_autoscale_bias": 4, | |
209 | "pg_num_min": 64, | |
210 | }, | |
211 | "expected_final_pg_target": 512, | |
212 | }, | |
213 | 114: { | |
214 | "pool_name": "cephfs.teuthology.data", | |
215 | "pg_num_target": 256, | |
216 | "size": 3, | |
217 | "crush_rule": 0, | |
218 | "capacity_ratio": 0.0006, | |
219 | "options": { | |
220 | "pg_num_min": 128, | |
221 | }, | |
222 | "expected_final_pg_target": 1024, | |
223 | "expected_final_pg_target": 256, | |
224 | }, | |
225 | 117: { | |
226 | "pool_name": "cephfs.scratch.meta", | |
227 | "pg_num_target": 32, | |
228 | "size": 3, | |
229 | "crush_rule": 4, | |
230 | "capacity_ratio": 0.0027, | |
231 | "options": { | |
232 | "pg_autoscale_bias": 4, | |
233 | "pg_num_min": 16, | |
234 | }, | |
235 | "expected_final_pg_target": 64, | |
236 | }, | |
237 | 118: { | |
238 | "pool_name": "cephfs.scratch.data", | |
239 | "pg_num_target": 32, | |
240 | "size": 3, | |
241 | "crush_rule": 0, | |
242 | "capacity_ratio": 0.0027, | |
243 | "options": {}, | |
244 | "expected_final_pg_target": 128, | |
245 | }, | |
246 | 119: { | |
247 | "pool_name": "cephfs.teuthology.data-ec", | |
248 | "pg_num_target": 1024, | |
249 | "size": 6, | |
250 | "crush_rule": 1, | |
251 | "capacity_ratio": 0.8490, | |
252 | "options": { | |
253 | "pg_num_min": 1024 | |
254 | }, | |
255 | "expected_final_pg_target": 1024, | |
256 | }, | |
257 | 121: { | |
258 | "pool_name": "cephsqlite", | |
259 | "pg_num_target": 32, | |
260 | "size": 3, | |
261 | "crush_rule": 4, | |
262 | "capacity_ratio": 0, | |
263 | "options": {}, | |
264 | "expected_final_pg_target": 128, | |
265 | }, | |
266 | } | |
267 | expected_overlapped_roots = {-40, -1, -5} | |
268 | self.helper_test(osd_dic, rules, pools, expected_overlapped_roots) | |
269 | ||
270 | def test_no_overlaps(self): | |
271 | osd_dic = { | |
272 | -1: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], | |
273 | -40: [11, 12, 13, 14, 15], | |
274 | -5: [16, 17, 18], | |
275 | } | |
276 | ||
277 | rules = [ | |
278 | { | |
279 | "rule_id": 0, | |
280 | "rule_name": "data", | |
281 | "ruleset": 0, | |
282 | "type": 1, | |
283 | "min_size": 1, | |
284 | "max_size": 10, | |
285 | "root_id": -1, | |
286 | }, | |
287 | { | |
288 | "rule_id": 1, | |
289 | "rule_name": "teuthology-data-ec", | |
290 | "ruleset": 1, | |
291 | "type": 3, | |
292 | "min_size": 3, | |
293 | "max_size": 6, | |
294 | "root_id": -5, | |
295 | }, | |
296 | { | |
297 | "rule_id": 4, | |
298 | "rule_name": "rep-ssd", | |
299 | "ruleset": 4, | |
300 | "type": 1, | |
301 | "min_size": 1, | |
302 | "max_size": 10, | |
303 | "root_id": -40, | |
304 | }, | |
305 | ] | |
306 | pools = { | |
307 | 0: { | |
308 | "pool_name": "data", | |
309 | "pg_num_target": 1024, | |
310 | "size": 3, | |
311 | "crush_rule": 0, | |
312 | "capacity_ratio": 0.1624, | |
313 | "options": { | |
314 | "pg_num_min": 1024, | |
315 | }, | |
316 | "expected_final_pg_target": 1024, | |
317 | }, | |
318 | 1: { | |
319 | "pool_name": "metadata", | |
320 | "pg_num_target": 64, | |
321 | "size": 3, | |
322 | "crush_rule": 4, | |
323 | "capacity_ratio": 0.0144, | |
324 | "options": { | |
325 | "pg_num_min": 64, | |
326 | }, | |
327 | "expected_final_pg_target": 64, | |
328 | }, | |
329 | 4: { | |
330 | "pool_name": "libvirt-pool", | |
331 | "pg_num_target": 32, | |
332 | "size": 3, | |
333 | "crush_rule": 0, | |
334 | "capacity_ratio": 0.0001, | |
335 | "options": {}, | |
336 | "expected_final_pg_target": 128, | |
337 | }, | |
338 | 93: { | |
339 | "pool_name": ".rgw.root", | |
340 | "pg_num_target": 32, | |
341 | "size": 3, | |
342 | "crush_rule": 4, | |
343 | "capacity_ratio": 0, | |
344 | "options": {}, | |
345 | "expected_final_pg_target": 32, | |
346 | }, | |
347 | 94: { | |
348 | "pool_name": "default.rgw.control", | |
349 | "pg_num_target": 32, | |
350 | "size": 3, | |
351 | "crush_rule": 4, | |
352 | "capacity_ratio": 0, | |
353 | "options": {}, | |
354 | "expected_final_pg_target": 32, | |
355 | }, | |
356 | 95: { | |
357 | "pool_name": "default.rgw.meta", | |
358 | "pg_num_target": 32, | |
359 | "size": 3, | |
360 | "crush_rule": 4, | |
361 | "capacity_ratio": 0, | |
362 | "options": {}, | |
363 | "expected_final_pg_target": 32, | |
364 | }, | |
365 | 96: { | |
366 | "pool_name": "default.rgw.log", | |
367 | "pg_num_target": 32, | |
368 | "size": 3, | |
369 | "crush_rule": 4, | |
370 | "capacity_ratio": 0, | |
371 | "options": {}, | |
372 | "expected_final_pg_target": 32, | |
373 | }, | |
374 | 97: { | |
375 | "pool_name": "default.rgw.buckets.index", | |
376 | "pg_num_target": 32, | |
377 | "size": 3, | |
378 | "crush_rule": 4, | |
379 | "capacity_ratio": 0.0002, | |
380 | "options": {}, | |
381 | "expected_final_pg_target": 32, | |
382 | }, | |
383 | 98: { | |
384 | "pool_name": "default.rgw.buckets.data", | |
385 | "pg_num_target": 32, | |
386 | "size": 3, | |
387 | "crush_rule": 0, | |
388 | "capacity_ratio": 0.0457, | |
389 | "options": {}, | |
390 | "expected_final_pg_target": 128, | |
391 | }, | |
392 | 99: { | |
393 | "pool_name": "default.rgw.buckets.non-ec", | |
394 | "pg_num_target": 32, | |
395 | "size": 3, | |
396 | "crush_rule": 0, | |
397 | "capacity_ratio": 0, | |
398 | "options": {}, | |
399 | "expected_final_pg_target": 32, | |
400 | }, | |
401 | 100: { | |
402 | "pool_name": "device_health_metrics", | |
403 | "pg_num_target": 1, | |
404 | "size": 3, | |
405 | "crush_rule": 0, | |
406 | "capacity_ratio": 0, | |
407 | "options": { | |
408 | "pg_num_min": 1 | |
409 | }, | |
410 | "expected_final_pg_target": 1, | |
411 | }, | |
412 | 113: { | |
413 | "pool_name": "cephfs.teuthology.meta", | |
414 | "pg_num_target": 64, | |
415 | "size": 3, | |
416 | "crush_rule": 4, | |
417 | "capacity_ratio": 0.1389, | |
418 | "options": { | |
419 | "pg_autoscale_bias": 4, | |
420 | "pg_num_min": 64, | |
421 | }, | |
422 | "expected_final_pg_target": 512, | |
423 | }, | |
424 | 114: { | |
425 | "pool_name": "cephfs.teuthology.data", | |
426 | "pg_num_target": 256, | |
427 | "size": 3, | |
428 | "crush_rule": 0, | |
429 | "capacity_ratio": 0.0006, | |
430 | "options": { | |
431 | "pg_num_min": 128, | |
432 | }, | |
433 | "expected_final_pg_target": 1024, | |
434 | "expected_final_pg_target": 256, | |
435 | }, | |
436 | 117: { | |
437 | "pool_name": "cephfs.scratch.meta", | |
438 | "pg_num_target": 32, | |
439 | "size": 3, | |
440 | "crush_rule": 4, | |
441 | "capacity_ratio": 0.0027, | |
442 | "options": { | |
443 | "pg_autoscale_bias": 4, | |
444 | "pg_num_min": 16, | |
445 | }, | |
446 | "expected_final_pg_target": 64, | |
447 | }, | |
448 | 118: { | |
449 | "pool_name": "cephfs.scratch.data", | |
450 | "pg_num_target": 32, | |
451 | "size": 3, | |
452 | "crush_rule": 0, | |
453 | "capacity_ratio": 0.0027, | |
454 | "options": {}, | |
455 | "expected_final_pg_target": 128, | |
456 | }, | |
457 | 119: { | |
458 | "pool_name": "cephfs.teuthology.data-ec", | |
459 | "pg_num_target": 1024, | |
460 | "size": 6, | |
461 | "crush_rule": 1, | |
462 | "capacity_ratio": 0.8490, | |
463 | "options": { | |
464 | "pg_num_min": 1024 | |
465 | }, | |
466 | "expected_final_pg_target": 1024, | |
467 | }, | |
468 | 121: { | |
469 | "pool_name": "cephsqlite", | |
470 | "pg_num_target": 32, | |
471 | "size": 3, | |
472 | "crush_rule": 4, | |
473 | "capacity_ratio": 0, | |
474 | "options": {}, | |
475 | "expected_final_pg_target": 128, | |
476 | }, | |
477 | } | |
478 | expected_overlapped_roots = set() | |
479 | self.helper_test(osd_dic, rules, pools, expected_overlapped_roots) |