]>
Commit | Line | Data |
---|---|---|
522d829b TL |
1 | # python unit test |
2 | import unittest | |
3 | from tests import mock | |
4 | import pytest | |
5 | import json | |
6 | from pg_autoscaler import module | |
7 | ||
8 | ||
9 | class OSDMAP: | |
10 | def __init__(self, pools): | |
11 | self.pools = pools | |
12 | ||
13 | def get_pools(self): | |
14 | return self.pools | |
15 | ||
16 | def pool_raw_used_rate(pool_id): | |
17 | return 1 | |
18 | ||
19 | ||
20 | class CRUSH: | |
21 | def __init__(self, rules, osd_dic): | |
22 | self.rules = rules | |
23 | self.osd_dic = osd_dic | |
24 | ||
25 | def get_rule_by_id(self, rule_id): | |
26 | for rule in self.rules: | |
27 | if rule['rule_id'] == rule_id: | |
28 | return rule | |
29 | ||
30 | return None | |
31 | ||
32 | def get_rule_root(self, rule_name): | |
33 | for rule in self.rules: | |
34 | if rule['rule_name'] == rule_name: | |
35 | return rule['root_id'] | |
36 | ||
37 | return None | |
38 | ||
39 | def get_osds_under(self, root_id): | |
40 | return self.osd_dic[root_id] | |
41 | ||
42 | ||
43 | class TestPgAutoscaler(object): | |
44 | ||
39ae355f | 45 | def setup_method(self): |
522d829b TL |
46 | # a bunch of attributes for testing. |
47 | self.autoscaler = module.PgAutoscaler('module_name', 0, 0) | |
48 | ||
49 | def helper_test(self, osd_dic, rules, pools, expected_overlapped_roots): | |
50 | result = {} | |
51 | roots = [] | |
52 | overlapped_roots = set() | |
53 | osdmap = OSDMAP(pools) | |
54 | crush = CRUSH(rules, osd_dic) | |
aee94f69 TL |
55 | roots, overlapped_roots = self.autoscaler.identify_subtrees_and_overlaps( |
56 | osdmap, pools, crush, result, overlapped_roots, roots | |
57 | ) | |
522d829b TL |
58 | assert overlapped_roots == expected_overlapped_roots |
59 | ||
60 | def test_subtrees_and_overlaps(self): | |
61 | osd_dic = { | |
62 | -1: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], | |
63 | -40: [11, 12, 13, 14, 15], | |
64 | -5: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], | |
65 | } | |
66 | ||
67 | rules = [ | |
68 | { | |
69 | "rule_id": 0, | |
70 | "rule_name": "data", | |
71 | "ruleset": 0, | |
72 | "type": 1, | |
73 | "min_size": 1, | |
74 | "max_size": 10, | |
75 | "root_id": -1, | |
76 | }, | |
77 | { | |
78 | "rule_id": 1, | |
79 | "rule_name": "teuthology-data-ec", | |
80 | "ruleset": 1, | |
81 | "type": 3, | |
82 | "min_size": 3, | |
83 | "max_size": 6, | |
84 | "root_id": -5, | |
85 | }, | |
86 | { | |
87 | "rule_id": 4, | |
88 | "rule_name": "rep-ssd", | |
89 | "ruleset": 4, | |
90 | "type": 1, | |
91 | "min_size": 1, | |
92 | "max_size": 10, | |
93 | "root_id": -40, | |
94 | }, | |
95 | ] | |
96 | pools = { | |
aee94f69 TL |
97 | "data": { |
98 | "pool": 0, | |
522d829b TL |
99 | "pool_name": "data", |
100 | "pg_num_target": 1024, | |
101 | "size": 3, | |
102 | "crush_rule": 0, | |
103 | "capacity_ratio": 0.1624, | |
104 | "options": { | |
105 | "pg_num_min": 1024, | |
106 | }, | |
107 | "expected_final_pg_target": 1024, | |
108 | }, | |
aee94f69 TL |
109 | "metadata": { |
110 | "pool": 1, | |
522d829b TL |
111 | "pool_name": "metadata", |
112 | "pg_num_target": 64, | |
113 | "size": 3, | |
114 | "crush_rule": 4, | |
115 | "capacity_ratio": 0.0144, | |
116 | "options": { | |
117 | "pg_num_min": 64, | |
118 | }, | |
119 | "expected_final_pg_target": 64, | |
120 | }, | |
aee94f69 TL |
121 | "libvirt-pool": { |
122 | "pool": 4, | |
522d829b TL |
123 | "pool_name": "libvirt-pool", |
124 | "pg_num_target": 32, | |
125 | "size": 3, | |
126 | "crush_rule": 0, | |
127 | "capacity_ratio": 0.0001, | |
128 | "options": {}, | |
129 | "expected_final_pg_target": 128, | |
130 | }, | |
aee94f69 TL |
131 | ".rgw.root": { |
132 | "pool": 93, | |
522d829b TL |
133 | "pool_name": ".rgw.root", |
134 | "pg_num_target": 32, | |
135 | "size": 3, | |
136 | "crush_rule": 4, | |
137 | "capacity_ratio": 0, | |
138 | "options": {}, | |
139 | "expected_final_pg_target": 32, | |
140 | }, | |
aee94f69 TL |
141 | "default.rgw.control": { |
142 | "pool": 94, | |
522d829b TL |
143 | "pool_name": "default.rgw.control", |
144 | "pg_num_target": 32, | |
145 | "size": 3, | |
146 | "crush_rule": 4, | |
147 | "capacity_ratio": 0, | |
148 | "options": {}, | |
149 | "expected_final_pg_target": 32, | |
150 | }, | |
aee94f69 TL |
151 | "default.rgw.meta": { |
152 | "pool": 95, | |
522d829b TL |
153 | "pool_name": "default.rgw.meta", |
154 | "pg_num_target": 32, | |
155 | "size": 3, | |
156 | "crush_rule": 4, | |
157 | "capacity_ratio": 0, | |
158 | "options": {}, | |
159 | "expected_final_pg_target": 32, | |
160 | }, | |
aee94f69 TL |
161 | "default.rgw.log": { |
162 | "pool": 96, | |
522d829b TL |
163 | "pool_name": "default.rgw.log", |
164 | "pg_num_target": 32, | |
165 | "size": 3, | |
166 | "crush_rule": 4, | |
167 | "capacity_ratio": 0, | |
168 | "options": {}, | |
169 | "expected_final_pg_target": 32, | |
170 | }, | |
aee94f69 TL |
171 | "default.rgw.buckets.index": { |
172 | "pool": 97, | |
522d829b TL |
173 | "pool_name": "default.rgw.buckets.index", |
174 | "pg_num_target": 32, | |
175 | "size": 3, | |
176 | "crush_rule": 4, | |
177 | "capacity_ratio": 0.0002, | |
178 | "options": {}, | |
179 | "expected_final_pg_target": 32, | |
180 | }, | |
aee94f69 TL |
181 | "default.rgw.buckets.data": { |
182 | "pool": 98, | |
522d829b TL |
183 | "pool_name": "default.rgw.buckets.data", |
184 | "pg_num_target": 32, | |
185 | "size": 3, | |
186 | "crush_rule": 0, | |
187 | "capacity_ratio": 0.0457, | |
188 | "options": {}, | |
189 | "expected_final_pg_target": 128, | |
190 | }, | |
aee94f69 TL |
191 | "default.rgw.buckets.non-ec": { |
192 | "pool": 99, | |
522d829b TL |
193 | "pool_name": "default.rgw.buckets.non-ec", |
194 | "pg_num_target": 32, | |
195 | "size": 3, | |
196 | "crush_rule": 0, | |
197 | "capacity_ratio": 0, | |
198 | "options": {}, | |
199 | "expected_final_pg_target": 32, | |
200 | }, | |
aee94f69 TL |
201 | "device_health_metrics": { |
202 | "pool": 100, | |
522d829b TL |
203 | "pool_name": "device_health_metrics", |
204 | "pg_num_target": 1, | |
205 | "size": 3, | |
206 | "crush_rule": 0, | |
207 | "capacity_ratio": 0, | |
208 | "options": { | |
209 | "pg_num_min": 1 | |
210 | }, | |
211 | "expected_final_pg_target": 1, | |
212 | }, | |
aee94f69 TL |
213 | "cephfs.teuthology.meta": { |
214 | "pool": 113, | |
522d829b TL |
215 | "pool_name": "cephfs.teuthology.meta", |
216 | "pg_num_target": 64, | |
217 | "size": 3, | |
218 | "crush_rule": 4, | |
219 | "capacity_ratio": 0.1389, | |
220 | "options": { | |
221 | "pg_autoscale_bias": 4, | |
222 | "pg_num_min": 64, | |
223 | }, | |
224 | "expected_final_pg_target": 512, | |
225 | }, | |
aee94f69 TL |
226 | "cephfs.teuthology.data": { |
227 | "pool": 114, | |
522d829b TL |
228 | "pool_name": "cephfs.teuthology.data", |
229 | "pg_num_target": 256, | |
230 | "size": 3, | |
231 | "crush_rule": 0, | |
232 | "capacity_ratio": 0.0006, | |
233 | "options": { | |
234 | "pg_num_min": 128, | |
235 | }, | |
236 | "expected_final_pg_target": 1024, | |
237 | "expected_final_pg_target": 256, | |
238 | }, | |
aee94f69 TL |
239 | "cephfs.scratch.meta": { |
240 | "pool": 117, | |
522d829b TL |
241 | "pool_name": "cephfs.scratch.meta", |
242 | "pg_num_target": 32, | |
243 | "size": 3, | |
244 | "crush_rule": 4, | |
245 | "capacity_ratio": 0.0027, | |
246 | "options": { | |
247 | "pg_autoscale_bias": 4, | |
248 | "pg_num_min": 16, | |
249 | }, | |
250 | "expected_final_pg_target": 64, | |
251 | }, | |
aee94f69 TL |
252 | "cephfs.scratch.data": { |
253 | "pool": 118, | |
522d829b TL |
254 | "pool_name": "cephfs.scratch.data", |
255 | "pg_num_target": 32, | |
256 | "size": 3, | |
257 | "crush_rule": 0, | |
258 | "capacity_ratio": 0.0027, | |
259 | "options": {}, | |
260 | "expected_final_pg_target": 128, | |
261 | }, | |
aee94f69 TL |
262 | "cephfs.teuthology.data-ec": { |
263 | "pool": 119, | |
522d829b TL |
264 | "pool_name": "cephfs.teuthology.data-ec", |
265 | "pg_num_target": 1024, | |
266 | "size": 6, | |
267 | "crush_rule": 1, | |
268 | "capacity_ratio": 0.8490, | |
269 | "options": { | |
270 | "pg_num_min": 1024 | |
271 | }, | |
272 | "expected_final_pg_target": 1024, | |
273 | }, | |
aee94f69 TL |
274 | "cephsqlite": { |
275 | "pool": 121, | |
522d829b TL |
276 | "pool_name": "cephsqlite", |
277 | "pg_num_target": 32, | |
278 | "size": 3, | |
279 | "crush_rule": 4, | |
280 | "capacity_ratio": 0, | |
281 | "options": {}, | |
282 | "expected_final_pg_target": 128, | |
283 | }, | |
284 | } | |
285 | expected_overlapped_roots = {-40, -1, -5} | |
286 | self.helper_test(osd_dic, rules, pools, expected_overlapped_roots) | |
287 | ||
288 | def test_no_overlaps(self): | |
289 | osd_dic = { | |
290 | -1: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], | |
291 | -40: [11, 12, 13, 14, 15], | |
292 | -5: [16, 17, 18], | |
293 | } | |
294 | ||
295 | rules = [ | |
296 | { | |
297 | "rule_id": 0, | |
298 | "rule_name": "data", | |
299 | "ruleset": 0, | |
300 | "type": 1, | |
301 | "min_size": 1, | |
302 | "max_size": 10, | |
303 | "root_id": -1, | |
304 | }, | |
305 | { | |
306 | "rule_id": 1, | |
307 | "rule_name": "teuthology-data-ec", | |
308 | "ruleset": 1, | |
309 | "type": 3, | |
310 | "min_size": 3, | |
311 | "max_size": 6, | |
312 | "root_id": -5, | |
313 | }, | |
314 | { | |
315 | "rule_id": 4, | |
316 | "rule_name": "rep-ssd", | |
317 | "ruleset": 4, | |
318 | "type": 1, | |
319 | "min_size": 1, | |
320 | "max_size": 10, | |
321 | "root_id": -40, | |
322 | }, | |
323 | ] | |
324 | pools = { | |
aee94f69 TL |
325 | "data": { |
326 | "pool": 0, | |
522d829b TL |
327 | "pool_name": "data", |
328 | "pg_num_target": 1024, | |
329 | "size": 3, | |
330 | "crush_rule": 0, | |
331 | "capacity_ratio": 0.1624, | |
332 | "options": { | |
333 | "pg_num_min": 1024, | |
334 | }, | |
335 | "expected_final_pg_target": 1024, | |
336 | }, | |
aee94f69 TL |
337 | "metadata": { |
338 | "pool": 1, | |
522d829b TL |
339 | "pool_name": "metadata", |
340 | "pg_num_target": 64, | |
341 | "size": 3, | |
342 | "crush_rule": 4, | |
343 | "capacity_ratio": 0.0144, | |
344 | "options": { | |
345 | "pg_num_min": 64, | |
346 | }, | |
347 | "expected_final_pg_target": 64, | |
348 | }, | |
aee94f69 TL |
349 | "libvirt-pool": { |
350 | "pool": 4, | |
522d829b TL |
351 | "pool_name": "libvirt-pool", |
352 | "pg_num_target": 32, | |
353 | "size": 3, | |
354 | "crush_rule": 0, | |
355 | "capacity_ratio": 0.0001, | |
356 | "options": {}, | |
357 | "expected_final_pg_target": 128, | |
358 | }, | |
aee94f69 TL |
359 | ".rgw.root": { |
360 | "pool": 93, | |
522d829b TL |
361 | "pool_name": ".rgw.root", |
362 | "pg_num_target": 32, | |
363 | "size": 3, | |
364 | "crush_rule": 4, | |
365 | "capacity_ratio": 0, | |
366 | "options": {}, | |
367 | "expected_final_pg_target": 32, | |
368 | }, | |
aee94f69 TL |
369 | "default.rgw.control": { |
370 | "pool": 94, | |
522d829b TL |
371 | "pool_name": "default.rgw.control", |
372 | "pg_num_target": 32, | |
373 | "size": 3, | |
374 | "crush_rule": 4, | |
375 | "capacity_ratio": 0, | |
376 | "options": {}, | |
377 | "expected_final_pg_target": 32, | |
378 | }, | |
aee94f69 TL |
379 | "default.rgw.meta": { |
380 | "pool": 95, | |
522d829b TL |
381 | "pool_name": "default.rgw.meta", |
382 | "pg_num_target": 32, | |
383 | "size": 3, | |
384 | "crush_rule": 4, | |
385 | "capacity_ratio": 0, | |
386 | "options": {}, | |
387 | "expected_final_pg_target": 32, | |
388 | }, | |
aee94f69 TL |
389 | "default.rgw.log": { |
390 | "pool": 96, | |
522d829b TL |
391 | "pool_name": "default.rgw.log", |
392 | "pg_num_target": 32, | |
393 | "size": 3, | |
394 | "crush_rule": 4, | |
395 | "capacity_ratio": 0, | |
396 | "options": {}, | |
397 | "expected_final_pg_target": 32, | |
398 | }, | |
aee94f69 TL |
399 | "default.rgw.buckets.index": { |
400 | "pool": 97, | |
522d829b TL |
401 | "pool_name": "default.rgw.buckets.index", |
402 | "pg_num_target": 32, | |
403 | "size": 3, | |
404 | "crush_rule": 4, | |
405 | "capacity_ratio": 0.0002, | |
406 | "options": {}, | |
407 | "expected_final_pg_target": 32, | |
408 | }, | |
aee94f69 TL |
409 | "default.rgw.buckets.data": { |
410 | "pool": 98, | |
522d829b TL |
411 | "pool_name": "default.rgw.buckets.data", |
412 | "pg_num_target": 32, | |
413 | "size": 3, | |
414 | "crush_rule": 0, | |
415 | "capacity_ratio": 0.0457, | |
416 | "options": {}, | |
417 | "expected_final_pg_target": 128, | |
418 | }, | |
aee94f69 TL |
419 | "default.rgw.buckets.non-ec": { |
420 | "pool": 99, | |
522d829b TL |
421 | "pool_name": "default.rgw.buckets.non-ec", |
422 | "pg_num_target": 32, | |
423 | "size": 3, | |
424 | "crush_rule": 0, | |
425 | "capacity_ratio": 0, | |
426 | "options": {}, | |
427 | "expected_final_pg_target": 32, | |
428 | }, | |
aee94f69 TL |
429 | "device_health_metrics": { |
430 | "pool": 100, | |
522d829b TL |
431 | "pool_name": "device_health_metrics", |
432 | "pg_num_target": 1, | |
433 | "size": 3, | |
434 | "crush_rule": 0, | |
435 | "capacity_ratio": 0, | |
436 | "options": { | |
437 | "pg_num_min": 1 | |
438 | }, | |
439 | "expected_final_pg_target": 1, | |
440 | }, | |
aee94f69 TL |
441 | "cephfs.teuthology.meta": { |
442 | "pool": 113, | |
522d829b TL |
443 | "pool_name": "cephfs.teuthology.meta", |
444 | "pg_num_target": 64, | |
445 | "size": 3, | |
446 | "crush_rule": 4, | |
447 | "capacity_ratio": 0.1389, | |
448 | "options": { | |
449 | "pg_autoscale_bias": 4, | |
450 | "pg_num_min": 64, | |
451 | }, | |
452 | "expected_final_pg_target": 512, | |
453 | }, | |
aee94f69 TL |
454 | "cephfs.teuthology.data": { |
455 | "pool": 114, | |
522d829b TL |
456 | "pool_name": "cephfs.teuthology.data", |
457 | "pg_num_target": 256, | |
458 | "size": 3, | |
459 | "crush_rule": 0, | |
460 | "capacity_ratio": 0.0006, | |
461 | "options": { | |
462 | "pg_num_min": 128, | |
463 | }, | |
464 | "expected_final_pg_target": 1024, | |
465 | "expected_final_pg_target": 256, | |
466 | }, | |
aee94f69 TL |
467 | "cephfs.scratch.meta": { |
468 | "pool": 117, | |
522d829b TL |
469 | "pool_name": "cephfs.scratch.meta", |
470 | "pg_num_target": 32, | |
471 | "size": 3, | |
472 | "crush_rule": 4, | |
473 | "capacity_ratio": 0.0027, | |
474 | "options": { | |
475 | "pg_autoscale_bias": 4, | |
476 | "pg_num_min": 16, | |
477 | }, | |
478 | "expected_final_pg_target": 64, | |
479 | }, | |
aee94f69 TL |
480 | "cephfs.scratch.data": { |
481 | "pool": 118, | |
522d829b TL |
482 | "pool_name": "cephfs.scratch.data", |
483 | "pg_num_target": 32, | |
484 | "size": 3, | |
485 | "crush_rule": 0, | |
486 | "capacity_ratio": 0.0027, | |
487 | "options": {}, | |
488 | "expected_final_pg_target": 128, | |
489 | }, | |
aee94f69 TL |
490 | "cephfs.teuthology.data-ec": { |
491 | "pool": 119, | |
522d829b TL |
492 | "pool_name": "cephfs.teuthology.data-ec", |
493 | "pg_num_target": 1024, | |
494 | "size": 6, | |
495 | "crush_rule": 1, | |
496 | "capacity_ratio": 0.8490, | |
497 | "options": { | |
498 | "pg_num_min": 1024 | |
499 | }, | |
500 | "expected_final_pg_target": 1024, | |
501 | }, | |
aee94f69 TL |
502 | "cephsqlite": { |
503 | "pool": 121, | |
522d829b TL |
504 | "pool_name": "cephsqlite", |
505 | "pg_num_target": 32, | |
506 | "size": 3, | |
507 | "crush_rule": 4, | |
508 | "capacity_ratio": 0, | |
509 | "options": {}, | |
510 | "expected_final_pg_target": 128, | |
511 | }, | |
512 | } | |
513 | expected_overlapped_roots = set() | |
514 | self.helper_test(osd_dic, rules, pools, expected_overlapped_roots) |