ceph/src/pybind/mgr/pg_autoscaler/tests/test_overlapping_roots.py

   1 # python unit test
   2 import unittest
   3 from tests import mock
   4 import pytest
   5 import json
   6 from pg_autoscaler import module
   7
   8
   9 class OSDMAP:
  10     def __init__(self, pools):
  11         self.pools = pools
  12
  13     def get_pools(self):
  14         return self.pools
  15
  16     def pool_raw_used_rate(pool_id):
  17         return 1
  18
  19
  20 class CRUSH:
  21     def __init__(self, rules, osd_dic):
  22         self.rules = rules
  23         self.osd_dic = osd_dic
  24
  25     def get_rule_by_id(self, rule_id):
  26         for rule in self.rules:
  27             if rule['rule_id'] == rule_id:
  28                 return rule
  29
  30         return None
  31
  32     def get_rule_root(self, rule_name):
  33         for rule in self.rules:
  34             if rule['rule_name'] == rule_name:
  35                 return rule['root_id']
  36
  37         return None
  38
  39     def get_osds_under(self, root_id):
  40         return self.osd_dic[root_id]
  41
  42
  43 class TestPgAutoscaler(object):
  44
  45     def setup(self):
  46         # a bunch of attributes for testing.
  47         self.autoscaler = module.PgAutoscaler('module_name', 0, 0)
  48
  49     def helper_test(self, osd_dic, rules, pools, expected_overlapped_roots):
  50         result = {}
  51         roots = []
  52         overlapped_roots = set()
  53         osdmap = OSDMAP(pools)
  54         crush = CRUSH(rules, osd_dic)
  55         roots, overlapped_roots = self.autoscaler.identify_subtrees_and_overlaps(osdmap,
  56                                                                                  crush, result, overlapped_roots, roots)
  57         assert overlapped_roots == expected_overlapped_roots
  58
  59     def test_subtrees_and_overlaps(self):
  60         osd_dic = {
  61             -1: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
  62             -40: [11, 12, 13, 14, 15],
  63             -5: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
  64         }
  65
  66         rules = [
  67             {
  68                 "rule_id": 0,
  69                 "rule_name": "data",
  70                 "ruleset": 0,
  71                 "type": 1,
  72                 "min_size": 1,
  73                 "max_size": 10,
  74                 "root_id": -1,
  75             },
  76             {
  77                 "rule_id": 1,
  78                 "rule_name": "teuthology-data-ec",
  79                 "ruleset": 1,
  80                 "type": 3,
  81                 "min_size": 3,
  82                 "max_size": 6,
  83                 "root_id": -5,
  84             },
  85             {
  86                 "rule_id": 4,
  87                 "rule_name": "rep-ssd",
  88                 "ruleset": 4,
  89                 "type": 1,
  90                 "min_size": 1,
  91                 "max_size": 10,
  92                 "root_id": -40,
  93             },
  94         ]
  95         pools = {
  96             0: {
  97                 "pool_name": "data",
  98                 "pg_num_target": 1024,
  99                 "size": 3,
 100                 "crush_rule": 0,
 101                 "capacity_ratio": 0.1624,
 102                 "options": {
 103                     "pg_num_min": 1024,
 104                 },
 105                 "expected_final_pg_target": 1024,
 106             },
 107             1: {
 108                 "pool_name": "metadata",
 109                 "pg_num_target": 64,
 110                 "size": 3,
 111                 "crush_rule": 4,
 112                 "capacity_ratio": 0.0144,
 113                 "options": {
 114                     "pg_num_min": 64,
 115                 },
 116                 "expected_final_pg_target": 64,
 117             },
 118             4: {
 119                 "pool_name": "libvirt-pool",
 120                 "pg_num_target": 32,
 121                 "size": 3,
 122                 "crush_rule": 0,
 123                 "capacity_ratio": 0.0001,
 124                 "options": {},
 125                 "expected_final_pg_target": 128,
 126             },
 127             93: {
 128                 "pool_name": ".rgw.root",
 129                 "pg_num_target": 32,
 130                 "size": 3,
 131                 "crush_rule": 4,
 132                 "capacity_ratio": 0,
 133                 "options": {},
 134                 "expected_final_pg_target": 32,
 135             },
 136             94: {
 137                 "pool_name": "default.rgw.control",
 138                 "pg_num_target": 32,
 139                 "size": 3,
 140                 "crush_rule": 4,
 141                 "capacity_ratio": 0,
 142                 "options": {},
 143                 "expected_final_pg_target": 32,
 144             },
 145             95: {
 146                 "pool_name": "default.rgw.meta",
 147                 "pg_num_target": 32,
 148                 "size": 3,
 149                 "crush_rule": 4,
 150                 "capacity_ratio": 0,
 151                 "options": {},
 152                 "expected_final_pg_target": 32,
 153             },
 154             96: {
 155                 "pool_name": "default.rgw.log",
 156                 "pg_num_target": 32,
 157                 "size": 3,
 158                 "crush_rule": 4,
 159                 "capacity_ratio": 0,
 160                 "options": {},
 161                 "expected_final_pg_target": 32,
 162             },
 163             97: {
 164                 "pool_name": "default.rgw.buckets.index",
 165                 "pg_num_target": 32,
 166                 "size": 3,
 167                 "crush_rule": 4,
 168                 "capacity_ratio": 0.0002,
 169                 "options": {},
 170                 "expected_final_pg_target": 32,
 171             },
 172             98: {
 173                 "pool_name": "default.rgw.buckets.data",
 174                 "pg_num_target": 32,
 175                 "size": 3,
 176                 "crush_rule": 0,
 177                 "capacity_ratio": 0.0457,
 178                 "options": {},
 179                 "expected_final_pg_target": 128,
 180             },
 181             99: {
 182                 "pool_name": "default.rgw.buckets.non-ec",
 183                 "pg_num_target": 32,
 184                 "size": 3,
 185                 "crush_rule": 0,
 186                 "capacity_ratio": 0,
 187                 "options": {},
 188                 "expected_final_pg_target": 32,
 189             },
 190             100: {
 191                 "pool_name": "device_health_metrics",
 192                 "pg_num_target": 1,
 193                 "size": 3,
 194                 "crush_rule": 0,
 195                 "capacity_ratio": 0,
 196                 "options": {
 197                     "pg_num_min": 1
 198                 },
 199                 "expected_final_pg_target": 1,
 200             },
 201             113: {
 202                 "pool_name": "cephfs.teuthology.meta",
 203                 "pg_num_target": 64,
 204                 "size": 3,
 205                 "crush_rule": 4,
 206                 "capacity_ratio": 0.1389,
 207                 "options": {
 208                     "pg_autoscale_bias": 4,
 209                     "pg_num_min": 64,
 210                 },
 211                 "expected_final_pg_target": 512,
 212             },
 213             114: {
 214                 "pool_name": "cephfs.teuthology.data",
 215                 "pg_num_target": 256,
 216                 "size": 3,
 217                 "crush_rule": 0,
 218                 "capacity_ratio": 0.0006,
 219                 "options": {
 220                     "pg_num_min": 128,
 221                 },
 222                 "expected_final_pg_target": 1024,
 223                 "expected_final_pg_target": 256,
 224             },
 225             117: {
 226                 "pool_name": "cephfs.scratch.meta",
 227                 "pg_num_target": 32,
 228                 "size": 3,
 229                 "crush_rule": 4,
 230                 "capacity_ratio": 0.0027,
 231                 "options": {
 232                     "pg_autoscale_bias": 4,
 233                     "pg_num_min": 16,
 234                 },
 235                 "expected_final_pg_target": 64,
 236             },
 237             118: {
 238                 "pool_name": "cephfs.scratch.data",
 239                 "pg_num_target": 32,
 240                 "size": 3,
 241                 "crush_rule": 0,
 242                 "capacity_ratio": 0.0027,
 243                 "options": {},
 244                 "expected_final_pg_target": 128,
 245             },
 246             119: {
 247                 "pool_name": "cephfs.teuthology.data-ec",
 248                 "pg_num_target": 1024,
 249                 "size": 6,
 250                 "crush_rule": 1,
 251                 "capacity_ratio": 0.8490,
 252                 "options": {
 253                     "pg_num_min": 1024
 254                 },
 255                 "expected_final_pg_target": 1024,
 256             },
 257             121: {
 258                 "pool_name": "cephsqlite",
 259                 "pg_num_target": 32,
 260                 "size": 3,
 261                 "crush_rule": 4,
 262                 "capacity_ratio": 0,
 263                 "options": {},
 264                 "expected_final_pg_target": 128,
 265             },
 266         }
 267         expected_overlapped_roots = {-40, -1, -5}
 268         self.helper_test(osd_dic, rules, pools, expected_overlapped_roots)
 269
 270     def test_no_overlaps(self):
 271         osd_dic = {
 272             -1: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
 273             -40: [11, 12, 13, 14, 15],
 274             -5: [16, 17, 18],
 275         }
 276
 277         rules = [
 278             {
 279                 "rule_id": 0,
 280                 "rule_name": "data",
 281                 "ruleset": 0,
 282                 "type": 1,
 283                 "min_size": 1,
 284                 "max_size": 10,
 285                 "root_id": -1,
 286             },
 287             {
 288                 "rule_id": 1,
 289                 "rule_name": "teuthology-data-ec",
 290                 "ruleset": 1,
 291                 "type": 3,
 292                 "min_size": 3,
 293                 "max_size": 6,
 294                 "root_id": -5,
 295             },
 296             {
 297                 "rule_id": 4,
 298                 "rule_name": "rep-ssd",
 299                 "ruleset": 4,
 300                 "type": 1,
 301                 "min_size": 1,
 302                 "max_size": 10,
 303                 "root_id": -40,
 304             },
 305         ]
 306         pools = {
 307             0: {
 308                 "pool_name": "data",
 309                 "pg_num_target": 1024,
 310                 "size": 3,
 311                 "crush_rule": 0,
 312                 "capacity_ratio": 0.1624,
 313                 "options": {
 314                     "pg_num_min": 1024,
 315                 },
 316                 "expected_final_pg_target": 1024,
 317             },
 318             1: {
 319                 "pool_name": "metadata",
 320                 "pg_num_target": 64,
 321                 "size": 3,
 322                 "crush_rule": 4,
 323                 "capacity_ratio": 0.0144,
 324                 "options": {
 325                     "pg_num_min": 64,
 326                 },
 327                 "expected_final_pg_target": 64,
 328             },
 329             4: {
 330                 "pool_name": "libvirt-pool",
 331                 "pg_num_target": 32,
 332                 "size": 3,
 333                 "crush_rule": 0,
 334                 "capacity_ratio": 0.0001,
 335                 "options": {},
 336                 "expected_final_pg_target": 128,
 337             },
 338             93: {
 339                 "pool_name": ".rgw.root",
 340                 "pg_num_target": 32,
 341                 "size": 3,
 342                 "crush_rule": 4,
 343                 "capacity_ratio": 0,
 344                 "options": {},
 345                 "expected_final_pg_target": 32,
 346             },
 347             94: {
 348                 "pool_name": "default.rgw.control",
 349                 "pg_num_target": 32,
 350                 "size": 3,
 351                 "crush_rule": 4,
 352                 "capacity_ratio": 0,
 353                 "options": {},
 354                 "expected_final_pg_target": 32,
 355             },
 356             95: {
 357                 "pool_name": "default.rgw.meta",
 358                 "pg_num_target": 32,
 359                 "size": 3,
 360                 "crush_rule": 4,
 361                 "capacity_ratio": 0,
 362                 "options": {},
 363                 "expected_final_pg_target": 32,
 364             },
 365             96: {
 366                 "pool_name": "default.rgw.log",
 367                 "pg_num_target": 32,
 368                 "size": 3,
 369                 "crush_rule": 4,
 370                 "capacity_ratio": 0,
 371                 "options": {},
 372                 "expected_final_pg_target": 32,
 373             },
 374             97: {
 375                 "pool_name": "default.rgw.buckets.index",
 376                 "pg_num_target": 32,
 377                 "size": 3,
 378                 "crush_rule": 4,
 379                 "capacity_ratio": 0.0002,
 380                 "options": {},
 381                 "expected_final_pg_target": 32,
 382             },
 383             98: {
 384                 "pool_name": "default.rgw.buckets.data",
 385                 "pg_num_target": 32,
 386                 "size": 3,
 387                 "crush_rule": 0,
 388                 "capacity_ratio": 0.0457,
 389                 "options": {},
 390                 "expected_final_pg_target": 128,
 391             },
 392             99: {
 393                 "pool_name": "default.rgw.buckets.non-ec",
 394                 "pg_num_target": 32,
 395                 "size": 3,
 396                 "crush_rule": 0,
 397                 "capacity_ratio": 0,
 398                 "options": {},
 399                 "expected_final_pg_target": 32,
 400             },
 401             100: {
 402                 "pool_name": "device_health_metrics",
 403                 "pg_num_target": 1,
 404                 "size": 3,
 405                 "crush_rule": 0,
 406                 "capacity_ratio": 0,
 407                 "options": {
 408                     "pg_num_min": 1
 409                 },
 410                 "expected_final_pg_target": 1,
 411             },
 412             113: {
 413                 "pool_name": "cephfs.teuthology.meta",
 414                 "pg_num_target": 64,
 415                 "size": 3,
 416                 "crush_rule": 4,
 417                 "capacity_ratio": 0.1389,
 418                 "options": {
 419                     "pg_autoscale_bias": 4,
 420                     "pg_num_min": 64,
 421                 },
 422                 "expected_final_pg_target": 512,
 423             },
 424             114: {
 425                 "pool_name": "cephfs.teuthology.data",
 426                 "pg_num_target": 256,
 427                 "size": 3,
 428                 "crush_rule": 0,
 429                 "capacity_ratio": 0.0006,
 430                 "options": {
 431                     "pg_num_min": 128,
 432                 },
 433                 "expected_final_pg_target": 1024,
 434                 "expected_final_pg_target": 256,
 435             },
 436             117: {
 437                 "pool_name": "cephfs.scratch.meta",
 438                 "pg_num_target": 32,
 439                 "size": 3,
 440                 "crush_rule": 4,
 441                 "capacity_ratio": 0.0027,
 442                 "options": {
 443                     "pg_autoscale_bias": 4,
 444                     "pg_num_min": 16,
 445                 },
 446                 "expected_final_pg_target": 64,
 447             },
 448             118: {
 449                 "pool_name": "cephfs.scratch.data",
 450                 "pg_num_target": 32,
 451                 "size": 3,
 452                 "crush_rule": 0,
 453                 "capacity_ratio": 0.0027,
 454                 "options": {},
 455                 "expected_final_pg_target": 128,
 456             },
 457             119: {
 458                 "pool_name": "cephfs.teuthology.data-ec",
 459                 "pg_num_target": 1024,
 460                 "size": 6,
 461                 "crush_rule": 1,
 462                 "capacity_ratio": 0.8490,
 463                 "options": {
 464                     "pg_num_min": 1024
 465                 },
 466                 "expected_final_pg_target": 1024,
 467             },
 468             121: {
 469                 "pool_name": "cephsqlite",
 470                 "pg_num_target": 32,
 471                 "size": 3,
 472                 "crush_rule": 4,
 473                 "capacity_ratio": 0,
 474                 "options": {},
 475                 "expected_final_pg_target": 128,
 476             },
 477         }
 478         expected_overlapped_roots = set()
 479         self.helper_test(osd_dic, rules, pools, expected_overlapped_roots)