ceph/src/pybind/mgr/pg_autoscaler/tests/test_overlapping_roots.py

   1 # python unit test
   2 import unittest
   3 from tests import mock
   4 import pytest
   5 import json
   6 from pg_autoscaler import module
   7
   8
   9 class OSDMAP:
  10     def __init__(self, pools):
  11         self.pools = pools
  12
  13     def get_pools(self):
  14         return self.pools
  15
  16     def pool_raw_used_rate(pool_id):
  17         return 1
  18
  19
  20 class CRUSH:
  21     def __init__(self, rules, osd_dic):
  22         self.rules = rules
  23         self.osd_dic = osd_dic
  24
  25     def get_rule_by_id(self, rule_id):
  26         for rule in self.rules:
  27             if rule['rule_id'] == rule_id:
  28                 return rule
  29
  30         return None
  31
  32     def get_rule_root(self, rule_name):
  33         for rule in self.rules:
  34             if rule['rule_name'] == rule_name:
  35                 return rule['root_id']
  36
  37         return None
  38
  39     def get_osds_under(self, root_id):
  40         return self.osd_dic[root_id]
  41
  42
  43 class TestPgAutoscaler(object):
  44
  45     def setup_method(self):
  46         # a bunch of attributes for testing.
  47         self.autoscaler = module.PgAutoscaler('module_name', 0, 0)
  48
  49     def helper_test(self, osd_dic, rules, pools, expected_overlapped_roots):
  50         result = {}
  51         roots = []
  52         overlapped_roots = set()
  53         osdmap = OSDMAP(pools)
  54         crush = CRUSH(rules, osd_dic)
  55         roots, overlapped_roots = self.autoscaler.identify_subtrees_and_overlaps(
  56             osdmap, pools, crush, result, overlapped_roots, roots
  57         )
  58         assert overlapped_roots == expected_overlapped_roots
  59
  60     def test_subtrees_and_overlaps(self):
  61         osd_dic = {
  62             -1: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
  63             -40: [11, 12, 13, 14, 15],
  64             -5: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
  65         }
  66
  67         rules = [
  68             {
  69                 "rule_id": 0,
  70                 "rule_name": "data",
  71                 "ruleset": 0,
  72                 "type": 1,
  73                 "min_size": 1,
  74                 "max_size": 10,
  75                 "root_id": -1,
  76             },
  77             {
  78                 "rule_id": 1,
  79                 "rule_name": "teuthology-data-ec",
  80                 "ruleset": 1,
  81                 "type": 3,
  82                 "min_size": 3,
  83                 "max_size": 6,
  84                 "root_id": -5,
  85             },
  86             {
  87                 "rule_id": 4,
  88                 "rule_name": "rep-ssd",
  89                 "ruleset": 4,
  90                 "type": 1,
  91                 "min_size": 1,
  92                 "max_size": 10,
  93                 "root_id": -40,
  94             },
  95         ]
  96         pools = {
  97             "data": {
  98                 "pool": 0,
  99                 "pool_name": "data",
 100                 "pg_num_target": 1024,
 101                 "size": 3,
 102                 "crush_rule": 0,
 103                 "capacity_ratio": 0.1624,
 104                 "options": {
 105                     "pg_num_min": 1024,
 106                 },
 107                 "expected_final_pg_target": 1024,
 108             },
 109             "metadata": {
 110                 "pool": 1,
 111                 "pool_name": "metadata",
 112                 "pg_num_target": 64,
 113                 "size": 3,
 114                 "crush_rule": 4,
 115                 "capacity_ratio": 0.0144,
 116                 "options": {
 117                     "pg_num_min": 64,
 118                 },
 119                 "expected_final_pg_target": 64,
 120             },
 121             "libvirt-pool": {
 122                 "pool": 4,
 123                 "pool_name": "libvirt-pool",
 124                 "pg_num_target": 32,
 125                 "size": 3,
 126                 "crush_rule": 0,
 127                 "capacity_ratio": 0.0001,
 128                 "options": {},
 129                 "expected_final_pg_target": 128,
 130             },
 131             ".rgw.root": {
 132                 "pool": 93,
 133                 "pool_name": ".rgw.root",
 134                 "pg_num_target": 32,
 135                 "size": 3,
 136                 "crush_rule": 4,
 137                 "capacity_ratio": 0,
 138                 "options": {},
 139                 "expected_final_pg_target": 32,
 140             },
 141             "default.rgw.control": {
 142                 "pool": 94,
 143                 "pool_name": "default.rgw.control",
 144                 "pg_num_target": 32,
 145                 "size": 3,
 146                 "crush_rule": 4,
 147                 "capacity_ratio": 0,
 148                 "options": {},
 149                 "expected_final_pg_target": 32,
 150             },
 151             "default.rgw.meta": {
 152                 "pool": 95,
 153                 "pool_name": "default.rgw.meta",
 154                 "pg_num_target": 32,
 155                 "size": 3,
 156                 "crush_rule": 4,
 157                 "capacity_ratio": 0,
 158                 "options": {},
 159                 "expected_final_pg_target": 32,
 160             },
 161             "default.rgw.log": {
 162                 "pool": 96,
 163                 "pool_name": "default.rgw.log",
 164                 "pg_num_target": 32,
 165                 "size": 3,
 166                 "crush_rule": 4,
 167                 "capacity_ratio": 0,
 168                 "options": {},
 169                 "expected_final_pg_target": 32,
 170             },
 171             "default.rgw.buckets.index": {
 172                 "pool": 97,
 173                 "pool_name": "default.rgw.buckets.index",
 174                 "pg_num_target": 32,
 175                 "size": 3,
 176                 "crush_rule": 4,
 177                 "capacity_ratio": 0.0002,
 178                 "options": {},
 179                 "expected_final_pg_target": 32,
 180             },
 181             "default.rgw.buckets.data": {
 182                 "pool": 98,
 183                 "pool_name": "default.rgw.buckets.data",
 184                 "pg_num_target": 32,
 185                 "size": 3,
 186                 "crush_rule": 0,
 187                 "capacity_ratio": 0.0457,
 188                 "options": {},
 189                 "expected_final_pg_target": 128,
 190             },
 191             "default.rgw.buckets.non-ec": {
 192                 "pool": 99,
 193                 "pool_name": "default.rgw.buckets.non-ec",
 194                 "pg_num_target": 32,
 195                 "size": 3,
 196                 "crush_rule": 0,
 197                 "capacity_ratio": 0,
 198                 "options": {},
 199                 "expected_final_pg_target": 32,
 200             },
 201             "device_health_metrics": {
 202                 "pool": 100,
 203                 "pool_name": "device_health_metrics",
 204                 "pg_num_target": 1,
 205                 "size": 3,
 206                 "crush_rule": 0,
 207                 "capacity_ratio": 0,
 208                 "options": {
 209                     "pg_num_min": 1
 210                 },
 211                 "expected_final_pg_target": 1,
 212             },
 213             "cephfs.teuthology.meta": {
 214                 "pool": 113,
 215                 "pool_name": "cephfs.teuthology.meta",
 216                 "pg_num_target": 64,
 217                 "size": 3,
 218                 "crush_rule": 4,
 219                 "capacity_ratio": 0.1389,
 220                 "options": {
 221                     "pg_autoscale_bias": 4,
 222                     "pg_num_min": 64,
 223                 },
 224                 "expected_final_pg_target": 512,
 225             },
 226             "cephfs.teuthology.data": {
 227                 "pool": 114,
 228                 "pool_name": "cephfs.teuthology.data",
 229                 "pg_num_target": 256,
 230                 "size": 3,
 231                 "crush_rule": 0,
 232                 "capacity_ratio": 0.0006,
 233                 "options": {
 234                     "pg_num_min": 128,
 235                 },
 236                 "expected_final_pg_target": 1024,
 237                 "expected_final_pg_target": 256,
 238             },
 239             "cephfs.scratch.meta": {
 240                 "pool": 117,
 241                 "pool_name": "cephfs.scratch.meta",
 242                 "pg_num_target": 32,
 243                 "size": 3,
 244                 "crush_rule": 4,
 245                 "capacity_ratio": 0.0027,
 246                 "options": {
 247                     "pg_autoscale_bias": 4,
 248                     "pg_num_min": 16,
 249                 },
 250                 "expected_final_pg_target": 64,
 251             },
 252             "cephfs.scratch.data": {
 253                 "pool": 118,
 254                 "pool_name": "cephfs.scratch.data",
 255                 "pg_num_target": 32,
 256                 "size": 3,
 257                 "crush_rule": 0,
 258                 "capacity_ratio": 0.0027,
 259                 "options": {},
 260                 "expected_final_pg_target": 128,
 261             },
 262             "cephfs.teuthology.data-ec": {
 263                 "pool": 119,
 264                 "pool_name": "cephfs.teuthology.data-ec",
 265                 "pg_num_target": 1024,
 266                 "size": 6,
 267                 "crush_rule": 1,
 268                 "capacity_ratio": 0.8490,
 269                 "options": {
 270                     "pg_num_min": 1024
 271                 },
 272                 "expected_final_pg_target": 1024,
 273             },
 274             "cephsqlite": {
 275                 "pool": 121,
 276                 "pool_name": "cephsqlite",
 277                 "pg_num_target": 32,
 278                 "size": 3,
 279                 "crush_rule": 4,
 280                 "capacity_ratio": 0,
 281                 "options": {},
 282                 "expected_final_pg_target": 128,
 283             },
 284         }
 285         expected_overlapped_roots = {-40, -1, -5}
 286         self.helper_test(osd_dic, rules, pools, expected_overlapped_roots)
 287
 288     def test_no_overlaps(self):
 289         osd_dic = {
 290             -1: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
 291             -40: [11, 12, 13, 14, 15],
 292             -5: [16, 17, 18],
 293         }
 294
 295         rules = [
 296             {
 297                 "rule_id": 0,
 298                 "rule_name": "data",
 299                 "ruleset": 0,
 300                 "type": 1,
 301                 "min_size": 1,
 302                 "max_size": 10,
 303                 "root_id": -1,
 304             },
 305             {
 306                 "rule_id": 1,
 307                 "rule_name": "teuthology-data-ec",
 308                 "ruleset": 1,
 309                 "type": 3,
 310                 "min_size": 3,
 311                 "max_size": 6,
 312                 "root_id": -5,
 313             },
 314             {
 315                 "rule_id": 4,
 316                 "rule_name": "rep-ssd",
 317                 "ruleset": 4,
 318                 "type": 1,
 319                 "min_size": 1,
 320                 "max_size": 10,
 321                 "root_id": -40,
 322             },
 323         ]
 324         pools = {
 325             "data": {
 326                 "pool": 0,
 327                 "pool_name": "data",
 328                 "pg_num_target": 1024,
 329                 "size": 3,
 330                 "crush_rule": 0,
 331                 "capacity_ratio": 0.1624,
 332                 "options": {
 333                     "pg_num_min": 1024,
 334                 },
 335                 "expected_final_pg_target": 1024,
 336             },
 337             "metadata": {
 338                 "pool": 1,
 339                 "pool_name": "metadata",
 340                 "pg_num_target": 64,
 341                 "size": 3,
 342                 "crush_rule": 4,
 343                 "capacity_ratio": 0.0144,
 344                 "options": {
 345                     "pg_num_min": 64,
 346                 },
 347                 "expected_final_pg_target": 64,
 348             },
 349             "libvirt-pool": {
 350                 "pool": 4,
 351                 "pool_name": "libvirt-pool",
 352                 "pg_num_target": 32,
 353                 "size": 3,
 354                 "crush_rule": 0,
 355                 "capacity_ratio": 0.0001,
 356                 "options": {},
 357                 "expected_final_pg_target": 128,
 358             },
 359             ".rgw.root": {
 360                 "pool": 93,
 361                 "pool_name": ".rgw.root",
 362                 "pg_num_target": 32,
 363                 "size": 3,
 364                 "crush_rule": 4,
 365                 "capacity_ratio": 0,
 366                 "options": {},
 367                 "expected_final_pg_target": 32,
 368             },
 369             "default.rgw.control": {
 370                 "pool": 94,
 371                 "pool_name": "default.rgw.control",
 372                 "pg_num_target": 32,
 373                 "size": 3,
 374                 "crush_rule": 4,
 375                 "capacity_ratio": 0,
 376                 "options": {},
 377                 "expected_final_pg_target": 32,
 378             },
 379             "default.rgw.meta": {
 380                 "pool": 95,
 381                 "pool_name": "default.rgw.meta",
 382                 "pg_num_target": 32,
 383                 "size": 3,
 384                 "crush_rule": 4,
 385                 "capacity_ratio": 0,
 386                 "options": {},
 387                 "expected_final_pg_target": 32,
 388             },
 389             "default.rgw.log": {
 390                 "pool": 96,
 391                 "pool_name": "default.rgw.log",
 392                 "pg_num_target": 32,
 393                 "size": 3,
 394                 "crush_rule": 4,
 395                 "capacity_ratio": 0,
 396                 "options": {},
 397                 "expected_final_pg_target": 32,
 398             },
 399             "default.rgw.buckets.index": {
 400                 "pool": 97,
 401                 "pool_name": "default.rgw.buckets.index",
 402                 "pg_num_target": 32,
 403                 "size": 3,
 404                 "crush_rule": 4,
 405                 "capacity_ratio": 0.0002,
 406                 "options": {},
 407                 "expected_final_pg_target": 32,
 408             },
 409             "default.rgw.buckets.data": {
 410                 "pool": 98,
 411                 "pool_name": "default.rgw.buckets.data",
 412                 "pg_num_target": 32,
 413                 "size": 3,
 414                 "crush_rule": 0,
 415                 "capacity_ratio": 0.0457,
 416                 "options": {},
 417                 "expected_final_pg_target": 128,
 418             },
 419             "default.rgw.buckets.non-ec": {
 420                 "pool": 99,
 421                 "pool_name": "default.rgw.buckets.non-ec",
 422                 "pg_num_target": 32,
 423                 "size": 3,
 424                 "crush_rule": 0,
 425                 "capacity_ratio": 0,
 426                 "options": {},
 427                 "expected_final_pg_target": 32,
 428             },
 429             "device_health_metrics": {
 430                 "pool": 100,
 431                 "pool_name": "device_health_metrics",
 432                 "pg_num_target": 1,
 433                 "size": 3,
 434                 "crush_rule": 0,
 435                 "capacity_ratio": 0,
 436                 "options": {
 437                     "pg_num_min": 1
 438                 },
 439                 "expected_final_pg_target": 1,
 440             },
 441             "cephfs.teuthology.meta": {
 442                 "pool": 113,
 443                 "pool_name": "cephfs.teuthology.meta",
 444                 "pg_num_target": 64,
 445                 "size": 3,
 446                 "crush_rule": 4,
 447                 "capacity_ratio": 0.1389,
 448                 "options": {
 449                     "pg_autoscale_bias": 4,
 450                     "pg_num_min": 64,
 451                 },
 452                 "expected_final_pg_target": 512,
 453             },
 454             "cephfs.teuthology.data": {
 455                 "pool": 114,
 456                 "pool_name": "cephfs.teuthology.data",
 457                 "pg_num_target": 256,
 458                 "size": 3,
 459                 "crush_rule": 0,
 460                 "capacity_ratio": 0.0006,
 461                 "options": {
 462                     "pg_num_min": 128,
 463                 },
 464                 "expected_final_pg_target": 1024,
 465                 "expected_final_pg_target": 256,
 466             },
 467             "cephfs.scratch.meta": {
 468                 "pool": 117,
 469                 "pool_name": "cephfs.scratch.meta",
 470                 "pg_num_target": 32,
 471                 "size": 3,
 472                 "crush_rule": 4,
 473                 "capacity_ratio": 0.0027,
 474                 "options": {
 475                     "pg_autoscale_bias": 4,
 476                     "pg_num_min": 16,
 477                 },
 478                 "expected_final_pg_target": 64,
 479             },
 480             "cephfs.scratch.data": {
 481                 "pool": 118,
 482                 "pool_name": "cephfs.scratch.data",
 483                 "pg_num_target": 32,
 484                 "size": 3,
 485                 "crush_rule": 0,
 486                 "capacity_ratio": 0.0027,
 487                 "options": {},
 488                 "expected_final_pg_target": 128,
 489             },
 490             "cephfs.teuthology.data-ec": {
 491                 "pool": 119,
 492                 "pool_name": "cephfs.teuthology.data-ec",
 493                 "pg_num_target": 1024,
 494                 "size": 6,
 495                 "crush_rule": 1,
 496                 "capacity_ratio": 0.8490,
 497                 "options": {
 498                     "pg_num_min": 1024
 499                 },
 500                 "expected_final_pg_target": 1024,
 501             },
 502             "cephsqlite": {
 503                 "pool": 121,
 504                 "pool_name": "cephsqlite",
 505                 "pg_num_target": 32,
 506                 "size": 3,
 507                 "crush_rule": 4,
 508                 "capacity_ratio": 0,
 509                 "options": {},
 510                 "expected_final_pg_target": 128,
 511             },
 512         }
 513         expected_overlapped_roots = set()
 514         self.helper_test(osd_dic, rules, pools, expected_overlapped_roots)