ceph/src/pybind/mgr/pg_autoscaler/tests/test_cal_final_pg_target.py

   1 # python unit test
   2 import unittest
   3 from tests import mock
   4 import pytest
   5 import json
   6 from pg_autoscaler import module
   7
   8
   9 class RootMapItem:
  10
  11     def __init__(self, pool_count, pg_target, pg_left):
  12
  13         self.pool_count = pool_count
  14         self.pg_target = pg_target
  15         self.pg_left = pg_left
  16         self.pool_used = 0
  17
  18
  19 class TestPgAutoscaler(object):
  20
  21     def setup(self):
  22         # a bunch of attributes for testing.
  23         self.autoscaler = module.PgAutoscaler('module_name', 0, 0)
  24
  25     def helper_test(self, pools, root_map, bias, overlapped_roots):
  26         # Here we simulate how _get_pool_pg_target() works.
  27
  28         bulk_pools = {}
  29         even_pools = {}
  30
  31         # first pass
  32         for pool_name, p in pools.items():
  33             root_id = p['root_id']
  34             if root_id in overlapped_roots:
  35                 # skip pools with overlapping roots
  36                 assert p['no_scale']
  37                 continue
  38
  39             final_ratio, pool_pg_target, final_pg_target = self.autoscaler._calc_final_pg_target(
  40                 p, pool_name, root_map,
  41                 p['root_id'], p['capacity_ratio'],
  42                 bias, even_pools, bulk_pools, 'first', p['bulk'])
  43
  44             if final_ratio == None:
  45                 # no final_ratio means current pool is an even pool
  46                 # and we do not have to do any assertion on it.
  47                 continue
  48
  49             assert p['expected_final_pg_target'] == final_pg_target
  50             assert p['expected_final_ratio'] == final_ratio
  51             assert not p['expected_bulk_pool'] and pool_name not in bulk_pools
  52
  53         # second pass
  54         for pool_name, p in bulk_pools.items():
  55             final_ratio, pool_pg_target, final_pg_target = self.autoscaler._calc_final_pg_target(
  56                 p, pool_name, root_map,
  57                 p['root_id'], p['capacity_ratio'],
  58                 bias, even_pools, bulk_pools, 'second', p['bulk'])
  59
  60             if final_ratio == None:
  61                 # no final_ratio means current pool is an even pool
  62                 # and we do not have to do any assertion on it.
  63                 continue
  64
  65             assert p['expected_final_pg_target'] == final_pg_target
  66             assert p['expected_final_ratio'] == final_ratio
  67             assert not p['even_pools'] and pool_name not in even_pools
  68
  69         #third pass
  70         for pool_name, p in even_pools.items():
  71             final_ratio, pool_pg_target, final_pg_target = self.autoscaler._calc_final_pg_target(
  72                 p, pool_name, root_map,
  73                 p['root_id'], p['capacity_ratio'],
  74                 bias, even_pools, bulk_pools, 'third',  p['bulk'])
  75
  76             assert p['expected_final_pg_target'] == final_pg_target
  77             assert p['expected_final_ratio'] == final_ratio
  78             assert p['even_pools'] and pool_name in even_pools
  79
  80     def test_even_pools_one_meta_three_bulk(self):
  81         pools = {
  82
  83             "meta_0": {
  84
  85                 "pool": 0,
  86                 "pool_name": "meta_0",
  87                 "pg_num_target": 32,
  88                 "capacity_ratio": 0.2,
  89                 "root_id": 0,
  90                 "expected_final_pg_target": 64,
  91                 "expected_final_ratio": 0.2,
  92                 "expected_bulk_pool": False,
  93                 "even_pools": False,
  94                 "size": 1,
  95                 "no_scale": False,
  96                 "bulk": False,
  97             },
  98
  99             "bulk_0": {
 100
 101                 "pool": 1,
 102                 "pool_name": "bulk_0",
 103                 "pg_num_target": 32,
 104                 "capacity_ratio": 0.2,
 105                 "root_id": 0,
 106                 "expected_final_pg_target": 128,
 107                 "expected_final_ratio": 1/3,
 108                 "expected_bulk_pool": True,
 109                 "even_pools": True,
 110                 "size": 1,
 111                 "no_scale": False,
 112                 "bulk": True,
 113             },
 114
 115             "bulk_1": {
 116
 117                 "pool": 2,
 118                 "pool_name": "bulk_1",
 119                 "pg_num_target": 32,
 120                 "capacity_ratio": 0.2,
 121                 "root_id": 0,
 122                 "expected_final_pg_target": 128,
 123                 "expected_final_ratio": 1/3,
 124                 "expected_bulk_pool": True,
 125                 "even_pools": True,
 126                 "size": 1,
 127                 "no_scale": False,
 128                 "bulk": True,
 129             },
 130
 131             "bulk_2": {
 132
 133                 "pool": 3,
 134                 "pool_name": "bulk_2",
 135                 "pg_num_target": 32,
 136                 "capacity_ratio": 0.1,
 137                 "root_id": 0,
 138                 "expected_final_pg_target": 128,
 139                 "expected_final_ratio": 1/3,
 140                 "expected_bulk_pool": True,
 141                 "even_pools": True,
 142                 "size": 1,
 143                 "no_scale": False,
 144                 "bulk": True,
 145             },
 146
 147         }
 148
 149         root_map = {
 150
 151             0: RootMapItem(4, 400, 400),
 152             1: RootMapItem(4, 400, 400),
 153
 154         }
 155
 156         bias = 1
 157         overlapped_roots = set()
 158         self.helper_test(pools, root_map, bias, overlapped_roots)
 159
 160     def test_even_pools_two_meta_two_bulk(self):
 161         pools = {
 162
 163             "meta0": {
 164
 165                 "pool": 0,
 166                 "pool_name": "meta0",
 167                 "pg_num_target": 32,
 168                 "capacity_ratio": 0.2,
 169                 "root_id": 0,
 170                 "expected_final_pg_target": 64,
 171                 "expected_final_ratio": 0.2,
 172                 "expected_bulk_pool": False,
 173                 "even_pools": True,
 174                 "size": 1,
 175                 "no_scale": False,
 176                 "bulk": False,
 177             },
 178
 179             "meta1": {
 180
 181                 "pool": 1,
 182                 "pool_name": "meta1",
 183                 "pg_num_target": 32,
 184                 "capacity_ratio": 0.2,
 185                 "root_id": 0,
 186                 "expected_final_pg_target": 64,
 187                 "expected_final_ratio": 0.2,
 188                 "expected_bulk_pool": False,
 189                 "even_pools": True,
 190                 "size": 1,
 191                 "no_scale": False,
 192                 "bulk": False,
 193             },
 194
 195             "bulk0": {
 196
 197                 "pool": 2,
 198                 "pool_name": "bulk0",
 199                 "pg_num_target": 32,
 200                 "capacity_ratio": 0.2,
 201                 "root_id": 0,
 202                 "expected_final_pg_target": 128,
 203                 "expected_final_ratio": 0.5,
 204                 "expected_bulk_pool": True,
 205                 "even_pools": True,
 206                 "size": 1,
 207                 "no_scale": False,
 208                 "bulk": True,
 209             },
 210
 211             "bulk1": {
 212
 213                 "pool": 3,
 214                 "pool_name": "test3",
 215                 "pg_num_target": 32,
 216                 "capacity_ratio": 0.1,
 217                 "root_id": 0,
 218                 "expected_final_pg_target": 128,
 219                 "expected_final_ratio": 0.5,
 220                 "expected_bulk_pool": True,
 221                 "even_pools": True,
 222                 "size": 1,
 223                 "no_scale": False,
 224                 "bulk": True,
 225             },
 226
 227         }
 228
 229         root_map = {
 230
 231             0: RootMapItem(4, 400, 400),
 232             1: RootMapItem(4, 400, 400),
 233
 234         }
 235
 236         bias = 1
 237         overlapped_roots = set()
 238         self.helper_test(pools, root_map, bias, overlapped_roots)
 239
 240     def test_uneven_pools_one_meta_three_bulk(self):
 241         pools = {
 242
 243             "meta0": {
 244
 245                 "pool": 0,
 246                 "pool_name": "meta0",
 247                 "pg_num_target": 32,
 248                 "capacity_ratio": 0.1,
 249                 "root_id": 0,
 250                 "expected_final_pg_target": 32,
 251                 "expected_final_ratio": 0.1,
 252                 "expected_bulk_pool": False,
 253                 "even_pools": True,
 254                 "size": 1,
 255                 "no_scale": False,
 256                 "bulk": False,
 257             },
 258
 259             "bulk0": {
 260
 261                 "pool": 1,
 262                 "pool_name": "bulk0",
 263                 "pg_num_target": 32,
 264                 "capacity_ratio": 0.5,
 265                 "root_id": 0,
 266                 "expected_final_pg_target": 128,
 267                 "expected_final_ratio": 0.5,
 268                 "expected_bulk_pool": True,
 269                 "even_pools": False,
 270                 "size": 1,
 271                 "no_scale": False,
 272                 "bulk": True,
 273             },
 274
 275             "bulk1": {
 276
 277                 "pool": 2,
 278                 "pool_name": "bulk1",
 279                 "pg_num_target": 32,
 280                 "capacity_ratio": 0.1,
 281                 "root_id": 0,
 282                 "expected_final_pg_target": 64,
 283                 "expected_final_ratio": 0.5,
 284                 "expected_bulk_pool": True,
 285                 "even_pools": True,
 286                 "size": 1,
 287                 "no_scale": False,
 288                 "bulk": True,
 289             },
 290
 291             "bulk2": {
 292
 293                 "pool": 3,
 294                 "pool_name": "bulk2",
 295                 "pg_num_target": 32,
 296                 "capacity_ratio": 0.1,
 297                 "root_id": 0,
 298                 "expected_final_pg_target": 64,
 299                 "expected_final_ratio": 0.5,
 300                 "expected_bulk_pool": True,
 301                 "even_pools": True,
 302                 "size": 1,
 303                 "no_scale": False,
 304                 "bulk": True,
 305             },
 306
 307         }
 308
 309         root_map = {
 310
 311             0: RootMapItem(4, 400, 400),
 312             1: RootMapItem(4, 400, 400),
 313
 314         }
 315
 316         bias = 1
 317         overlapped_roots = set()
 318         self.helper_test(pools, root_map, bias, overlapped_roots)
 319
 320     def test_uneven_pools_two_meta_two_bulk(self):
 321         pools = {
 322
 323             "meta0": {
 324
 325                 "pool": 0,
 326                 "pool_name": "meta0",
 327                 "pg_num_target": 32,
 328                 "capacity_ratio": 0.1,
 329                 "root_id": 0,
 330                 "expected_final_pg_target": 32,
 331                 "expected_final_ratio": 0.1,
 332                 "expected_bulk_pool": False,
 333                 "even_pools": True,
 334                 "size": 1,
 335                 "no_scale": False,
 336                 "bulk": False,
 337             },
 338
 339             "meta1": {
 340
 341                 "pool": 1,
 342                 "pool_name": "meta1",
 343                 "pg_num_target": 32,
 344                 "capacity_ratio": 0.1,
 345                 "root_id": 0,
 346                 "expected_final_pg_target": 32,
 347                 "expected_final_ratio": 0.1,
 348                 "expected_bulk_pool": False,
 349                 "even_pools": False,
 350                 "size": 1,
 351                 "no_scale": False,
 352                 "bulk": False,
 353             },
 354
 355             "bulk0": {
 356
 357                 "pool": 2,
 358                 "pool_name": "bulk0",
 359                 "pg_num_target": 32,
 360                 "capacity_ratio": 0.5,
 361                 "root_id": 0,
 362                 "expected_final_pg_target": 128,
 363                 "expected_final_ratio": 0.5,
 364                 "expected_bulk_pool": True,
 365                 "even_pools": True,
 366                 "size": 1,
 367                 "no_scale": False,
 368                 "bulk": True,
 369             },
 370
 371             "bulk1": {
 372
 373                 "pool": 3,
 374                 "pool_name": "bulk1",
 375                 "pg_num_target": 32,
 376                 "capacity_ratio": 0.1,
 377                 "root_id": 0,
 378                 "expected_final_pg_target": 128,
 379                 "expected_final_ratio": 0.5,
 380                 "expected_bulk_pool": True,
 381                 "even_pools": True,
 382                 "size": 1,
 383                 "no_scale": False,
 384                 "bulk": True,
 385             },
 386
 387         }
 388
 389         root_map = {
 390
 391             0: RootMapItem(4, 400, 400),
 392             1: RootMapItem(4, 400, 400),
 393
 394         }
 395
 396         bias = 1
 397         overlapped_roots = set()
 398         self.helper_test(pools, root_map, bias, overlapped_roots)
 399
 400     def test_uneven_pools_with_diff_roots(self):
 401         pools = {
 402
 403             "meta0": {
 404
 405                 "pool": 0,
 406                 "pool_name": "meta0",
 407                 "pg_num_target": 32,
 408                 "capacity_ratio": 0.3,
 409                 "root_id": 0,
 410                 "expected_final_pg_target": 1024,
 411                 "expected_final_ratio": 0.3,
 412                 "expected_bulk_pool": False,
 413                 "even_pools": False,
 414                 "size": 1,
 415                 "no_scale": False,
 416                 "bulk": False,
 417             },
 418
 419             "meta1": {
 420
 421                 "pool": 1,
 422                 "pool_name": "meta1",
 423                 "pg_num_target": 32,
 424                 "capacity_ratio": 0.6,
 425                 "root_id": 1,
 426                 "expected_final_pg_target": 2048,
 427                 "expected_final_ratio": 0.6,
 428                 "expected_bulk_pool": False,
 429                 "even_pools": False,
 430                 "size": 1,
 431                 "no_scale": False,
 432                 "bulk": False,
 433             },
 434
 435             "bulk2": {
 436
 437                 "pool": 2,
 438                 "pool_name": "bulk2",
 439                 "pg_num_target": 32,
 440                 "capacity_ratio": 0.6,
 441                 "root_id": 0,
 442                 "expected_final_pg_target": 2048,
 443                 "expected_final_ratio": 0.6,
 444                 "expected_bulk_pool": True,
 445                 "even_pools": False,
 446                 "size": 1,
 447                 "no_scale": False,
 448                 "bulk": True,
 449             },
 450
 451             "bulk3": {
 452
 453                 "pool": 3,
 454                 "pool_name": "test3",
 455                 "pg_num_target": 32,
 456                 "capacity_ratio": 0.1,
 457                 "root_id": 0,
 458                 "expected_final_pg_target": 1024,
 459                 "expected_final_ratio": 1,
 460                 "expected_bulk_pool": True,
 461                 "even_pools": True,
 462                 "size": 1,
 463                 "no_scale": False,
 464                 "bulk": True,
 465             },
 466
 467             "bulk4": {
 468
 469                 "pool": 4,
 470                 "pool_name": "bulk4",
 471                 "pg_num_target": 32,
 472                 "capacity_ratio": 0.4,
 473                 "root_id": 1,
 474                 "expected_final_pg_target": 2048,
 475                 "expected_final_ratio": 1,
 476                 "expected_bulk_pool": True,
 477                 "even_pools": True,
 478                 "size": 1,
 479                 "no_scale": False,
 480                 "bulk": True,
 481             },
 482
 483         }
 484
 485         root_map = {
 486
 487             0: RootMapItem(3, 5000, 5000),
 488             1: RootMapItem(2, 5000, 5000),
 489
 490         }
 491
 492         bias = 1
 493         overlapped_roots = set()
 494         self.helper_test(pools, root_map, bias, overlapped_roots)
 495
 496     def test_even_pools_with_diff_roots(self):
 497         pools = {
 498
 499             "meta0": {
 500
 501                 "pool": 0,
 502                 "pool_name": "meta0",
 503                 "pg_num_target": 32,
 504                 "capacity_ratio": 0.4,
 505                 "root_id": 0,
 506                 "expected_final_pg_target": 2048,
 507                 "expected_final_ratio": 0.4,
 508                 "expected_bulk_pool": False,
 509                 "even_pools": False,
 510                 "size": 1,
 511                 "no_scale": False,
 512                 "bulk": False,
 513             },
 514
 515             "meta1": {
 516
 517                 "pool": 1,
 518                 "pool_name": "meta1",
 519                 "pg_num_target": 32,
 520                 "capacity_ratio": 0.6,
 521                 "root_id": 1,
 522                 "expected_final_pg_target": 2048,
 523                 "expected_final_ratio": 0.6,
 524                 "expected_bulk_pool": False,
 525                 "even_pools": False,
 526                 "size": 1,
 527                 "no_scale": False,
 528                 "bulk": False,
 529             },
 530
 531             "bulk1": {
 532
 533                 "pool": 2,
 534                 "pool_name": "bulk1",
 535                 "pg_num_target": 32,
 536                 "capacity_ratio": 0.2,
 537                 "root_id": 0,
 538                 "expected_final_pg_target": 1024,
 539                 "expected_final_ratio": 0.5,
 540                 "expected_bulk_pool": True,
 541                 "even_pools": True,
 542                 "size": 1,
 543                 "no_scale": False,
 544                 "bulk": True,
 545             },
 546
 547             "bulk2": {
 548
 549                 "pool": 3,
 550                 "pool_name": "bulk2",
 551                 "pg_num_target": 32,
 552                 "capacity_ratio": 0.1,
 553                 "root_id": 0,
 554                 "expected_final_pg_target": 1024,
 555                 "expected_final_ratio": 0.5,
 556                 "expected_bulk_pool": True,
 557                 "even_pools": True,
 558                 "size": 1,
 559                 "no_scale": False,
 560                 "bulk": True,
 561             },
 562
 563             "bulk3": {
 564
 565                 "pool": 4,
 566                 "pool_name": "bulk4",
 567                 "pg_num_target": 32,
 568                 "capacity_ratio": 0.25,
 569                 "root_id": 1,
 570                 "expected_final_pg_target": 2048,
 571                 "expected_final_ratio": 1,
 572                 "expected_bulk_pool": True,
 573                 "even_pools": True,
 574                 "size": 1,
 575                 "no_scale": False,
 576                 "bulk": True,
 577             },
 578
 579         }
 580
 581         root_map = {
 582
 583             0: RootMapItem(3, 5000, 5000),
 584             1: RootMapItem(2, 5000, 5000),
 585
 586         }
 587
 588         bias = 1
 589         overlapped_roots = set()
 590         self.helper_test(pools, root_map, bias, overlapped_roots)
 591
 592     def test_uneven_pools_with_overlapped_roots(self):
 593         pools = {
 594
 595             "test0": {
 596
 597                 "pool": 0,
 598                 "pool_name": "test0",
 599                 "pg_num_target": 32,
 600                 "capacity_ratio": 0.4,
 601                 "root_id": 0,
 602                 "expected_final_pg_target": 2048,
 603                 "expected_final_ratio": 0.4,
 604                 "even_pools": False,
 605                 "size": 1,
 606                 "no_scale": True,
 607             },
 608
 609             "test1": {
 610
 611                 "pool": 1,
 612                 "pool_name": "test1",
 613                 "pg_num_target": 32,
 614                 "capacity_ratio": 0.6,
 615                 "root_id": 1,
 616                 "expected_final_pg_target": 2048,
 617                 "expected_final_ratio": 0.6,
 618                 "even_pools": False,
 619                 "size": 1,
 620                 "no_scale": True,
 621             },
 622
 623             "test2": {
 624
 625                 "pool": 2,
 626                 "pool_name": "test2",
 627                 "pg_num_target": 32,
 628                 "capacity_ratio": 0.5,
 629                 "root_id": 0,
 630                 "expected_final_pg_target": 2048,
 631                 "expected_final_ratio": 0.5,
 632                 "even_pools": False,
 633                 "size": 1,
 634                 "no_scale": True,
 635             },
 636
 637             "test3": {
 638
 639                 "pool": 3,
 640                 "pool_name": "test3",
 641                 "pg_num_target": 32,
 642                 "capacity_ratio": 0.1,
 643                 "root_id": 0,
 644                 "expected_final_pg_target": 512,
 645                 "expected_final_ratio": 1,
 646                 "even_pools": True,
 647                 "size": 1,
 648                 "no_scale": True,
 649             },
 650
 651             "test4": {
 652
 653                 "pool": 4,
 654                 "pool_name": "test4",
 655                 "pg_num_target": 32,
 656                 "capacity_ratio": 0.4,
 657                 "root_id": 1,
 658                 "expected_final_pg_target": 2048,
 659                 "expected_final_ratio": 1,
 660                 "even_pools": True,
 661                 "size": 1,
 662                 "no_scale": True,
 663             },
 664
 665         }
 666
 667         root_map = {
 668
 669             0: RootMapItem(3, 5000, 5000),
 670             1: RootMapItem(2, 5000, 5000),
 671
 672         }
 673
 674         bias = 1
 675         overlapped_roots = {0, 1}
 676         self.helper_test(pools, root_map, bias, overlapped_roots)