ceph/src/test/lazy-omap-stats/lazy_omap_stats_test.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3 /*
   4  * Ceph - scalable distributed file system
   5  *
   6  * Copyright (C) 2019 Red Hat
   7  *
   8  * This is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU Lesser General Public
  10  * License version 2.1, as published by the Free Software
  11  * Foundation.  See file COPYING.
  12  *
  13  */
  14
  15
  16 #include <algorithm>
  17 #include <boost/algorithm/string/trim.hpp>
  18 #include <boost/tokenizer.hpp>
  19 #include <boost/uuid/uuid.hpp>             // uuid class
  20 #include <boost/uuid/uuid_generators.hpp>  // generators
  21 #include <boost/uuid/uuid_io.hpp>          // streaming operators etc.
  22 #include <chrono>
  23 #include <iostream>
  24 #include <thread>
  25 #include <vector>
  26
  27 #include "common/ceph_json.h"
  28 #include "global/global_init.h"
  29 #include "include/compat.h"
  30
  31 #include "lazy_omap_stats_test.h"
  32
  33 using namespace std;
  34
  35 void LazyOmapStatsTest::init(const int argc, const char** argv)
  36 {
  37   int ret = rados.init("admin");
  38   if (ret < 0) {
  39     ret = -ret;
  40     cerr << "Failed to initialise rados! Error: " << ret << " " << strerror(ret)
  41          << endl;
  42     exit(ret);
  43   }
  44
  45   ret = rados.conf_parse_argv(argc, argv);
  46   if (ret < 0) {
  47     ret = -ret;
  48     cerr << "Failed to parse command line config options! Error: " << ret << " "
  49          << strerror(ret) << endl;
  50     exit(ret);
  51   }
  52
  53   rados.conf_parse_env(NULL);
  54   if (ret < 0) {
  55     ret = -ret;
  56     cerr << "Failed to parse environment! Error: " << ret << " "
  57          << strerror(ret) << endl;
  58     exit(ret);
  59   }
  60
  61   rados.conf_read_file(NULL);
  62   if (ret < 0) {
  63     ret = -ret;
  64     cerr << "Failed to read config file! Error: " << ret << " " << strerror(ret)
  65          << endl;
  66     exit(ret);
  67   }
  68
  69   ret = rados.connect();
  70   if (ret < 0) {
  71     ret = -ret;
  72     cerr << "Failed to connect to running cluster! Error: " << ret << " "
  73          << strerror(ret) << endl;
  74     exit(ret);
  75   }
  76
  77   string command = R"(
  78     {
  79       "prefix": "osd pool create",
  80       "pool": ")" + conf.pool_name +
  81                    R"(",
  82       "pool_type": "replicated",
  83       "size": )" + to_string(conf.replica_count) +
  84                    R"(
  85     })";
  86   librados::bufferlist inbl;
  87   string output;
  88   ret = rados.mon_command(command, inbl, nullptr, &output);
  89   if (output.length()) cout << output << endl;
  90   if (ret < 0) {
  91     ret = -ret;
  92     cerr << "Failed to create pool! Error: " << ret << " " << strerror(ret)
  93          << endl;
  94     exit(ret);
  95   }
  96
  97   ret = rados.ioctx_create(conf.pool_name.c_str(), io_ctx);
  98   if (ret < 0) {
  99     ret = -ret;
 100     cerr << "Failed to create ioctx! Error: " << ret << " " << strerror(ret)
 101          << endl;
 102     exit(ret);
 103   }
 104
 105   get_pool_id(conf.pool_name);
 106 }
 107
 108 void LazyOmapStatsTest::shutdown()
 109 {
 110   rados.pool_delete(conf.pool_name.c_str());
 111   rados.shutdown();
 112 }
 113
 114 void LazyOmapStatsTest::write_omap(const string& object_name)
 115 {
 116   librados::bufferlist bl;
 117   int ret = io_ctx.write_full(object_name, bl);
 118   if (ret < 0) {
 119     ret = -ret;
 120     cerr << "Failed to create object! Error: " << ret << " " << strerror(ret)
 121          << endl;
 122     exit(ret);
 123   }
 124   ret = io_ctx.omap_set(object_name, payload);
 125   if (ret < 0) {
 126     ret = -ret;
 127     cerr << "Failed to write omap payload! Error: " << ret << " "
 128          << strerror(ret) << endl;
 129     exit(ret);
 130   }
 131   cout << "Wrote " << conf.keys << " omap keys of " << conf.payload_size
 132        << " bytes to "
 133        << "the " << object_name << " object" << endl;
 134 }
 135
 136 const string LazyOmapStatsTest::get_name() const
 137 {
 138   boost::uuids::uuid uuid = boost::uuids::random_generator()();
 139   return boost::uuids::to_string(uuid);
 140 }
 141
 142 void LazyOmapStatsTest::write_many(uint how_many)
 143 {
 144   for (uint i = 0; i < how_many; i++) {
 145     write_omap(get_name());
 146   }
 147 }
 148
 149 void LazyOmapStatsTest::create_payload()
 150 {
 151   librados::bufferlist Lorem;
 152   Lorem.append(
 153       "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do "
 154       "eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut "
 155       "enim ad minim veniam, quis nostrud exercitation ullamco laboris "
 156       "nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in "
 157       "reprehenderit in voluptate velit esse cillum dolore eu fugiat "
 158       "nulla pariatur. Excepteur sint occaecat cupidatat non proident, "
 159       "sunt in culpa qui officia deserunt mollit anim id est laborum.");
 160   conf.payload_size = Lorem.length();
 161   conf.total_bytes = conf.keys * conf.payload_size * conf.how_many;
 162   conf.total_keys = conf.keys * conf.how_many;
 163   uint i = 0;
 164   for (i = 1; i < conf.keys + 1; ++i) {
 165     payload[get_name()] = Lorem;
 166   }
 167   cout << "Created payload with " << conf.keys << " keys of "
 168        << conf.payload_size
 169        << " bytes each. Total size in bytes = " << conf.keys * conf.payload_size
 170        << endl;
 171 }
 172
 173 void LazyOmapStatsTest::scrub()
 174 {
 175   cout << "Scrubbing" << endl;
 176
 177   cout << "Before scrub stamps:" << endl;
 178   string target_pool(conf.pool_id);
 179   target_pool.append(".");
 180   bool target_pool_found = false;
 181   map<string, string> before_scrub = get_scrub_stamps();
 182   for (auto [pg, stamp] : before_scrub) {
 183     cout << "pg = " << pg << " stamp = " << stamp << endl;
 184     if (pg.rfind(target_pool, 0) == 0) {
 185         target_pool_found = true;
 186     }
 187   }
 188   if (!target_pool_found) {
 189     cout << "Error: Target pool " << conf.pool_name << ":" << conf.pool_id
 190          << " not found!" << endl;
 191     exit(2); // ENOENT
 192   }
 193   cout << endl;
 194
 195   // Short sleep to make sure the new pool is visible
 196   sleep(5);
 197
 198   string command = R"({"prefix": "osd deep-scrub", "who": "all"})";
 199   auto output = get_output(command);
 200   cout << output << endl;
 201
 202   cout << "Waiting for deep-scrub to complete..." << endl;
 203   while (sleep(1) == 0) {
 204     cout << "Current scrub stamps:" <<  endl;
 205     bool complete = true;
 206     map<string, string> current_stamps = get_scrub_stamps();
 207     for (auto [pg, stamp] : current_stamps) {
 208         cout << "pg = " << pg << " stamp = " << stamp << endl;
 209         if (stamp == before_scrub[pg]) {
 210           // See if stamp for each pg has changed
 211           // If not, we haven't completed the deep-scrub
 212           complete = false;
 213         }
 214     }
 215     cout << endl;
 216     if (complete) {
 217       break;
 218     }
 219   }
 220   cout << "Scrubbing complete" << endl;
 221 }
 222
 223 const int LazyOmapStatsTest::find_matches(string& output, regex& reg) const
 224 {
 225   sregex_iterator cur(output.begin(), output.end(), reg);
 226   uint x = 0;
 227   for (auto end = std::sregex_iterator(); cur != end; ++cur) {
 228     cout << (*cur)[1].str() << endl;
 229     x++;
 230   }
 231   return x;
 232 }
 233
 234 const string LazyOmapStatsTest::get_output(const string command,
 235                                            const bool silent,
 236                                            const CommandTarget target)
 237 {
 238   librados::bufferlist inbl, outbl;
 239   string output;
 240   int ret = 0;
 241   if (target == CommandTarget::TARGET_MON) {
 242     ret = rados.mon_command(command, inbl, &outbl, &output);
 243   } else {
 244     ret = rados.mgr_command(command, inbl, &outbl, &output);
 245   }
 246   if (output.length() && !silent) {
 247     cout << output << endl;
 248   }
 249   if (ret < 0) {
 250     ret = -ret;
 251     cerr << "Failed to get " << command << "! Error: " << ret << " "
 252          << strerror(ret) << endl;
 253     exit(ret);
 254   }
 255   return string(outbl.c_str(), outbl.length());
 256 }
 257
 258 void LazyOmapStatsTest::get_pool_id(const string& pool)
 259 {
 260   cout << R"(Querying pool id)" << endl;
 261
 262   string command = R"({"prefix": "osd pool ls", "detail": "detail", "format": "json"})";
 263   librados::bufferlist inbl, outbl;
 264   auto output = get_output(command, false, CommandTarget::TARGET_MON);
 265   JSONParser parser;
 266   parser.parse(output.c_str(), output.size());
 267   for (const auto& pool : parser.get_array_elements()) {
 268     JSONParser parser2;
 269     parser2.parse(pool.c_str(), static_cast<int>(pool.size()));
 270     auto* obj = parser2.find_obj("pool_name");
 271     if (obj->get_data().compare(conf.pool_name) == 0) {
 272       obj = parser2.find_obj("pool_id");
 273       conf.pool_id = obj->get_data();
 274     }
 275   }
 276   if (conf.pool_id.empty()) {
 277     cout << "Failed to find pool ID for pool " << conf.pool_name << "!" << endl;
 278     exit(2);    // ENOENT
 279   } else {
 280     cout << "Found pool ID: " << conf.pool_id << endl;
 281   }
 282 }
 283
 284 map<string, string> LazyOmapStatsTest::get_scrub_stamps() {
 285   map<string, string> stamps;
 286   string command = R"({"prefix": "pg dump", "format": "json"})";
 287   auto output = get_output(command);
 288   JSONParser parser;
 289   parser.parse(output.c_str(), output.size());
 290   auto* obj = parser.find_obj("pg_map")->find_obj("pg_stats");
 291   for (auto pg = obj->find_first(); !pg.end(); ++pg) {
 292     stamps.insert({(*pg)->find_obj("pgid")->get_data(),
 293                   (*pg)->find_obj("last_deep_scrub_stamp")->get_data()});
 294   }
 295   return stamps;
 296 }
 297
 298 void LazyOmapStatsTest::check_one()
 299 {
 300   string full_output = get_output();
 301   cout << full_output << endl;
 302   regex reg(
 303       "\n"
 304       R"((PG_STAT[\s\S]*)"
 305       "\n)OSD_STAT"); // Strip OSD_STAT table so we don't find matches there
 306   smatch match;
 307   regex_search(full_output, match, reg);
 308   auto truncated_output = match[1].str();
 309   cout << truncated_output << endl;
 310   reg = regex(
 311       "\n"
 312       R"(([0-9,s].*\s)" +
 313       to_string(conf.keys) +
 314       R"(\s.*))"
 315       "\n");
 316
 317   cout << "Checking number of keys " << conf.keys << endl;
 318   cout << "Found the following lines" << endl;
 319   cout << "*************************" << endl;
 320   uint result = find_matches(truncated_output, reg);
 321   cout << "**********************" << endl;
 322   cout << "Found " << result << " matching line(s)" << endl;
 323   uint total = result;
 324
 325   reg = regex(
 326       "\n"
 327       R"(([0-9,s].*\s)" +
 328       to_string(conf.payload_size * conf.keys) +
 329       R"(\s.*))"
 330       "\n");
 331   cout << "Checking number of bytes "
 332        << conf.payload_size * conf.keys << endl;
 333   cout << "Found the following lines" << endl;
 334   cout << "*************************" << endl;
 335   result = find_matches(truncated_output, reg);
 336   cout << "**********************" << endl;
 337   cout << "Found " << result << " matching line(s)" << endl;
 338
 339   total += result;
 340   if (total != 6) {
 341     cout << "Error: Found " << total << " matches, expected 6! Exiting..."
 342          << endl;
 343     exit(22);  // EINVAL
 344   }
 345   cout << "check_one successful. Found " << total << " matches as expected"
 346        << endl;
 347 }
 348
 349 const int LazyOmapStatsTest::find_index(string& haystack, regex& needle,
 350                                         string label) const
 351 {
 352   smatch match;
 353   regex_search(haystack, match, needle);
 354   auto line = match[1].str();
 355   boost::algorithm::trim(line);
 356   boost::char_separator<char> sep{" "};
 357   boost::tokenizer<boost::char_separator<char>> tok(line, sep);
 358   vector<string> tokens(tok.begin(), tok.end());
 359   auto it = find(tokens.begin(), tokens.end(), label);
 360   if (it != tokens.end()) {
 361     return distance(tokens.begin(), it);
 362   }
 363
 364   cerr << "find_index failed to find index for " << label << endl;
 365   exit(2);    // ENOENT
 366   return -1;  // Unreachable
 367 }
 368
 369 const uint LazyOmapStatsTest::tally_column(const uint omap_bytes_index,
 370                                            const string& table,
 371                                            bool header) const
 372 {
 373   istringstream buffer(table);
 374   string line;
 375   uint64_t total = 0;
 376   while (std::getline(buffer, line)) {
 377     if (header) {
 378       header = false;
 379       continue;
 380     }
 381     boost::char_separator<char> sep{" "};
 382     boost::tokenizer<boost::char_separator<char>> tok(line, sep);
 383     vector<string> tokens(tok.begin(), tok.end());
 384     total += stoi(tokens.at(omap_bytes_index));
 385   }
 386
 387   return total;
 388 }
 389
 390 void LazyOmapStatsTest::check_column(const int index, const string& table,
 391                                      const string& type, bool header) const
 392 {
 393   uint expected;
 394   string errormsg;
 395   if (type.compare("bytes") == 0) {
 396     expected = conf.total_bytes;
 397     errormsg = "Error. Got unexpected byte count!";
 398   } else {
 399     expected = conf.total_keys;
 400     errormsg = "Error. Got unexpected key count!";
 401   }
 402   uint sum = tally_column(index, table, header);
 403   cout << "Got: " << sum << " Expected: " << expected << endl;
 404   if (sum != expected) {
 405     cout << errormsg << endl;
 406     exit(22);  // EINVAL
 407   }
 408 }
 409
 410 index_t LazyOmapStatsTest::get_indexes(regex& reg, string& output) const
 411 {
 412   index_t indexes;
 413   indexes.byte_index = find_index(output, reg, "OMAP_BYTES*");
 414   indexes.key_index = find_index(output, reg, "OMAP_KEYS*");
 415
 416   return indexes;
 417 }
 418
 419 void LazyOmapStatsTest::check_pg_dump()
 420 {
 421   cout << R"(Checking "pg dump" output)" << endl;
 422
 423   string dump_output = get_output();
 424   cout << dump_output << endl;
 425
 426   regex reg(
 427       "\n"
 428       R"((PG_STAT\s.*))"
 429       "\n");
 430   index_t indexes = get_indexes(reg, dump_output);
 431
 432   reg =
 433       "\n"
 434       R"((PG_STAT[\s\S]*))"
 435       "\n +\n[0-9]";
 436   smatch match;
 437   regex_search(dump_output, match, reg);
 438   auto table = match[1].str();
 439
 440   cout << "Checking bytes" << endl;
 441   check_column(indexes.byte_index, table, string("bytes"));
 442
 443   cout << "Checking keys" << endl;
 444   check_column(indexes.key_index, table, string("keys"));
 445
 446   cout << endl;
 447 }
 448
 449 void LazyOmapStatsTest::check_pg_dump_summary()
 450 {
 451   cout << R"(Checking "pg dump summary" output)" << endl;
 452
 453   string command = R"({"prefix": "pg dump", "dumpcontents": ["summary"]})";
 454   string dump_output = get_output(command);
 455   cout << dump_output << endl;
 456
 457   regex reg(
 458       "\n"
 459       R"((PG_STAT\s.*))"
 460       "\n");
 461   index_t indexes = get_indexes(reg, dump_output);
 462
 463   reg =
 464       "\n"
 465       R"((sum\s.*))"
 466       "\n";
 467   smatch match;
 468   regex_search(dump_output, match, reg);
 469   auto table = match[1].str();
 470
 471   cout << "Checking bytes" << endl;
 472   check_column(indexes.byte_index, table, string("bytes"), false);
 473
 474   cout << "Checking keys" << endl;
 475   check_column(indexes.key_index, table, string("keys"), false);
 476   cout << endl;
 477 }
 478
 479 void LazyOmapStatsTest::check_pg_dump_pgs()
 480 {
 481   cout << R"(Checking "pg dump pgs" output)" << endl;
 482
 483   string command = R"({"prefix": "pg dump", "dumpcontents": ["pgs"]})";
 484   string dump_output = get_output(command);
 485   cout << dump_output << endl;
 486
 487   regex reg(R"(^(PG_STAT\s.*))"
 488             "\n");
 489   index_t indexes = get_indexes(reg, dump_output);
 490
 491   reg = R"(^(PG_STAT[\s\S]*))"
 492         "\n\n";
 493   smatch match;
 494   regex_search(dump_output, match, reg);
 495   auto table = match[1].str();
 496
 497   cout << "Checking bytes" << endl;
 498   check_column(indexes.byte_index, table, string("bytes"));
 499
 500   cout << "Checking keys" << endl;
 501   check_column(indexes.key_index, table, string("keys"));
 502   cout << endl;
 503 }
 504
 505 void LazyOmapStatsTest::check_pg_dump_pools()
 506 {
 507   cout << R"(Checking "pg dump pools" output)" << endl;
 508
 509   string command = R"({"prefix": "pg dump", "dumpcontents": ["pools"]})";
 510   string dump_output = get_output(command);
 511   cout << dump_output << endl;
 512
 513   regex reg(R"(^(POOLID\s.*))"
 514             "\n");
 515   index_t indexes = get_indexes(reg, dump_output);
 516
 517   reg =
 518       "\n"
 519       R"(()" +
 520       conf.pool_id +
 521       R"(\s.*))"
 522       "\n";
 523   smatch match;
 524   regex_search(dump_output, match, reg);
 525   auto line = match[1].str();
 526
 527   cout << "Checking bytes" << endl;
 528   check_column(indexes.byte_index, line, string("bytes"), false);
 529
 530   cout << "Checking keys" << endl;
 531   check_column(indexes.key_index, line, string("keys"), false);
 532   cout << endl;
 533 }
 534
 535 void LazyOmapStatsTest::check_pg_ls()
 536 {
 537   cout << R"(Checking "pg ls" output)" << endl;
 538
 539   string command = R"({"prefix": "pg ls"})";
 540   string dump_output = get_output(command);
 541   cout << dump_output << endl;
 542
 543   regex reg(R"(^(PG\s.*))"
 544             "\n");
 545   index_t indexes = get_indexes(reg, dump_output);
 546
 547   reg = R"(^(PG[\s\S]*))"
 548         "\n\n";
 549   smatch match;
 550   regex_search(dump_output, match, reg);
 551   auto table = match[1].str();
 552
 553   cout << "Checking bytes" << endl;
 554   check_column(indexes.byte_index, table, string("bytes"));
 555
 556   cout << "Checking keys" << endl;
 557   check_column(indexes.key_index, table, string("keys"));
 558   cout << endl;
 559 }
 560
 561 void LazyOmapStatsTest::wait_for_active_clean()
 562 {
 563   cout << "Waiting for active+clean" << endl;
 564
 565   int index = -1;
 566   regex reg(
 567       "\n"
 568       R"((PG_STAT[\s\S]*))"
 569       "\n +\n[0-9]");
 570   string command = R"({"prefix": "pg dump"})";
 571   int num_not_clean;
 572   do {
 573     string dump_output = get_output(command, true);
 574     if (index == -1) {
 575       regex ireg(
 576           "\n"
 577           R"((PG_STAT\s.*))"
 578           "\n");
 579       index = find_index(dump_output, ireg, "STATE");
 580     }
 581     smatch match;
 582     regex_search(dump_output, match, reg);
 583     istringstream buffer(match[1].str());
 584     string line;
 585     num_not_clean = 0;
 586     while (std::getline(buffer, line)) {
 587       if (line.compare(0, 1, "P") == 0) continue;
 588       boost::char_separator<char> sep{" "};
 589       boost::tokenizer<boost::char_separator<char>> tok(line, sep);
 590       vector<string> tokens(tok.begin(), tok.end());
 591       num_not_clean += tokens.at(index).compare("active+clean");
 592     }
 593     cout << "." << flush;
 594     this_thread::sleep_for(chrono::milliseconds(250));
 595   } while (num_not_clean);
 596
 597   cout << endl;
 598 }
 599
 600 const int LazyOmapStatsTest::run(const int argc, const char** argv)
 601 {
 602   init(argc, argv);
 603   create_payload();
 604   wait_for_active_clean();
 605   write_omap(get_name());
 606   scrub();
 607   check_one();
 608
 609   write_many(conf.how_many - 1);  // Since we already wrote one
 610   scrub();
 611   check_pg_dump();
 612   check_pg_dump_summary();
 613   check_pg_dump_pgs();
 614   check_pg_dump_pools();
 615   check_pg_ls();
 616   cout << "All tests passed. Success!" << endl;
 617
 618   shutdown();
 619
 620   return 0;
 621 }