ceph/src/tools/rbd/action/Perf.cc

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #include "tools/rbd/ArgumentTypes.h"
   5 #include "tools/rbd/Shell.h"
   6 #include "tools/rbd/Utils.h"
   7 #include "include/stringify.h"
   8 #include "common/ceph_context.h"
   9 #include "common/ceph_json.h"
  10 #include "common/errno.h"
  11 #include "common/Formatter.h"
  12 #include "common/TextTable.h"
  13 #include "global/global_context.h"
  14 #include <ncurses.h>
  15 #include <stdio.h>
  16 #include <unistd.h>
  17 #include <sys/time.h>
  18 #include <sys/types.h>
  19 #include <iostream>
  20 #include <vector>
  21 #include <boost/algorithm/string.hpp>
  22 #include <boost/assign.hpp>
  23 #include <boost/bimap.hpp>
  24 #include <boost/program_options.hpp>
  25 #include "json_spirit/json_spirit.h"
  26
  27 namespace rbd {
  28 namespace action {
  29 namespace perf {
  30
  31 namespace at = argument_types;
  32 namespace po = boost::program_options;
  33
  34 namespace {
  35
  36 enum class StatDescriptor {
  37   WRITE_OPS = 0,
  38   READ_OPS,
  39   WRITE_BYTES,
  40   READ_BYTES,
  41   WRITE_LATENCY,
  42   READ_LATENCY
  43 };
  44
  45 typedef boost::bimap<StatDescriptor, std::string> StatDescriptors;
  46
  47 static const StatDescriptors STAT_DESCRIPTORS =
  48   boost::assign::list_of<StatDescriptors::relation>
  49     (StatDescriptor::WRITE_OPS, "write_ops")
  50     (StatDescriptor::READ_OPS, "read_ops")
  51     (StatDescriptor::WRITE_BYTES, "write_bytes")
  52     (StatDescriptor::READ_BYTES, "read_bytes")
  53     (StatDescriptor::WRITE_LATENCY, "write_latency")
  54     (StatDescriptor::READ_LATENCY, "read_latency");
  55
  56 std::ostream& operator<<(std::ostream& os, const StatDescriptor& val) {
  57   auto it = STAT_DESCRIPTORS.left.find(val);
  58   if (it == STAT_DESCRIPTORS.left.end()) {
  59     os << "unknown (" << static_cast<int>(val) << ")";
  60   } else {
  61     os << it->second;
  62   }
  63   return os;
  64 }
  65
  66 void validate(boost::any& v, const std::vector<std::string>& values,
  67               StatDescriptor *target_type, int) {
  68   po::validators::check_first_occurrence(v);
  69   std::string s = po::validators::get_single_string(values);
  70   boost::replace_all(s, "_", " ");
  71   boost::replace_all(s, "-", "_");
  72
  73   auto it = STAT_DESCRIPTORS.right.find(s);
  74   if (it == STAT_DESCRIPTORS.right.end()) {
  75     throw po::validation_error(po::validation_error::invalid_option_value);
  76   }
  77   v = boost::any(it->second);
  78 }
  79
  80 struct ImageStat {
  81   ImageStat(const std::string& pool_name, const std::string& pool_namespace,
  82             const std::string& image_name)
  83     : pool_name(pool_name), pool_namespace(pool_namespace),
  84       image_name(image_name) {
  85     stats.resize(STAT_DESCRIPTORS.size());
  86   }
  87
  88   std::string pool_name;
  89   std::string pool_namespace;
  90   std::string image_name;
  91   std::vector<double> stats;
  92 };
  93
  94 typedef std::vector<ImageStat> ImageStats;
  95
  96 typedef std::pair<std::string, std::string> SpecPair;
  97
  98 std::string format_pool_spec(const std::string& pool,
  99                              const std::string& pool_namespace) {
 100   std::string pool_spec{pool};
 101   if (!pool_namespace.empty()) {
 102     pool_spec += "/" + pool_namespace;
 103   }
 104   return pool_spec;
 105 }
 106
 107 int query_iostats(librados::Rados& rados, const std::string& pool_spec,
 108                   StatDescriptor sort_by, ImageStats* image_stats,
 109                   std::ostream& err_os) {
 110   auto sort_by_str = STAT_DESCRIPTORS.left.find(sort_by)->second;
 111
 112   std::string cmd = R"(
 113     {
 114       "prefix": "rbd perf image stats",
 115       "pool_spec": ")" + pool_spec + R"(",
 116       "sort_by": ")" + sort_by_str + R"(",
 117       "format": "json"
 118     }")";
 119
 120   bufferlist in_bl;
 121   bufferlist out_bl;
 122   std::string outs;
 123   int r = rados.mgr_command(cmd, in_bl, &out_bl, &outs);
 124   if (r == -EOPNOTSUPP) {
 125     err_os << "rbd: 'rbd_support' mgr module is not enabled."
 126            << std::endl << std::endl
 127            << "Use 'ceph mgr module enable rbd_support' to enable."
 128            << std::endl;
 129     return r;
 130   } else if (r < 0) {
 131     err_os << "rbd: mgr command failed: " << cpp_strerror(r);
 132     if (!outs.empty()) {
 133       err_os << ": " << outs;
 134     }
 135     err_os << std::endl;
 136     return r;
 137   }
 138
 139   json_spirit::mValue json_root;
 140   if (!json_spirit::read(out_bl.to_str(), json_root)) {
 141     err_os << "rbd: error parsing perf stats" << std::endl;
 142     return -EINVAL;
 143   }
 144
 145   image_stats->clear();
 146   try {
 147     auto& root = json_root.get_obj();
 148
 149     // map JSON stat descriptor order to our internal order
 150     std::map<uint32_t, uint32_t> json_to_internal_stats;
 151     auto& json_stat_descriptors = root["stat_descriptors"].get_array();
 152     for (size_t idx = 0; idx < json_stat_descriptors.size(); ++idx) {
 153       auto it = STAT_DESCRIPTORS.right.find(
 154         json_stat_descriptors[idx].get_str());
 155       if (it == STAT_DESCRIPTORS.right.end()) {
 156         continue;
 157       }
 158       json_to_internal_stats[idx] = static_cast<uint32_t>(it->second);
 159     }
 160
 161     // cache a mapping from pool descriptors back to pool-specs
 162     std::map<std::string, SpecPair> json_to_internal_pools;
 163     auto& pool_descriptors = root["pool_descriptors"].get_obj();
 164     for (auto& pool : pool_descriptors) {
 165       auto& pool_spec = pool.second.get_str();
 166       auto pos = pool_spec.rfind("/");
 167
 168       SpecPair pair{pool_spec.substr(0, pos), ""};
 169       if (pos != std::string::npos) {
 170         pair.second = pool_spec.substr(pos + 1);
 171       }
 172
 173       json_to_internal_pools[pool.first] = pair;
 174     }
 175
 176     auto& stats = root["stats"].get_array();
 177     for (auto& stat : stats) {
 178       auto& stat_obj = stat.get_obj();
 179       if (!stat_obj.empty()) {
 180         auto& image_spec = stat_obj.begin()->first;
 181
 182         auto pos = image_spec.find("/");
 183         SpecPair pair{image_spec.substr(0, pos), ""};
 184         if (pos != std::string::npos) {
 185           pair.second = image_spec.substr(pos + 1);
 186         }
 187
 188         const auto pool_it = json_to_internal_pools.find(pair.first);
 189         if (pool_it == json_to_internal_pools.end()) {
 190           continue;
 191         }
 192
 193         image_stats->emplace_back(
 194           pool_it->second.first, pool_it->second.second, pair.second);
 195
 196         auto& image_stat = image_stats->back();
 197         auto& data = stat_obj.begin()->second.get_array();
 198         for (auto& indexes : json_to_internal_stats) {
 199           image_stat.stats[indexes.second] = data[indexes.first].get_real();
 200         }
 201       }
 202     }
 203   } catch (std::runtime_error &e) {
 204     err_os << "rbd: error parsing perf stats: " << e.what() << std::endl;
 205     return -EINVAL;
 206   }
 207
 208   return 0;
 209 }
 210
 211 void format_stat(StatDescriptor stat_descriptor, double stat,
 212                  std::ostream& os) {
 213   switch (stat_descriptor) {
 214   case StatDescriptor::WRITE_OPS:
 215   case StatDescriptor::READ_OPS:
 216     os << si_u_t(stat) << "/s";
 217     break;
 218   case StatDescriptor::WRITE_BYTES:
 219   case StatDescriptor::READ_BYTES:
 220     os << byte_u_t(stat) << "/s";
 221     break;
 222   case StatDescriptor::WRITE_LATENCY:
 223   case StatDescriptor::READ_LATENCY:
 224     os << std::fixed << std::setprecision(2);
 225     if (stat >= 1000000000) {
 226       os << (stat / 1000000000) << " s";
 227     } else if (stat >= 1000000) {
 228       os << (stat / 1000000) << " ms";
 229     } else if (stat >= 1000) {
 230       os << (stat / 1000) << " us";
 231     } else {
 232       os << stat << " ns";
 233     }
 234     break;
 235   default:
 236     ceph_assert(false);
 237     break;
 238   }
 239 }
 240
 241 } // anonymous namespace
 242
 243 namespace iostat {
 244
 245 struct Iterations {};
 246
 247 void validate(boost::any& v, const std::vector<std::string>& values,
 248               Iterations *target_type, int) {
 249   po::validators::check_first_occurrence(v);
 250   auto& s = po::validators::get_single_string(values);
 251
 252   try {
 253     auto iterations = boost::lexical_cast<uint32_t>(s);
 254     if (iterations > 0) {
 255       v = boost::any(iterations);
 256       return;
 257     }
 258   } catch (const boost::bad_lexical_cast &) {
 259   }
 260   throw po::validation_error(po::validation_error::invalid_option_value);
 261 }
 262
 263 void format(const ImageStats& image_stats, Formatter* f, bool global_search) {
 264   TextTable tbl;
 265   if (f) {
 266     f->open_array_section("images");
 267   } else {
 268     tbl.define_column("NAME", TextTable::LEFT, TextTable::LEFT);
 269     for (auto& stat : STAT_DESCRIPTORS.left) {
 270       std::string title;
 271       switch (stat.first) {
 272       case StatDescriptor::WRITE_OPS:
 273         title = "WR ";
 274         break;
 275       case StatDescriptor::READ_OPS:
 276         title = "RD ";
 277         break;
 278       case StatDescriptor::WRITE_BYTES:
 279         title = "WR_BYTES ";
 280         break;
 281       case StatDescriptor::READ_BYTES:
 282         title = "RD_BYTES ";
 283         break;
 284       case StatDescriptor::WRITE_LATENCY:
 285         title = "WR_LAT ";
 286         break;
 287       case StatDescriptor::READ_LATENCY:
 288         title = "RD_LAT ";
 289         break;
 290       default:
 291         ceph_assert(false);
 292         break;
 293       }
 294       tbl.define_column(title, TextTable::RIGHT, TextTable::RIGHT);
 295     }
 296   }
 297
 298   for (auto& image_stat : image_stats) {
 299     if (f)  {
 300       f->open_object_section("image");
 301       f->dump_string("pool", image_stat.pool_name);
 302       f->dump_string("pool_namespace", image_stat.pool_namespace);
 303       f->dump_string("image", image_stat.image_name);
 304       for (auto& pair : STAT_DESCRIPTORS.left) {
 305         f->dump_float(pair.second.c_str(),
 306                       image_stat.stats[static_cast<size_t>(pair.first)]);
 307       }
 308       f->close_section();
 309     } else {
 310       std::string name;
 311       if (global_search) {
 312         name += image_stat.pool_name + "/";
 313         if (!image_stat.pool_namespace.empty()) {
 314           name += image_stat.pool_namespace + "/";
 315         }
 316       }
 317       name += image_stat.image_name;
 318
 319       tbl << name;
 320       for (auto& pair : STAT_DESCRIPTORS.left) {
 321         std::stringstream str;
 322         format_stat(pair.first,
 323                     image_stat.stats[static_cast<size_t>(pair.first)], str);
 324         str << ' ';
 325         tbl << str.str();
 326       }
 327       tbl << TextTable::endrow;
 328     }
 329   }
 330
 331   if (f) {
 332     f->close_section();
 333     f->flush(std::cout);
 334   } else {
 335     std::cout << tbl << std::endl;
 336   }
 337 }
 338
 339 } // namespace iostat
 340
 341 namespace iotop {
 342
 343 class MainWindow {
 344 public:
 345   MainWindow(librados::Rados& rados, const std::string& pool_spec)
 346   : m_rados(rados), m_pool_spec(pool_spec) {
 347     initscr();
 348     curs_set(0);
 349     cbreak();
 350     noecho();
 351     keypad(stdscr, TRUE);
 352     nodelay(stdscr, TRUE);
 353
 354     init_columns();
 355   }
 356
 357   int run() {
 358     redraw();
 359
 360     int r = 0;
 361     std::stringstream err_str;
 362     while (true) {
 363       r = query_iostats(m_rados, m_pool_spec, m_sort_by, &m_image_stats,
 364                         err_str);
 365       if (r < 0) {
 366         break;
 367         return r;
 368       }
 369
 370       redraw();
 371       wait_for_key_or_delay();
 372
 373       int ch = getch();
 374       if (ch == 'q' || ch == 'Q') {
 375         break;
 376       } else if (ch == '<' || ch == KEY_LEFT) {
 377         auto it = STAT_DESCRIPTORS.left.find(m_sort_by);
 378         if (it != STAT_DESCRIPTORS.left.begin()) {
 379           m_sort_by = (--it)->first;
 380         }
 381       } else if (ch == '>' || ch == KEY_RIGHT) {
 382         auto it = STAT_DESCRIPTORS.left.find(m_sort_by);
 383         if (it != STAT_DESCRIPTORS.left.end() &&
 384             ++it != STAT_DESCRIPTORS.left.end()) {
 385           m_sort_by = it->first;
 386         }
 387       }
 388     }
 389
 390     endwin();
 391
 392     if (r < 0) {
 393       std::cerr << err_str.str() << std::endl;
 394     }
 395     return r;
 396   }
 397
 398 private:
 399   static const size_t STAT_COLUMN_WIDTH = 12;
 400
 401   librados::Rados& m_rados;
 402   std::string m_pool_spec;
 403
 404   ImageStats m_image_stats;
 405   StatDescriptor m_sort_by = StatDescriptor::WRITE_OPS;
 406
 407   bool m_pending_win_opened = false;
 408   WINDOW* m_pending_win = nullptr;
 409
 410   int m_height = 1;
 411   int m_width = 1;
 412
 413   std::map<StatDescriptor, std::string> m_columns;
 414
 415   void init_columns() {
 416     m_columns.clear();
 417     for (auto& pair : STAT_DESCRIPTORS.left) {
 418       std::string title;
 419       switch (pair.first) {
 420       case StatDescriptor::WRITE_OPS:
 421         title = "WRITES OPS";
 422         break;
 423       case StatDescriptor::READ_OPS:
 424         title = "READS OPS";
 425         break;
 426       case StatDescriptor::WRITE_BYTES:
 427         title = "WRITE BYTES";
 428         break;
 429       case StatDescriptor::READ_BYTES:
 430         title = "READ BYTES";
 431         break;
 432       case StatDescriptor::WRITE_LATENCY:
 433         title = "WRITE LAT";
 434         break;
 435       case StatDescriptor::READ_LATENCY:
 436         title = "READ LAT";
 437         break;
 438       default:
 439         ceph_assert(false);
 440         break;
 441       }
 442       m_columns[pair.first] = (title);
 443     }
 444   }
 445
 446   void redraw() {
 447     getmaxyx(stdscr, m_height, m_width);
 448
 449     redraw_main_window();
 450     redraw_pending_window();
 451
 452     doupdate();
 453   }
 454
 455   void redraw_main_window() {
 456     werase(stdscr);
 457     mvhline(0, 0, ' ' | A_REVERSE, m_width);
 458
 459     // print header for all metrics
 460     int remaining_cols = m_width;
 461     std::stringstream str;
 462     for (auto& pair : m_columns) {
 463       int attr = A_REVERSE;
 464       std::string title;
 465       if (pair.first == m_sort_by) {
 466         title += '>';
 467         attr |= A_BOLD;
 468       } else {
 469         title += ' ';
 470       }
 471       title += pair.second;
 472
 473       str.str("");
 474       str << std::right << std::setfill(' ')
 475          << std::setw(STAT_COLUMN_WIDTH)
 476          << title << ' ';
 477
 478       attrset(attr);
 479       addstr(str.str().c_str());
 480       remaining_cols -= title.size();
 481     }
 482
 483     attrset(A_REVERSE);
 484     addstr("IMAGE");
 485     attrset(A_NORMAL);
 486
 487     // print each image (one per line)
 488     int row = 1;
 489     int remaining_lines = m_height - 1;
 490     for (auto& image_stat : m_image_stats) {
 491       if (remaining_lines <= 0) {
 492         break;
 493       }
 494       --remaining_lines;
 495
 496       move(row++, 0);
 497       for (auto& pair : m_columns) {
 498         str.str("");
 499         format_stat(pair.first,
 500                     image_stat.stats[static_cast<size_t>(pair.first)], str);
 501         auto value = str.str().substr(0, STAT_COLUMN_WIDTH);
 502
 503         str.str("");
 504         str << std::right << std::setfill(' ')
 505             << std::setw(STAT_COLUMN_WIDTH)
 506             << value << ' ';
 507         addstr(str.str().c_str());
 508       }
 509
 510       std::string image;
 511       if (m_pool_spec.empty()) {
 512         image = format_pool_spec(image_stat.pool_name,
 513                                  image_stat.pool_namespace) + "/";
 514       }
 515       image += image_stat.image_name;
 516       addstr(image.substr(0, remaining_cols).c_str());
 517     }
 518
 519     wnoutrefresh(stdscr);
 520   }
 521
 522   void redraw_pending_window() {
 523     // draw a "please by patient" window while waiting
 524     const char* msg = "Waiting for initial stats";
 525     int height = 5;
 526     int width = strlen(msg) + 4;;
 527     int starty = (m_height - height) / 2;
 528     int startx = (m_width - width) / 2;
 529
 530     if (m_image_stats.empty() && !m_pending_win_opened) {
 531       m_pending_win_opened = true;
 532       m_pending_win = newwin(height, width, starty, startx);
 533     }
 534
 535     if (m_pending_win != nullptr) {
 536       if (m_image_stats.empty()) {
 537         box(m_pending_win, 0 , 0);
 538         mvwaddstr(m_pending_win, 2, 2, msg);
 539         wnoutrefresh(m_pending_win);
 540       } else {
 541         delwin(m_pending_win);
 542         m_pending_win = nullptr;
 543       }
 544     }
 545   }
 546
 547   void wait_for_key_or_delay() {
 548     fd_set fds;
 549     FD_ZERO(&fds);
 550     FD_SET(STDIN_FILENO, &fds);
 551
 552     // no point to refreshing faster than the stats period
 553     struct timeval tval;
 554     tval.tv_sec = std::min<uint32_t>(
 555       10, g_conf().get_val<int64_t>("mgr_stats_period"));
 556     tval.tv_usec = 0;
 557
 558     select(STDIN_FILENO + 1, &fds, NULL, NULL, &tval);
 559   }
 560 };
 561
 562 } // namespace iotop
 563
 564
 565 void get_arguments_iostat(po::options_description *positional,
 566                           po::options_description *options) {
 567   at::add_pool_options(positional, options, true);
 568   options->add_options()
 569     ("iterations", po::value<iostat::Iterations>(),
 570      "iterations of metric collection [> 0]")
 571     ("sort-by", po::value<StatDescriptor>()->default_value(StatDescriptor::WRITE_OPS),
 572      "sort-by IO metric "
 573      "(write-ops, read-ops, write-bytes, read-bytes, write-latency, read-latency) "
 574      "[default: write-ops]");
 575   at::add_format_options(options);
 576 }
 577
 578 int execute_iostat(const po::variables_map &vm,
 579                    const std::vector<std::string> &ceph_global_init_args) {
 580   std::string pool;
 581   std::string pool_namespace;
 582   size_t arg_index = 0;
 583   int r = utils::get_pool_and_namespace_names(vm, false, false, &pool,
 584                                               &pool_namespace, &arg_index);
 585   if (r < 0) {
 586     return r;
 587   }
 588
 589   uint32_t iterations = 0;
 590   if (vm.count("iterations")) {
 591     iterations = vm["iterations"].as<uint32_t>();
 592   }
 593   auto sort_by = vm["sort-by"].as<StatDescriptor>();
 594
 595   at::Format::Formatter formatter;
 596   r = utils::get_formatter(vm, &formatter);
 597   if (r < 0) {
 598     return r;
 599   }
 600
 601   auto f = formatter.get();
 602   if (iterations > 1 && f != nullptr) {
 603     std::cerr << "rbd: specifing iterations is not valid with formatted output"
 604               << std::endl;
 605     return -EINVAL;
 606   }
 607
 608   librados::Rados rados;
 609   r = utils::init_rados(&rados);
 610   if (r < 0) {
 611     return r;
 612   }
 613
 614   r = rados.wait_for_latest_osdmap();
 615   if (r < 0) {
 616     std::cerr << "rbd: failed to retrieve OSD map" << std::endl;
 617     return r;
 618   }
 619
 620   std::string pool_spec = format_pool_spec(pool, pool_namespace);
 621
 622   // no point to refreshing faster than the stats period
 623   auto delay = std::min<uint32_t>(10, g_conf().get_val<int64_t>("mgr_stats_period"));
 624
 625   ImageStats image_stats;
 626   uint32_t count = 0;
 627   bool printed_notice = false;
 628   while (count++ < iterations || iterations == 0) {
 629     r = query_iostats(rados, pool_spec, sort_by, &image_stats, std::cerr);
 630     if (r < 0) {
 631       return r;
 632     }
 633
 634     if (count == 1 && image_stats.empty()) {
 635       count = 0;
 636       if (!printed_notice) {
 637         std::cerr << "rbd: waiting for initial image stats"
 638                   << std::endl << std::endl;;
 639         printed_notice = true;
 640       }
 641     } else {
 642       iostat::format(image_stats, f, pool_spec.empty());
 643       if (f != nullptr) {
 644         break;
 645       }
 646     }
 647
 648     sleep(delay);
 649   }
 650
 651   return 0;
 652 }
 653
 654 void get_arguments_iotop(po::options_description *positional,
 655                          po::options_description *options) {
 656   at::add_pool_options(positional, options, true);
 657 }
 658
 659 int execute_iotop(const po::variables_map &vm,
 660                   const std::vector<std::string> &ceph_global_init_args) {
 661   std::string pool;
 662   std::string pool_namespace;
 663   size_t arg_index = 0;
 664   int r = utils::get_pool_and_namespace_names(vm, false, false, &pool,
 665                                               &pool_namespace, &arg_index);
 666   if (r < 0) {
 667     return r;
 668   }
 669
 670   librados::Rados rados;
 671   r = utils::init_rados(&rados);
 672   if (r < 0) {
 673     return r;
 674   }
 675
 676   r = rados.wait_for_latest_osdmap();
 677   if (r < 0) {
 678     std::cerr << "rbd: failed to retrieve OSD map" << std::endl;
 679     return r;
 680   }
 681
 682   iotop::MainWindow mainWindow(rados, format_pool_spec(pool, pool_namespace));
 683   r = mainWindow.run();
 684   if (r < 0) {
 685     return r;
 686   }
 687
 688   return 0;
 689 }
 690
 691 Shell::Action stat_action(
 692   {"perf", "image", "iostat"}, {}, "Display image IO statistics.", "",
 693   &get_arguments_iostat, &execute_iostat);
 694 Shell::Action top_action(
 695   {"perf", "image", "iotop"}, {}, "Display a top-like IO monitor.", "",
 696   &get_arguments_iotop, &execute_iotop);
 697
 698 } // namespace perf
 699 } // namespace action
 700 } // namespace rbd