ceph/src/seastar/tests/perf/perf_tests.cc

   1 /*
   2  * This file is open source software, licensed to you under the terms
   3  * of the Apache License, Version 2.0 (the "License").  See the NOTICE file
   4  * distributed with this work for additional information regarding copyright
   5  * ownership.  You may not use this file except in compliance with the License.
   6  *
   7  * You may obtain a copy of the License at
   8  *
   9  *   http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing,
  12  * software distributed under the License is distributed on an
  13  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  14  * KIND, either express or implied.  See the License for the
  15  * specific language governing permissions and limitations
  16  * under the License.
  17  */
  18 /*
  19  * Copyright (C) 2018 ScyllaDB Ltd.
  20  */
  21
  22 #include <seastar/testing/perf_tests.hh>
  23
  24 #include <fstream>
  25 #include <regex>
  26
  27 #include <boost/range.hpp>
  28 #include <boost/range/adaptors.hpp>
  29 #include <boost/range/algorithm.hpp>
  30
  31 #include <fmt/ostream.h>
  32
  33 #include <seastar/core/app-template.hh>
  34 #include <seastar/core/thread.hh>
  35 #include <seastar/json/formatter.hh>
  36 #include <seastar/util/later.hh>
  37
  38 #include <signal.h>
  39
  40 namespace perf_tests {
  41 namespace internal {
  42
  43 namespace {
  44
  45 // We need to use signal-based timer instead of seastar ones so that
  46 // tests that do not suspend can be interrupted.
  47 // This causes no overhead though since the timer is used only in a dry run.
  48 class signal_timer {
  49     std::function<void()> _fn;
  50     timer_t _timer;
  51 public:
  52     explicit signal_timer(std::function<void()> fn) : _fn(fn) {
  53         sigevent se{};
  54         se.sigev_notify = SIGEV_SIGNAL;
  55         se.sigev_signo = SIGALRM;
  56         se.sigev_value.sival_ptr = this;
  57         auto ret = timer_create(CLOCK_MONOTONIC, &se, &_timer);
  58         if (ret) {
  59             throw std::system_error(ret, std::system_category());
  60         }
  61     }
  62
  63     ~signal_timer() {
  64         timer_delete(_timer);
  65     }
  66
  67     void arm(std::chrono::steady_clock::duration dt) {
  68         time_t sec = std::chrono::duration_cast<std::chrono::seconds>(dt).count();
  69         auto nsec = std::chrono::duration_cast<std::chrono::nanoseconds>(dt).count();
  70         nsec -= std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::seconds(sec)).count();
  71
  72         itimerspec ts{};
  73         ts.it_value.tv_sec = sec;
  74         ts.it_value.tv_nsec = nsec;
  75         auto ret = timer_settime(_timer, 0, &ts, nullptr);
  76         if (ret) {
  77             throw std::system_error(ret, std::system_category());
  78         }
  79     }
  80
  81     void cancel() {
  82         itimerspec ts{};
  83         auto ret = timer_settime(_timer, 0, &ts, nullptr);
  84         if (ret) {
  85             throw std::system_error(ret, std::system_category());
  86         }
  87     }
  88 public:
  89     static void init() {
  90         struct sigaction sa{};
  91         sa.sa_sigaction = &signal_timer::signal_handler;
  92         sa.sa_flags = SA_SIGINFO;
  93         auto ret = sigaction(SIGALRM, &sa, nullptr);
  94         if (ret) {
  95             throw std::system_error(ret, std::system_category());
  96         }
  97     }
  98 private:
  99     static void signal_handler(int, siginfo_t* si, void*) {
 100         auto t = static_cast<signal_timer*>(si->si_value.sival_ptr);
 101         t->_fn();
 102     }
 103 };
 104
 105 }
 106
 107 time_measurement measure_time;
 108
 109 struct config;
 110 struct result;
 111
 112 struct result_printer {
 113     virtual ~result_printer() = default;
 114
 115     virtual void print_configuration(const config&) = 0;
 116     virtual void print_result(const result&) = 0;
 117 };
 118
 119 struct config {
 120     uint64_t single_run_iterations;
 121     std::chrono::nanoseconds single_run_duration;
 122     unsigned number_of_runs;
 123     std::vector<std::unique_ptr<result_printer>> printers;
 124 };
 125
 126 struct result {
 127     sstring test_name;
 128
 129     uint64_t total_iterations;
 130     unsigned runs;
 131
 132     double median;
 133     double mad;
 134     double min;
 135     double max;
 136 };
 137
 138 namespace {
 139
 140 struct duration {
 141     double value;
 142 };
 143
 144 static inline std::ostream& operator<<(std::ostream& os, duration d)
 145 {
 146     auto value = d.value;
 147     if (value < 1'000) {
 148         os << fmt::format("{:.3f}ns", value);
 149     } else if (value < 1'000'000) {
 150         // fmt hasn't discovered unicode yet so we are stuck with uicroseconds
 151         // See: https://github.com/fmtlib/fmt/issues/628
 152         os << fmt::format("{:.3f}us", value / 1'000);
 153     } else if (value < 1'000'000'000) {
 154         os << fmt::format("{:.3f}ms", value / 1'000'000);
 155     } else {
 156         os << fmt::format("{:.3f}s", value / 1'000'000'000);
 157     }
 158     return os;
 159 }
 160
 161 }
 162
 163 static constexpr auto format_string = "{:<40} {:>11} {:>11} {:>11} {:>11} {:>11}\n";
 164
 165 struct stdout_printer final : result_printer {
 166   virtual void print_configuration(const config& c) override {
 167     fmt::print("{:<25} {}\n{:<25} {}\n{:<25} {}\n\n",
 168                "single run iterations:", c.single_run_iterations,
 169                "single run duration:", duration { double(c.single_run_duration.count()) },
 170                "number of runs:", c.number_of_runs);
 171     fmt::print(format_string, "test", "iterations", "median", "mad", "min", "max");
 172   }
 173
 174   virtual void print_result(const result& r) override {
 175     fmt::print(format_string, r.test_name, r.total_iterations / r.runs, duration { r.median },
 176                duration { r.mad }, duration { r.min }, duration { r.max });
 177   }
 178 };
 179
 180 class json_printer final : public result_printer {
 181     std::string _output_file;
 182     std::unordered_map<std::string,
 183                        std::unordered_map<std::string,
 184                                           std::unordered_map<std::string, double>>> _root;
 185 public:
 186     explicit json_printer(const std::string& file) : _output_file(file) { }
 187
 188     ~json_printer() {
 189         std::ofstream out(_output_file);
 190         out << json::formatter::to_json(_root);
 191     }
 192
 193     virtual void print_configuration(const config&) override { }
 194
 195     virtual void print_result(const result& r) override {
 196         auto& result = _root["results"][r.test_name];
 197         result["runs"] = r.runs;
 198         result["total_iterations"] = r.total_iterations;
 199         result["median"] = r.median;
 200         result["mad"] = r.mad;
 201         result["min"] = r.min;
 202         result["max"] = r.max;
 203     }
 204 };
 205
 206 void performance_test::do_run(const config& conf)
 207 {
 208     _max_single_run_iterations = conf.single_run_iterations;
 209     if (!_max_single_run_iterations) {
 210         _max_single_run_iterations = std::numeric_limits<uint64_t>::max();
 211     }
 212
 213     signal_timer tmr([this] {
 214         _max_single_run_iterations.store(0, std::memory_order_relaxed);
 215     });
 216
 217     // dry run, estimate the number of iterations
 218     if (conf.single_run_duration.count()) {
 219         // switch out of seastar thread
 220         later().then([&] {
 221             tmr.arm(conf.single_run_duration);
 222             return do_single_run().finally([&] {
 223                 tmr.cancel();
 224                 _max_single_run_iterations = _single_run_iterations;
 225             });
 226         }).get();
 227     }
 228
 229     auto results = std::vector<double>(conf.number_of_runs);
 230     uint64_t total_iterations = 0;
 231     for (auto i = 0u; i < conf.number_of_runs; i++) {
 232         // switch out of seastar thread
 233         later().then([&] {
 234             _single_run_iterations = 0;
 235             return do_single_run().then([&] (clock_type::duration dt) {
 236                 double ns = std::chrono::duration_cast<std::chrono::nanoseconds>(dt).count();
 237                 results[i] = ns / _single_run_iterations;
 238
 239                 total_iterations += _single_run_iterations;
 240             });
 241         }).get();
 242     }
 243
 244     result r{};
 245     r.test_name = name();
 246     r.total_iterations = total_iterations;
 247     r.runs = conf.number_of_runs;
 248
 249     auto mid = conf.number_of_runs / 2;
 250
 251     boost::range::sort(results);
 252     r.median = results[mid];
 253
 254     auto diffs = boost::copy_range<std::vector<double>>(
 255         results | boost::adaptors::transformed([&] (double x) { return fabs(x - r.median); })
 256     );
 257     boost::range::sort(diffs);
 258     r.mad = diffs[mid];
 259
 260     r.min = results[0];
 261     r.max = results[results.size() - 1];
 262
 263     for (auto& rp : conf.printers) {
 264         rp->print_result(r);
 265     }
 266 }
 267
 268 void performance_test::run(const config& conf)
 269 {
 270     set_up();
 271     try {
 272         do_run(conf);
 273     } catch (...) {
 274         tear_down();
 275         throw;
 276     }
 277     tear_down();
 278 }
 279
 280 std::vector<std::unique_ptr<performance_test>>& all_tests()
 281 {
 282     static std::vector<std::unique_ptr<performance_test>> tests;
 283     return tests;
 284 }
 285
 286 void performance_test::register_test(std::unique_ptr<performance_test> test)
 287 {
 288     all_tests().emplace_back(std::move(test));
 289 }
 290
 291 void run_all(const std::vector<std::string>& tests, const config& conf)
 292 {
 293     auto can_run = [tests = boost::copy_range<std::vector<std::regex>>(tests)] (auto&& test) {
 294         auto it = boost::range::find_if(tests, [&test] (const std::regex& regex) {
 295             return std::regex_match(test->name(), regex);
 296         });
 297         return tests.empty() || it != tests.end();
 298     };
 299
 300     for (auto& rp : conf.printers) {
 301         rp->print_configuration(conf);
 302     }
 303     for (auto&& test : all_tests() | boost::adaptors::filtered(std::move(can_run))) {
 304         test->run(conf);
 305     }
 306 }
 307
 308 }
 309 }
 310
 311 int main(int ac, char** av)
 312 {
 313     using namespace perf_tests::internal;
 314     namespace bpo = boost::program_options;
 315
 316     app_template app;
 317     app.add_options()
 318         ("iterations,i", bpo::value<size_t>()->default_value(0),
 319             "number of iterations in a single run")
 320         ("duration,d", bpo::value<double>()->default_value(1),
 321             "duration of a single run in seconds")
 322         ("runs,r", bpo::value<size_t>()->default_value(5), "number of runs")
 323         ("test,t", bpo::value<std::vector<std::string>>(), "tests to execute")
 324         ("no-stdout", "do not print to stdout")
 325         ("json-output", bpo::value<std::string>(), "output json file")
 326         ("list", "list available tests")
 327         ;
 328
 329     return app.run(ac, av, [&] {
 330         return async([&] {
 331             signal_timer::init();
 332
 333             config conf;
 334             conf.single_run_iterations = app.configuration()["iterations"].as<size_t>();
 335             auto dur = std::chrono::duration<double>(app.configuration()["duration"].as<double>());
 336             conf.single_run_duration = std::chrono::duration_cast<std::chrono::nanoseconds>(dur);
 337             conf.number_of_runs = app.configuration()["runs"].as<size_t>();
 338
 339             std::vector<std::string> tests_to_run;
 340             if (app.configuration().count("test")) {
 341                 tests_to_run = app.configuration()["test"].as<std::vector<std::string>>();
 342             }
 343
 344             if (app.configuration().count("list")) {
 345                 fmt::print("available tests:\n");
 346                 for (auto&& t : all_tests()) {
 347                     fmt::print("\t{}\n", t->name());
 348                 }
 349                 return;
 350             }
 351
 352             if (!app.configuration().count("no-stdout")) {
 353                 conf.printers.emplace_back(std::make_unique<stdout_printer>());
 354             }
 355
 356             if (app.configuration().count("json-output")) {
 357                 conf.printers.emplace_back(std::make_unique<json_printer>(
 358                     app.configuration()["json-output"].as<std::string>()
 359                 ));
 360             }
 361
 362             run_all(tests_to_run, conf);
 363         });
 364     });
 365 }