ceph/src/seastar/tests/perf/perf_tests.cc

   1 /*
   2  * This file is open source software, licensed to you under the terms
   3  * of the Apache License, Version 2.0 (the "License").  See the NOTICE file
   4  * distributed with this work for additional information regarding copyright
   5  * ownership.  You may not use this file except in compliance with the License.
   6  *
   7  * You may obtain a copy of the License at
   8  *
   9  *   http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing,
  12  * software distributed under the License is distributed on an
  13  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  14  * KIND, either express or implied.  See the License for the
  15  * specific language governing permissions and limitations
  16  * under the License.
  17  */
  18 /*
  19  * Copyright (C) 2018 ScyllaDB Ltd.
  20  */
  21
  22 #include <seastar/testing/perf_tests.hh>
  23
  24 #include <fstream>
  25 #include <regex>
  26
  27 #include <boost/range.hpp>
  28 #include <boost/range/adaptors.hpp>
  29 #include <boost/range/algorithm.hpp>
  30
  31 #include <fmt/ostream.h>
  32
  33 #include <seastar/core/app-template.hh>
  34 #include <seastar/core/thread.hh>
  35 #include <seastar/json/formatter.hh>
  36
  37 namespace perf_tests {
  38 namespace internal {
  39
  40 namespace {
  41
  42 // We need to use signal-based timer instead of seastar ones so that
  43 // tests that do not suspend can be interrupted.
  44 // This causes no overhead though since the timer is used only in a dry run.
  45 class signal_timer {
  46     std::function<void()> _fn;
  47     timer_t _timer;
  48 public:
  49     explicit signal_timer(std::function<void()> fn) : _fn(fn) {
  50         sigevent se{};
  51         se.sigev_notify = SIGEV_SIGNAL;
  52         se.sigev_signo = SIGALRM;
  53         se.sigev_value.sival_ptr = this;
  54         auto ret = timer_create(CLOCK_MONOTONIC, &se, &_timer);
  55         if (ret) {
  56             throw std::system_error(ret, std::system_category());
  57         }
  58     }
  59
  60     ~signal_timer() {
  61         timer_delete(_timer);
  62     }
  63
  64     void arm(std::chrono::steady_clock::duration dt) {
  65         time_t sec = std::chrono::duration_cast<std::chrono::seconds>(dt).count();
  66         auto nsec = std::chrono::duration_cast<std::chrono::nanoseconds>(dt).count();
  67         nsec -= std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::seconds(sec)).count();
  68
  69         itimerspec ts{};
  70         ts.it_value.tv_sec = sec;
  71         ts.it_value.tv_nsec = nsec;
  72         auto ret = timer_settime(_timer, 0, &ts, nullptr);
  73         if (ret) {
  74             throw std::system_error(ret, std::system_category());
  75         }
  76     }
  77
  78     void cancel() {
  79         itimerspec ts{};
  80         auto ret = timer_settime(_timer, 0, &ts, nullptr);
  81         if (ret) {
  82             throw std::system_error(ret, std::system_category());
  83         }
  84     }
  85 public:
  86     static void init() {
  87         struct sigaction sa{};
  88         sa.sa_sigaction = &signal_timer::signal_handler;
  89         sa.sa_flags = SA_SIGINFO;
  90         auto ret = sigaction(SIGALRM, &sa, nullptr);
  91         if (ret) {
  92             throw std::system_error(ret, std::system_category());
  93         }
  94     }
  95 private:
  96     static void signal_handler(int, siginfo_t* si, void*) {
  97         auto t = static_cast<signal_timer*>(si->si_value.sival_ptr);
  98         t->_fn();
  99     }
 100 };
 101
 102 }
 103
 104 time_measurement measure_time;
 105
 106 struct config;
 107 struct result;
 108
 109 struct result_printer {
 110     virtual ~result_printer() = default;
 111
 112     virtual void print_configuration(const config&) = 0;
 113     virtual void print_result(const result&) = 0;
 114 };
 115
 116 struct config {
 117     uint64_t single_run_iterations;
 118     std::chrono::nanoseconds single_run_duration;
 119     unsigned number_of_runs;
 120     std::vector<std::unique_ptr<result_printer>> printers;
 121 };
 122
 123 struct result {
 124     sstring test_name;
 125
 126     uint64_t total_iterations;
 127     unsigned runs;
 128
 129     double median;
 130     double mad;
 131     double min;
 132     double max;
 133 };
 134
 135 namespace {
 136
 137 struct duration {
 138     double value;
 139 };
 140
 141 static inline std::ostream& operator<<(std::ostream& os, duration d)
 142 {
 143     auto value = d.value;
 144     if (value < 1'000) {
 145         os << fmt::format("{:.3f}ns", value);
 146     } else if (value < 1'000'000) {
 147         // fmt hasn't discovered unicode yet so we are stuck with uicroseconds
 148         // See: https://github.com/fmtlib/fmt/issues/628
 149         os << fmt::format("{:.3f}us", value / 1'000);
 150     } else if (value < 1'000'000'000) {
 151         os << fmt::format("{:.3f}ms", value / 1'000'000);
 152     } else {
 153         os << fmt::format("{:.3f}s", value / 1'000'000'000);
 154     }
 155     return os;
 156 }
 157
 158 }
 159
 160 static constexpr auto format_string = "{:<40} {:>11} {:>11} {:>11} {:>11} {:>11}\n";
 161
 162 struct stdout_printer final : result_printer {
 163   virtual void print_configuration(const config& c) override {
 164     fmt::print("{:<25} {}\n{:<25} {}\n{:<25} {}\n\n",
 165                "single run iterations:", c.single_run_iterations,
 166                "single run duration:", duration { double(c.single_run_duration.count()) },
 167                "number of runs:", c.number_of_runs);
 168     fmt::print(format_string, "test", "iterations", "median", "mad", "min", "max");
 169   }
 170
 171   virtual void print_result(const result& r) override {
 172     fmt::print(format_string, r.test_name, r.total_iterations / r.runs, duration { r.median },
 173                duration { r.mad }, duration { r.min }, duration { r.max });
 174   }
 175 };
 176
 177 class json_printer final : public result_printer {
 178     std::string _output_file;
 179     std::unordered_map<std::string,
 180                        std::unordered_map<std::string,
 181                                           std::unordered_map<std::string, double>>> _root;
 182 public:
 183     explicit json_printer(const std::string& file) : _output_file(file) { }
 184
 185     ~json_printer() {
 186         std::ofstream out(_output_file);
 187         out << json::formatter::to_json(_root);
 188     }
 189
 190     virtual void print_configuration(const config&) override { }
 191
 192     virtual void print_result(const result& r) override {
 193         auto& result = _root["results"][r.test_name];
 194         result["runs"] = r.runs;
 195         result["total_iterations"] = r.total_iterations;
 196         result["median"] = r.median;
 197         result["mad"] = r.mad;
 198         result["min"] = r.min;
 199         result["max"] = r.max;
 200     }
 201 };
 202
 203 void performance_test::do_run(const config& conf)
 204 {
 205     _max_single_run_iterations = conf.single_run_iterations;
 206     if (!_max_single_run_iterations) {
 207         _max_single_run_iterations = std::numeric_limits<uint64_t>::max();
 208     }
 209
 210     signal_timer tmr([this] {
 211         _max_single_run_iterations.store(0, std::memory_order_relaxed);
 212     });
 213
 214     // dry run, estimate the number of iterations
 215     if (conf.single_run_duration.count()) {
 216         // switch out of seastar thread
 217         later().then([&] {
 218             tmr.arm(conf.single_run_duration);
 219             return do_single_run().finally([&] {
 220                 tmr.cancel();
 221                 _max_single_run_iterations = _single_run_iterations;
 222             });
 223         }).get();
 224     }
 225
 226     auto results = std::vector<double>(conf.number_of_runs);
 227     uint64_t total_iterations = 0;
 228     for (auto i = 0u; i < conf.number_of_runs; i++) {
 229         // switch out of seastar thread
 230         later().then([&] {
 231             _single_run_iterations = 0;
 232             return do_single_run().then([&] (clock_type::duration dt) {
 233                 double ns = std::chrono::duration_cast<std::chrono::nanoseconds>(dt).count();
 234                 results[i] = ns / _single_run_iterations;
 235
 236                 total_iterations += _single_run_iterations;
 237             });
 238         }).get();
 239     }
 240
 241     result r{};
 242     r.test_name = name();
 243     r.total_iterations = total_iterations;
 244     r.runs = conf.number_of_runs;
 245
 246     auto mid = conf.number_of_runs / 2;
 247
 248     boost::range::sort(results);
 249     r.median = results[mid];
 250
 251     auto diffs = boost::copy_range<std::vector<double>>(
 252         results | boost::adaptors::transformed([&] (double x) { return fabs(x - r.median); })
 253     );
 254     boost::range::sort(diffs);
 255     r.mad = diffs[mid];
 256
 257     r.min = results[0];
 258     r.max = results[results.size() - 1];
 259
 260     for (auto& rp : conf.printers) {
 261         rp->print_result(r);
 262     }
 263 }
 264
 265 void performance_test::run(const config& conf)
 266 {
 267     set_up();
 268     try {
 269         do_run(conf);
 270     } catch (...) {
 271         tear_down();
 272         throw;
 273     }
 274     tear_down();
 275 }
 276
 277 std::vector<std::unique_ptr<performance_test>>& all_tests()
 278 {
 279     static std::vector<std::unique_ptr<performance_test>> tests;
 280     return tests;
 281 }
 282
 283 void performance_test::register_test(std::unique_ptr<performance_test> test)
 284 {
 285     all_tests().emplace_back(std::move(test));
 286 }
 287
 288 void run_all(const std::vector<std::string>& tests, const config& conf)
 289 {
 290     auto can_run = [tests = boost::copy_range<std::vector<std::regex>>(tests)] (auto&& test) {
 291         auto it = boost::range::find_if(tests, [&test] (const std::regex& regex) {
 292             return std::regex_match(test->name(), regex);
 293         });
 294         return tests.empty() || it != tests.end();
 295     };
 296
 297     for (auto& rp : conf.printers) {
 298         rp->print_configuration(conf);
 299     }
 300     for (auto&& test : all_tests() | boost::adaptors::filtered(std::move(can_run))) {
 301         test->run(conf);
 302     }
 303 }
 304
 305 }
 306 }
 307
 308 int main(int ac, char** av)
 309 {
 310     using namespace perf_tests::internal;
 311     namespace bpo = boost::program_options;
 312
 313     app_template app;
 314     app.add_options()
 315         ("iterations,i", bpo::value<size_t>()->default_value(0),
 316             "number of iterations in a single run")
 317         ("duration,d", bpo::value<double>()->default_value(1),
 318             "duration of a single run in seconds")
 319         ("runs,r", bpo::value<size_t>()->default_value(5), "number of runs")
 320         ("test,t", bpo::value<std::vector<std::string>>(), "tests to execute")
 321         ("no-stdout", "do not print to stdout")
 322         ("json-output", bpo::value<std::string>(), "output json file")
 323         ("list", "list available tests")
 324         ;
 325
 326     return app.run(ac, av, [&] {
 327         return async([&] {
 328             signal_timer::init();
 329
 330             config conf;
 331             conf.single_run_iterations = app.configuration()["iterations"].as<size_t>();
 332             auto dur = std::chrono::duration<double>(app.configuration()["duration"].as<double>());
 333             conf.single_run_duration = std::chrono::duration_cast<std::chrono::nanoseconds>(dur);
 334             conf.number_of_runs = app.configuration()["runs"].as<size_t>();
 335
 336             std::vector<std::string> tests_to_run;
 337             if (app.configuration().count("test")) {
 338                 tests_to_run = app.configuration()["test"].as<std::vector<std::string>>();
 339             }
 340
 341             if (app.configuration().count("list")) {
 342                 fmt::print("available tests:\n");
 343                 for (auto&& t : all_tests()) {
 344                     fmt::print("\t{}\n", t->name());
 345                 }
 346                 return;
 347             }
 348
 349             if (!app.configuration().count("no-stdout")) {
 350                 conf.printers.emplace_back(std::make_unique<stdout_printer>());
 351             }
 352
 353             if (app.configuration().count("json-output")) {
 354                 conf.printers.emplace_back(std::make_unique<json_printer>(
 355                     app.configuration()["json-output"].as<std::string>()
 356                 ));
 357             }
 358
 359             run_all(tests_to_run, conf);
 360         });
 361     });
 362 }