]>
Commit | Line | Data |
---|---|---|
1e59de90 TL |
1 | // Copyright 2015 Google Inc. All rights reserved. |
2 | // | |
3 | // Licensed under the Apache License, Version 2.0 (the "License"); | |
4 | // you may not use this file except in compliance with the License. | |
5 | // You may obtain a copy of the License at | |
6 | // | |
7 | // http://www.apache.org/licenses/LICENSE-2.0 | |
8 | // | |
9 | // Unless required by applicable law or agreed to in writing, software | |
10 | // distributed under the License is distributed on an "AS IS" BASIS, | |
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
12 | // See the License for the specific language governing permissions and | |
13 | // limitations under the License. | |
14 | ||
15 | #include "benchmark/benchmark.h" | |
16 | #include "benchmark_api_internal.h" | |
17 | #include "benchmark_runner.h" | |
18 | #include "internal_macros.h" | |
19 | ||
20 | #ifndef BENCHMARK_OS_WINDOWS | |
21 | #ifndef BENCHMARK_OS_FUCHSIA | |
22 | #include <sys/resource.h> | |
23 | #endif | |
24 | #include <sys/time.h> | |
25 | #include <unistd.h> | |
26 | #endif | |
27 | ||
28 | #include <algorithm> | |
29 | #include <atomic> | |
30 | #include <condition_variable> | |
31 | #include <cstdio> | |
32 | #include <cstdlib> | |
33 | #include <fstream> | |
34 | #include <iostream> | |
35 | #include <memory> | |
36 | #include <string> | |
37 | #include <thread> | |
38 | #include <utility> | |
39 | ||
40 | #include "check.h" | |
41 | #include "colorprint.h" | |
42 | #include "commandlineflags.h" | |
43 | #include "complexity.h" | |
44 | #include "counter.h" | |
45 | #include "internal_macros.h" | |
46 | #include "log.h" | |
47 | #include "mutex.h" | |
48 | #include "re.h" | |
49 | #include "statistics.h" | |
50 | #include "string_util.h" | |
51 | #include "thread_manager.h" | |
52 | #include "thread_timer.h" | |
53 | ||
54 | // Print a list of benchmarks. This option overrides all other options. | |
55 | DEFINE_bool(benchmark_list_tests, false); | |
56 | ||
57 | // A regular expression that specifies the set of benchmarks to execute. If | |
58 | // this flag is empty, or if this flag is the string \"all\", all benchmarks | |
59 | // linked into the binary are run. | |
60 | DEFINE_string(benchmark_filter, "."); | |
61 | ||
62 | // Minimum number of seconds we should run benchmark before results are | |
63 | // considered significant. For cpu-time based tests, this is the lower bound | |
64 | // on the total cpu time used by all threads that make up the test. For | |
65 | // real-time based tests, this is the lower bound on the elapsed time of the | |
66 | // benchmark execution, regardless of number of threads. | |
67 | DEFINE_double(benchmark_min_time, 0.5); | |
68 | ||
69 | // The number of runs of each benchmark. If greater than 1, the mean and | |
70 | // standard deviation of the runs will be reported. | |
71 | DEFINE_int32(benchmark_repetitions, 1); | |
72 | ||
73 | // Report the result of each benchmark repetitions. When 'true' is specified | |
74 | // only the mean, standard deviation, and other statistics are reported for | |
75 | // repeated benchmarks. Affects all reporters. | |
76 | DEFINE_bool(benchmark_report_aggregates_only, false); | |
77 | ||
78 | // Display the result of each benchmark repetitions. When 'true' is specified | |
79 | // only the mean, standard deviation, and other statistics are displayed for | |
80 | // repeated benchmarks. Unlike benchmark_report_aggregates_only, only affects | |
81 | // the display reporter, but *NOT* file reporter, which will still contain | |
82 | // all the output. | |
83 | DEFINE_bool(benchmark_display_aggregates_only, false); | |
84 | ||
85 | // The format to use for console output. | |
86 | // Valid values are 'console', 'json', or 'csv'. | |
87 | DEFINE_string(benchmark_format, "console"); | |
88 | ||
89 | // The format to use for file output. | |
90 | // Valid values are 'console', 'json', or 'csv'. | |
91 | DEFINE_string(benchmark_out_format, "json"); | |
92 | ||
93 | // The file to write additional output to. | |
94 | DEFINE_string(benchmark_out, ""); | |
95 | ||
96 | // Whether to use colors in the output. Valid values: | |
97 | // 'true'/'yes'/1, 'false'/'no'/0, and 'auto'. 'auto' means to use colors if | |
98 | // the output is being sent to a terminal and the TERM environment variable is | |
99 | // set to a terminal type that supports colors. | |
100 | DEFINE_string(benchmark_color, "auto"); | |
101 | ||
102 | // Whether to use tabular format when printing user counters to the console. | |
103 | // Valid values: 'true'/'yes'/1, 'false'/'no'/0. Defaults to false. | |
104 | DEFINE_bool(benchmark_counters_tabular, false); | |
105 | ||
106 | // The level of verbose logging to output | |
107 | DEFINE_int32(v, 0); | |
108 | ||
109 | namespace benchmark { | |
110 | ||
111 | namespace internal { | |
112 | ||
113 | // FIXME: wouldn't LTO mess this up? | |
114 | void UseCharPointer(char const volatile*) {} | |
115 | ||
116 | } // namespace internal | |
117 | ||
118 | State::State(IterationCount max_iters, const std::vector<int64_t>& ranges, | |
119 | int thread_i, int n_threads, internal::ThreadTimer* timer, | |
120 | internal::ThreadManager* manager) | |
121 | : total_iterations_(0), | |
122 | batch_leftover_(0), | |
123 | max_iterations(max_iters), | |
124 | started_(false), | |
125 | finished_(false), | |
126 | error_occurred_(false), | |
127 | range_(ranges), | |
128 | complexity_n_(0), | |
129 | counters(), | |
130 | thread_index(thread_i), | |
131 | threads(n_threads), | |
132 | timer_(timer), | |
133 | manager_(manager) { | |
134 | CHECK(max_iterations != 0) << "At least one iteration must be run"; | |
135 | CHECK_LT(thread_index, threads) << "thread_index must be less than threads"; | |
136 | ||
137 | // Note: The use of offsetof below is technically undefined until C++17 | |
138 | // because State is not a standard layout type. However, all compilers | |
139 | // currently provide well-defined behavior as an extension (which is | |
140 | // demonstrated since constexpr evaluation must diagnose all undefined | |
141 | // behavior). However, GCC and Clang also warn about this use of offsetof, | |
142 | // which must be suppressed. | |
143 | #if defined(__INTEL_COMPILER) | |
144 | #pragma warning push | |
145 | #pragma warning(disable : 1875) | |
146 | #elif defined(__GNUC__) | |
147 | #pragma GCC diagnostic push | |
148 | #pragma GCC diagnostic ignored "-Winvalid-offsetof" | |
149 | #endif | |
150 | // Offset tests to ensure commonly accessed data is on the first cache line. | |
151 | const int cache_line_size = 64; | |
152 | static_assert(offsetof(State, error_occurred_) <= | |
153 | (cache_line_size - sizeof(error_occurred_)), | |
154 | ""); | |
155 | #if defined(__INTEL_COMPILER) | |
156 | #pragma warning pop | |
157 | #elif defined(__GNUC__) | |
158 | #pragma GCC diagnostic pop | |
159 | #endif | |
160 | } | |
161 | ||
162 | void State::PauseTiming() { | |
163 | // Add in time accumulated so far | |
164 | CHECK(started_ && !finished_ && !error_occurred_); | |
165 | timer_->StopTimer(); | |
166 | } | |
167 | ||
168 | void State::ResumeTiming() { | |
169 | CHECK(started_ && !finished_ && !error_occurred_); | |
170 | timer_->StartTimer(); | |
171 | } | |
172 | ||
173 | void State::SkipWithError(const char* msg) { | |
174 | CHECK(msg); | |
175 | error_occurred_ = true; | |
176 | { | |
177 | MutexLock l(manager_->GetBenchmarkMutex()); | |
178 | if (manager_->results.has_error_ == false) { | |
179 | manager_->results.error_message_ = msg; | |
180 | manager_->results.has_error_ = true; | |
181 | } | |
182 | } | |
183 | total_iterations_ = 0; | |
184 | if (timer_->running()) timer_->StopTimer(); | |
185 | } | |
186 | ||
187 | void State::SetIterationTime(double seconds) { | |
188 | timer_->SetIterationTime(seconds); | |
189 | } | |
190 | ||
191 | void State::SetLabel(const char* label) { | |
192 | MutexLock l(manager_->GetBenchmarkMutex()); | |
193 | manager_->results.report_label_ = label; | |
194 | } | |
195 | ||
196 | void State::StartKeepRunning() { | |
197 | CHECK(!started_ && !finished_); | |
198 | started_ = true; | |
199 | total_iterations_ = error_occurred_ ? 0 : max_iterations; | |
200 | manager_->StartStopBarrier(); | |
201 | if (!error_occurred_) ResumeTiming(); | |
202 | } | |
203 | ||
204 | void State::FinishKeepRunning() { | |
205 | CHECK(started_ && (!finished_ || error_occurred_)); | |
206 | if (!error_occurred_) { | |
207 | PauseTiming(); | |
208 | } | |
209 | // Total iterations has now wrapped around past 0. Fix this. | |
210 | total_iterations_ = 0; | |
211 | finished_ = true; | |
212 | manager_->StartStopBarrier(); | |
213 | } | |
214 | ||
215 | namespace internal { | |
216 | namespace { | |
217 | ||
218 | void RunBenchmarks(const std::vector<BenchmarkInstance>& benchmarks, | |
219 | BenchmarkReporter* display_reporter, | |
220 | BenchmarkReporter* file_reporter) { | |
221 | // Note the file_reporter can be null. | |
222 | CHECK(display_reporter != nullptr); | |
223 | ||
224 | // Determine the width of the name field using a minimum width of 10. | |
225 | bool might_have_aggregates = FLAGS_benchmark_repetitions > 1; | |
226 | size_t name_field_width = 10; | |
227 | size_t stat_field_width = 0; | |
228 | for (const BenchmarkInstance& benchmark : benchmarks) { | |
229 | name_field_width = | |
230 | std::max<size_t>(name_field_width, benchmark.name.str().size()); | |
231 | might_have_aggregates |= benchmark.repetitions > 1; | |
232 | ||
233 | for (const auto& Stat : *benchmark.statistics) | |
234 | stat_field_width = std::max<size_t>(stat_field_width, Stat.name_.size()); | |
235 | } | |
236 | if (might_have_aggregates) name_field_width += 1 + stat_field_width; | |
237 | ||
238 | // Print header here | |
239 | BenchmarkReporter::Context context; | |
240 | context.name_field_width = name_field_width; | |
241 | ||
242 | // Keep track of running times of all instances of current benchmark | |
243 | std::vector<BenchmarkReporter::Run> complexity_reports; | |
244 | ||
245 | // We flush streams after invoking reporter methods that write to them. This | |
246 | // ensures users get timely updates even when streams are not line-buffered. | |
247 | auto flushStreams = [](BenchmarkReporter* reporter) { | |
248 | if (!reporter) return; | |
249 | std::flush(reporter->GetOutputStream()); | |
250 | std::flush(reporter->GetErrorStream()); | |
251 | }; | |
252 | ||
253 | if (display_reporter->ReportContext(context) && | |
254 | (!file_reporter || file_reporter->ReportContext(context))) { | |
255 | flushStreams(display_reporter); | |
256 | flushStreams(file_reporter); | |
257 | ||
258 | for (const auto& benchmark : benchmarks) { | |
259 | RunResults run_results = RunBenchmark(benchmark, &complexity_reports); | |
260 | ||
261 | auto report = [&run_results](BenchmarkReporter* reporter, | |
262 | bool report_aggregates_only) { | |
263 | assert(reporter); | |
264 | // If there are no aggregates, do output non-aggregates. | |
265 | report_aggregates_only &= !run_results.aggregates_only.empty(); | |
266 | if (!report_aggregates_only) | |
267 | reporter->ReportRuns(run_results.non_aggregates); | |
268 | if (!run_results.aggregates_only.empty()) | |
269 | reporter->ReportRuns(run_results.aggregates_only); | |
270 | }; | |
271 | ||
272 | report(display_reporter, run_results.display_report_aggregates_only); | |
273 | if (file_reporter) | |
274 | report(file_reporter, run_results.file_report_aggregates_only); | |
275 | ||
276 | flushStreams(display_reporter); | |
277 | flushStreams(file_reporter); | |
278 | } | |
279 | } | |
280 | display_reporter->Finalize(); | |
281 | if (file_reporter) file_reporter->Finalize(); | |
282 | flushStreams(display_reporter); | |
283 | flushStreams(file_reporter); | |
284 | } | |
285 | ||
286 | // Disable deprecated warnings temporarily because we need to reference | |
287 | // CSVReporter but don't want to trigger -Werror=-Wdeprecated-declarations | |
288 | #ifdef __GNUC__ | |
289 | #pragma GCC diagnostic push | |
290 | #pragma GCC diagnostic ignored "-Wdeprecated-declarations" | |
291 | #endif | |
292 | ||
293 | std::unique_ptr<BenchmarkReporter> CreateReporter( | |
294 | std::string const& name, ConsoleReporter::OutputOptions output_opts) { | |
295 | typedef std::unique_ptr<BenchmarkReporter> PtrType; | |
296 | if (name == "console") { | |
297 | return PtrType(new ConsoleReporter(output_opts)); | |
298 | } else if (name == "json") { | |
299 | return PtrType(new JSONReporter); | |
300 | } else if (name == "csv") { | |
301 | return PtrType(new CSVReporter); | |
302 | } else { | |
303 | std::cerr << "Unexpected format: '" << name << "'\n"; | |
304 | std::exit(1); | |
305 | } | |
306 | } | |
307 | ||
308 | #ifdef __GNUC__ | |
309 | #pragma GCC diagnostic pop | |
310 | #endif | |
311 | ||
312 | } // end namespace | |
313 | ||
314 | bool IsZero(double n) { | |
315 | return std::abs(n) < std::numeric_limits<double>::epsilon(); | |
316 | } | |
317 | ||
318 | ConsoleReporter::OutputOptions GetOutputOptions(bool force_no_color) { | |
319 | int output_opts = ConsoleReporter::OO_Defaults; | |
320 | auto is_benchmark_color = [force_no_color]() -> bool { | |
321 | if (force_no_color) { | |
322 | return false; | |
323 | } | |
324 | if (FLAGS_benchmark_color == "auto") { | |
325 | return IsColorTerminal(); | |
326 | } | |
327 | return IsTruthyFlagValue(FLAGS_benchmark_color); | |
328 | }; | |
329 | if (is_benchmark_color()) { | |
330 | output_opts |= ConsoleReporter::OO_Color; | |
331 | } else { | |
332 | output_opts &= ~ConsoleReporter::OO_Color; | |
333 | } | |
334 | if (FLAGS_benchmark_counters_tabular) { | |
335 | output_opts |= ConsoleReporter::OO_Tabular; | |
336 | } else { | |
337 | output_opts &= ~ConsoleReporter::OO_Tabular; | |
338 | } | |
339 | return static_cast<ConsoleReporter::OutputOptions>(output_opts); | |
340 | } | |
341 | ||
342 | } // end namespace internal | |
343 | ||
344 | size_t RunSpecifiedBenchmarks() { | |
345 | return RunSpecifiedBenchmarks(nullptr, nullptr); | |
346 | } | |
347 | ||
348 | size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter) { | |
349 | return RunSpecifiedBenchmarks(display_reporter, nullptr); | |
350 | } | |
351 | ||
352 | size_t RunSpecifiedBenchmarks(BenchmarkReporter* display_reporter, | |
353 | BenchmarkReporter* file_reporter) { | |
354 | std::string spec = FLAGS_benchmark_filter; | |
355 | if (spec.empty() || spec == "all") | |
356 | spec = "."; // Regexp that matches all benchmarks | |
357 | ||
358 | // Setup the reporters | |
359 | std::ofstream output_file; | |
360 | std::unique_ptr<BenchmarkReporter> default_display_reporter; | |
361 | std::unique_ptr<BenchmarkReporter> default_file_reporter; | |
362 | if (!display_reporter) { | |
363 | default_display_reporter = internal::CreateReporter( | |
364 | FLAGS_benchmark_format, internal::GetOutputOptions()); | |
365 | display_reporter = default_display_reporter.get(); | |
366 | } | |
367 | auto& Out = display_reporter->GetOutputStream(); | |
368 | auto& Err = display_reporter->GetErrorStream(); | |
369 | ||
370 | std::string const& fname = FLAGS_benchmark_out; | |
371 | if (fname.empty() && file_reporter) { | |
372 | Err << "A custom file reporter was provided but " | |
373 | "--benchmark_out=<file> was not specified." | |
374 | << std::endl; | |
375 | std::exit(1); | |
376 | } | |
377 | if (!fname.empty()) { | |
378 | output_file.open(fname); | |
379 | if (!output_file.is_open()) { | |
380 | Err << "invalid file name: '" << fname << "'" << std::endl; | |
381 | std::exit(1); | |
382 | } | |
383 | if (!file_reporter) { | |
384 | default_file_reporter = internal::CreateReporter( | |
385 | FLAGS_benchmark_out_format, ConsoleReporter::OO_None); | |
386 | file_reporter = default_file_reporter.get(); | |
387 | } | |
388 | file_reporter->SetOutputStream(&output_file); | |
389 | file_reporter->SetErrorStream(&output_file); | |
390 | } | |
391 | ||
392 | std::vector<internal::BenchmarkInstance> benchmarks; | |
393 | if (!FindBenchmarksInternal(spec, &benchmarks, &Err)) return 0; | |
394 | ||
395 | if (benchmarks.empty()) { | |
396 | Err << "Failed to match any benchmarks against regex: " << spec << "\n"; | |
397 | return 0; | |
398 | } | |
399 | ||
400 | if (FLAGS_benchmark_list_tests) { | |
401 | for (auto const& benchmark : benchmarks) | |
402 | Out << benchmark.name.str() << "\n"; | |
403 | } else { | |
404 | internal::RunBenchmarks(benchmarks, display_reporter, file_reporter); | |
405 | } | |
406 | ||
407 | return benchmarks.size(); | |
408 | } | |
409 | ||
410 | void RegisterMemoryManager(MemoryManager* manager) { | |
411 | internal::memory_manager = manager; | |
412 | } | |
413 | ||
414 | namespace internal { | |
415 | ||
416 | void PrintUsageAndExit() { | |
417 | fprintf(stdout, | |
418 | "benchmark" | |
419 | " [--benchmark_list_tests={true|false}]\n" | |
420 | " [--benchmark_filter=<regex>]\n" | |
421 | " [--benchmark_min_time=<min_time>]\n" | |
422 | " [--benchmark_repetitions=<num_repetitions>]\n" | |
423 | " [--benchmark_report_aggregates_only={true|false}]\n" | |
424 | " [--benchmark_display_aggregates_only={true|false}]\n" | |
425 | " [--benchmark_format=<console|json|csv>]\n" | |
426 | " [--benchmark_out=<filename>]\n" | |
427 | " [--benchmark_out_format=<json|console|csv>]\n" | |
428 | " [--benchmark_color={auto|true|false}]\n" | |
429 | " [--benchmark_counters_tabular={true|false}]\n" | |
430 | " [--v=<verbosity>]\n"); | |
431 | exit(0); | |
432 | } | |
433 | ||
434 | void ParseCommandLineFlags(int* argc, char** argv) { | |
435 | using namespace benchmark; | |
436 | BenchmarkReporter::Context::executable_name = | |
437 | (argc && *argc > 0) ? argv[0] : "unknown"; | |
438 | for (int i = 1; argc && i < *argc; ++i) { | |
439 | if (ParseBoolFlag(argv[i], "benchmark_list_tests", | |
440 | &FLAGS_benchmark_list_tests) || | |
441 | ParseStringFlag(argv[i], "benchmark_filter", &FLAGS_benchmark_filter) || | |
442 | ParseDoubleFlag(argv[i], "benchmark_min_time", | |
443 | &FLAGS_benchmark_min_time) || | |
444 | ParseInt32Flag(argv[i], "benchmark_repetitions", | |
445 | &FLAGS_benchmark_repetitions) || | |
446 | ParseBoolFlag(argv[i], "benchmark_report_aggregates_only", | |
447 | &FLAGS_benchmark_report_aggregates_only) || | |
448 | ParseBoolFlag(argv[i], "benchmark_display_aggregates_only", | |
449 | &FLAGS_benchmark_display_aggregates_only) || | |
450 | ParseStringFlag(argv[i], "benchmark_format", &FLAGS_benchmark_format) || | |
451 | ParseStringFlag(argv[i], "benchmark_out", &FLAGS_benchmark_out) || | |
452 | ParseStringFlag(argv[i], "benchmark_out_format", | |
453 | &FLAGS_benchmark_out_format) || | |
454 | ParseStringFlag(argv[i], "benchmark_color", &FLAGS_benchmark_color) || | |
455 | // "color_print" is the deprecated name for "benchmark_color". | |
456 | // TODO: Remove this. | |
457 | ParseStringFlag(argv[i], "color_print", &FLAGS_benchmark_color) || | |
458 | ParseBoolFlag(argv[i], "benchmark_counters_tabular", | |
459 | &FLAGS_benchmark_counters_tabular) || | |
460 | ParseInt32Flag(argv[i], "v", &FLAGS_v)) { | |
461 | for (int j = i; j != *argc - 1; ++j) argv[j] = argv[j + 1]; | |
462 | ||
463 | --(*argc); | |
464 | --i; | |
465 | } else if (IsFlag(argv[i], "help")) { | |
466 | PrintUsageAndExit(); | |
467 | } | |
468 | } | |
469 | for (auto const* flag : | |
470 | {&FLAGS_benchmark_format, &FLAGS_benchmark_out_format}) | |
471 | if (*flag != "console" && *flag != "json" && *flag != "csv") { | |
472 | PrintUsageAndExit(); | |
473 | } | |
474 | if (FLAGS_benchmark_color.empty()) { | |
475 | PrintUsageAndExit(); | |
476 | } | |
477 | } | |
478 | ||
479 | int InitializeStreams() { | |
480 | static std::ios_base::Init init; | |
481 | return 0; | |
482 | } | |
483 | ||
484 | } // end namespace internal | |
485 | ||
486 | void Initialize(int* argc, char** argv) { | |
487 | internal::ParseCommandLineFlags(argc, argv); | |
488 | internal::LogLevel() = FLAGS_v; | |
489 | } | |
490 | ||
491 | bool ReportUnrecognizedArguments(int argc, char** argv) { | |
492 | for (int i = 1; i < argc; ++i) { | |
493 | fprintf(stderr, "%s: error: unrecognized command-line flag: %s\n", argv[0], | |
494 | argv[i]); | |
495 | } | |
496 | return argc > 1; | |
497 | } | |
498 | ||
499 | } // end namespace benchmark |