]>
Commit | Line | Data |
---|---|---|
1e59de90 TL |
1 | #include <cstdio> |
2 | #include <cstring> | |
3 | #include <fstream> | |
4 | #include <iostream> | |
5 | #include <map> | |
6 | #include <memory> | |
7 | #include <random> | |
8 | #include <sstream> | |
9 | #include <streambuf> | |
10 | ||
11 | #include "../src/benchmark_api_internal.h" | |
12 | #include "../src/check.h" // NOTE: check.h is for internal use only! | |
13 | #include "../src/re.h" // NOTE: re.h is for internal use only | |
14 | #include "output_test.h" | |
15 | ||
16 | // ========================================================================= // | |
17 | // ------------------------------ Internals -------------------------------- // | |
18 | // ========================================================================= // | |
19 | namespace internal { | |
20 | namespace { | |
21 | ||
22 | using TestCaseList = std::vector<TestCase>; | |
23 | ||
24 | // Use a vector because the order elements are added matters during iteration. | |
25 | // std::map/unordered_map don't guarantee that. | |
26 | // For example: | |
27 | // SetSubstitutions({{"%HelloWorld", "Hello"}, {"%Hello", "Hi"}}); | |
28 | // Substitute("%HelloWorld") // Always expands to Hello. | |
29 | using SubMap = std::vector<std::pair<std::string, std::string>>; | |
30 | ||
31 | TestCaseList& GetTestCaseList(TestCaseID ID) { | |
32 | // Uses function-local statics to ensure initialization occurs | |
33 | // before first use. | |
34 | static TestCaseList lists[TC_NumID]; | |
35 | return lists[ID]; | |
36 | } | |
37 | ||
38 | SubMap& GetSubstitutions() { | |
39 | // Don't use 'dec_re' from header because it may not yet be initialized. | |
40 | // clang-format off | |
41 | static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"; | |
42 | static std::string time_re = "([0-9]+[.])?[0-9]+"; | |
43 | static SubMap map = { | |
44 | {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"}, | |
45 | // human-readable float | |
46 | {"%hrfloat", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?[kMGTPEZYmunpfazy]?"}, | |
47 | {"%int", "[ ]*[0-9]+"}, | |
48 | {" %s ", "[ ]+"}, | |
49 | {"%time", "[ ]*" + time_re + "[ ]+ns"}, | |
50 | {"%console_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns [ ]*[0-9]+"}, | |
51 | {"%console_us_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us [ ]*[0-9]+"}, | |
52 | {"%console_ms_report", "[ ]*" + time_re + "[ ]+ms [ ]*" + time_re + "[ ]+ms [ ]*[0-9]+"}, | |
53 | {"%console_s_report", "[ ]*" + time_re + "[ ]+s [ ]*" + time_re + "[ ]+s [ ]*[0-9]+"}, | |
54 | {"%console_time_only_report", "[ ]*" + time_re + "[ ]+ns [ ]*" + time_re + "[ ]+ns"}, | |
55 | {"%console_us_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us [ ]*[0-9]+"}, | |
56 | {"%console_us_time_only_report", "[ ]*" + time_re + "[ ]+us [ ]*" + time_re + "[ ]+us"}, | |
57 | {"%csv_header", | |
58 | "name,iterations,real_time,cpu_time,time_unit,bytes_per_second," | |
59 | "items_per_second,label,error_occurred,error_message"}, | |
60 | {"%csv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,,,"}, | |
61 | {"%csv_us_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",us,,,,,"}, | |
62 | {"%csv_ms_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ms,,,,,"}, | |
63 | {"%csv_s_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",s,,,,,"}, | |
64 | {"%csv_bytes_report", | |
65 | "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + ",,,,"}, | |
66 | {"%csv_items_report", | |
67 | "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,," + safe_dec_re + ",,,"}, | |
68 | {"%csv_bytes_items_report", | |
69 | "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + | |
70 | "," + safe_dec_re + ",,,"}, | |
71 | {"%csv_label_report_begin", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,"}, | |
72 | {"%csv_label_report_end", ",,"}}; | |
73 | // clang-format on | |
74 | return map; | |
75 | } | |
76 | ||
77 | std::string PerformSubstitutions(std::string source) { | |
78 | SubMap const& subs = GetSubstitutions(); | |
79 | using SizeT = std::string::size_type; | |
80 | for (auto const& KV : subs) { | |
81 | SizeT pos; | |
82 | SizeT next_start = 0; | |
83 | while ((pos = source.find(KV.first, next_start)) != std::string::npos) { | |
84 | next_start = pos + KV.second.size(); | |
85 | source.replace(pos, KV.first.size(), KV.second); | |
86 | } | |
87 | } | |
88 | return source; | |
89 | } | |
90 | ||
91 | void CheckCase(std::stringstream& remaining_output, TestCase const& TC, | |
92 | TestCaseList const& not_checks) { | |
93 | std::string first_line; | |
94 | bool on_first = true; | |
95 | std::string line; | |
96 | while (remaining_output.eof() == false) { | |
97 | CHECK(remaining_output.good()); | |
98 | std::getline(remaining_output, line); | |
99 | if (on_first) { | |
100 | first_line = line; | |
101 | on_first = false; | |
102 | } | |
103 | for (const auto& NC : not_checks) { | |
104 | CHECK(!NC.regex->Match(line)) | |
105 | << "Unexpected match for line \"" << line << "\" for MR_Not regex \"" | |
106 | << NC.regex_str << "\"" | |
107 | << "\n actual regex string \"" << TC.substituted_regex << "\"" | |
108 | << "\n started matching near: " << first_line; | |
109 | } | |
110 | if (TC.regex->Match(line)) return; | |
111 | CHECK(TC.match_rule != MR_Next) | |
112 | << "Expected line \"" << line << "\" to match regex \"" << TC.regex_str | |
113 | << "\"" | |
114 | << "\n actual regex string \"" << TC.substituted_regex << "\"" | |
115 | << "\n started matching near: " << first_line; | |
116 | } | |
117 | CHECK(remaining_output.eof() == false) | |
118 | << "End of output reached before match for regex \"" << TC.regex_str | |
119 | << "\" was found" | |
120 | << "\n actual regex string \"" << TC.substituted_regex << "\"" | |
121 | << "\n started matching near: " << first_line; | |
122 | } | |
123 | ||
124 | void CheckCases(TestCaseList const& checks, std::stringstream& output) { | |
125 | std::vector<TestCase> not_checks; | |
126 | for (size_t i = 0; i < checks.size(); ++i) { | |
127 | const auto& TC = checks[i]; | |
128 | if (TC.match_rule == MR_Not) { | |
129 | not_checks.push_back(TC); | |
130 | continue; | |
131 | } | |
132 | CheckCase(output, TC, not_checks); | |
133 | not_checks.clear(); | |
134 | } | |
135 | } | |
136 | ||
137 | class TestReporter : public benchmark::BenchmarkReporter { | |
138 | public: | |
139 | TestReporter(std::vector<benchmark::BenchmarkReporter*> reps) | |
140 | : reporters_(reps) {} | |
141 | ||
142 | virtual bool ReportContext(const Context& context) { | |
143 | bool last_ret = false; | |
144 | bool first = true; | |
145 | for (auto rep : reporters_) { | |
146 | bool new_ret = rep->ReportContext(context); | |
147 | CHECK(first || new_ret == last_ret) | |
148 | << "Reports return different values for ReportContext"; | |
149 | first = false; | |
150 | last_ret = new_ret; | |
151 | } | |
152 | (void)first; | |
153 | return last_ret; | |
154 | } | |
155 | ||
156 | void ReportRuns(const std::vector<Run>& report) { | |
157 | for (auto rep : reporters_) rep->ReportRuns(report); | |
158 | } | |
159 | void Finalize() { | |
160 | for (auto rep : reporters_) rep->Finalize(); | |
161 | } | |
162 | ||
163 | private: | |
164 | std::vector<benchmark::BenchmarkReporter*> reporters_; | |
165 | }; | |
166 | } // namespace | |
167 | ||
168 | } // end namespace internal | |
169 | ||
170 | // ========================================================================= // | |
171 | // -------------------------- Results checking ----------------------------- // | |
172 | // ========================================================================= // | |
173 | ||
174 | namespace internal { | |
175 | ||
176 | // Utility class to manage subscribers for checking benchmark results. | |
177 | // It works by parsing the CSV output to read the results. | |
178 | class ResultsChecker { | |
179 | public: | |
180 | struct PatternAndFn : public TestCase { // reusing TestCase for its regexes | |
181 | PatternAndFn(const std::string& rx, ResultsCheckFn fn_) | |
182 | : TestCase(rx), fn(fn_) {} | |
183 | ResultsCheckFn fn; | |
184 | }; | |
185 | ||
186 | std::vector<PatternAndFn> check_patterns; | |
187 | std::vector<Results> results; | |
188 | std::vector<std::string> field_names; | |
189 | ||
190 | void Add(const std::string& entry_pattern, ResultsCheckFn fn); | |
191 | ||
192 | void CheckResults(std::stringstream& output); | |
193 | ||
194 | private: | |
195 | void SetHeader_(const std::string& csv_header); | |
196 | void SetValues_(const std::string& entry_csv_line); | |
197 | ||
198 | std::vector<std::string> SplitCsv_(const std::string& line); | |
199 | }; | |
200 | ||
201 | // store the static ResultsChecker in a function to prevent initialization | |
202 | // order problems | |
203 | ResultsChecker& GetResultsChecker() { | |
204 | static ResultsChecker rc; | |
205 | return rc; | |
206 | } | |
207 | ||
208 | // add a results checker for a benchmark | |
209 | void ResultsChecker::Add(const std::string& entry_pattern, ResultsCheckFn fn) { | |
210 | check_patterns.emplace_back(entry_pattern, fn); | |
211 | } | |
212 | ||
213 | // check the results of all subscribed benchmarks | |
214 | void ResultsChecker::CheckResults(std::stringstream& output) { | |
215 | // first reset the stream to the start | |
216 | { | |
217 | auto start = std::stringstream::pos_type(0); | |
218 | // clear before calling tellg() | |
219 | output.clear(); | |
220 | // seek to zero only when needed | |
221 | if (output.tellg() > start) output.seekg(start); | |
222 | // and just in case | |
223 | output.clear(); | |
224 | } | |
225 | // now go over every line and publish it to the ResultsChecker | |
226 | std::string line; | |
227 | bool on_first = true; | |
228 | while (output.eof() == false) { | |
229 | CHECK(output.good()); | |
230 | std::getline(output, line); | |
231 | if (on_first) { | |
232 | SetHeader_(line); // this is important | |
233 | on_first = false; | |
234 | continue; | |
235 | } | |
236 | SetValues_(line); | |
237 | } | |
238 | // finally we can call the subscribed check functions | |
239 | for (const auto& p : check_patterns) { | |
240 | VLOG(2) << "--------------------------------\n"; | |
241 | VLOG(2) << "checking for benchmarks matching " << p.regex_str << "...\n"; | |
242 | for (const auto& r : results) { | |
243 | if (!p.regex->Match(r.name)) { | |
244 | VLOG(2) << p.regex_str << " is not matched by " << r.name << "\n"; | |
245 | continue; | |
246 | } else { | |
247 | VLOG(2) << p.regex_str << " is matched by " << r.name << "\n"; | |
248 | } | |
249 | VLOG(1) << "Checking results of " << r.name << ": ... \n"; | |
250 | p.fn(r); | |
251 | VLOG(1) << "Checking results of " << r.name << ": OK.\n"; | |
252 | } | |
253 | } | |
254 | } | |
255 | ||
256 | // prepare for the names in this header | |
257 | void ResultsChecker::SetHeader_(const std::string& csv_header) { | |
258 | field_names = SplitCsv_(csv_header); | |
259 | } | |
260 | ||
261 | // set the values for a benchmark | |
262 | void ResultsChecker::SetValues_(const std::string& entry_csv_line) { | |
263 | if (entry_csv_line.empty()) return; // some lines are empty | |
264 | CHECK(!field_names.empty()); | |
265 | auto vals = SplitCsv_(entry_csv_line); | |
266 | CHECK_EQ(vals.size(), field_names.size()); | |
267 | results.emplace_back(vals[0]); // vals[0] is the benchmark name | |
268 | auto& entry = results.back(); | |
269 | for (size_t i = 1, e = vals.size(); i < e; ++i) { | |
270 | entry.values[field_names[i]] = vals[i]; | |
271 | } | |
272 | } | |
273 | ||
274 | // a quick'n'dirty csv splitter (eliminating quotes) | |
275 | std::vector<std::string> ResultsChecker::SplitCsv_(const std::string& line) { | |
276 | std::vector<std::string> out; | |
277 | if (line.empty()) return out; | |
278 | if (!field_names.empty()) out.reserve(field_names.size()); | |
279 | size_t prev = 0, pos = line.find_first_of(','), curr = pos; | |
280 | while (pos != line.npos) { | |
281 | CHECK(curr > 0); | |
282 | if (line[prev] == '"') ++prev; | |
283 | if (line[curr - 1] == '"') --curr; | |
284 | out.push_back(line.substr(prev, curr - prev)); | |
285 | prev = pos + 1; | |
286 | pos = line.find_first_of(',', pos + 1); | |
287 | curr = pos; | |
288 | } | |
289 | curr = line.size(); | |
290 | if (line[prev] == '"') ++prev; | |
291 | if (line[curr - 1] == '"') --curr; | |
292 | out.push_back(line.substr(prev, curr - prev)); | |
293 | return out; | |
294 | } | |
295 | ||
296 | } // end namespace internal | |
297 | ||
298 | size_t AddChecker(const char* bm_name, ResultsCheckFn fn) { | |
299 | auto& rc = internal::GetResultsChecker(); | |
300 | rc.Add(bm_name, fn); | |
301 | return rc.results.size(); | |
302 | } | |
303 | ||
304 | int Results::NumThreads() const { | |
305 | auto pos = name.find("/threads:"); | |
306 | if (pos == name.npos) return 1; | |
307 | auto end = name.find('/', pos + 9); | |
308 | std::stringstream ss; | |
309 | ss << name.substr(pos + 9, end); | |
310 | int num = 1; | |
311 | ss >> num; | |
312 | CHECK(!ss.fail()); | |
313 | return num; | |
314 | } | |
315 | ||
316 | double Results::NumIterations() const { | |
317 | return GetAs<double>("iterations"); | |
318 | } | |
319 | ||
320 | double Results::GetTime(BenchmarkTime which) const { | |
321 | CHECK(which == kCpuTime || which == kRealTime); | |
322 | const char* which_str = which == kCpuTime ? "cpu_time" : "real_time"; | |
323 | double val = GetAs<double>(which_str); | |
324 | auto unit = Get("time_unit"); | |
325 | CHECK(unit); | |
326 | if (*unit == "ns") { | |
327 | return val * 1.e-9; | |
328 | } else if (*unit == "us") { | |
329 | return val * 1.e-6; | |
330 | } else if (*unit == "ms") { | |
331 | return val * 1.e-3; | |
332 | } else if (*unit == "s") { | |
333 | return val; | |
334 | } else { | |
335 | CHECK(1 == 0) << "unknown time unit: " << *unit; | |
336 | return 0; | |
337 | } | |
338 | } | |
339 | ||
340 | // ========================================================================= // | |
341 | // -------------------------- Public API Definitions------------------------ // | |
342 | // ========================================================================= // | |
343 | ||
344 | TestCase::TestCase(std::string re, int rule) | |
345 | : regex_str(std::move(re)), | |
346 | match_rule(rule), | |
347 | substituted_regex(internal::PerformSubstitutions(regex_str)), | |
348 | regex(std::make_shared<benchmark::Regex>()) { | |
349 | std::string err_str; | |
350 | regex->Init(substituted_regex, &err_str); | |
351 | CHECK(err_str.empty()) << "Could not construct regex \"" << substituted_regex | |
352 | << "\"" | |
353 | << "\n originally \"" << regex_str << "\"" | |
354 | << "\n got error: " << err_str; | |
355 | } | |
356 | ||
357 | int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) { | |
358 | auto& L = internal::GetTestCaseList(ID); | |
359 | L.insert(L.end(), il); | |
360 | return 0; | |
361 | } | |
362 | ||
363 | int SetSubstitutions( | |
364 | std::initializer_list<std::pair<std::string, std::string>> il) { | |
365 | auto& subs = internal::GetSubstitutions(); | |
366 | for (auto KV : il) { | |
367 | bool exists = false; | |
368 | KV.second = internal::PerformSubstitutions(KV.second); | |
369 | for (auto& EKV : subs) { | |
370 | if (EKV.first == KV.first) { | |
371 | EKV.second = std::move(KV.second); | |
372 | exists = true; | |
373 | break; | |
374 | } | |
375 | } | |
376 | if (!exists) subs.push_back(std::move(KV)); | |
377 | } | |
378 | return 0; | |
379 | } | |
380 | ||
381 | // Disable deprecated warnings temporarily because we need to reference | |
382 | // CSVReporter but don't want to trigger -Werror=-Wdeprecated-declarations | |
383 | #ifdef __GNUC__ | |
384 | #pragma GCC diagnostic push | |
385 | #pragma GCC diagnostic ignored "-Wdeprecated-declarations" | |
386 | #endif | |
387 | void RunOutputTests(int argc, char* argv[]) { | |
388 | using internal::GetTestCaseList; | |
389 | benchmark::Initialize(&argc, argv); | |
390 | auto options = benchmark::internal::GetOutputOptions(/*force_no_color*/ true); | |
391 | benchmark::ConsoleReporter CR(options); | |
392 | benchmark::JSONReporter JR; | |
393 | benchmark::CSVReporter CSVR; | |
394 | struct ReporterTest { | |
395 | const char* name; | |
396 | std::vector<TestCase>& output_cases; | |
397 | std::vector<TestCase>& error_cases; | |
398 | benchmark::BenchmarkReporter& reporter; | |
399 | std::stringstream out_stream; | |
400 | std::stringstream err_stream; | |
401 | ||
402 | ReporterTest(const char* n, std::vector<TestCase>& out_tc, | |
403 | std::vector<TestCase>& err_tc, | |
404 | benchmark::BenchmarkReporter& br) | |
405 | : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) { | |
406 | reporter.SetOutputStream(&out_stream); | |
407 | reporter.SetErrorStream(&err_stream); | |
408 | } | |
409 | } TestCases[] = { | |
410 | {"ConsoleReporter", GetTestCaseList(TC_ConsoleOut), | |
411 | GetTestCaseList(TC_ConsoleErr), CR}, | |
412 | {"JSONReporter", GetTestCaseList(TC_JSONOut), GetTestCaseList(TC_JSONErr), | |
413 | JR}, | |
414 | {"CSVReporter", GetTestCaseList(TC_CSVOut), GetTestCaseList(TC_CSVErr), | |
415 | CSVR}, | |
416 | }; | |
417 | ||
418 | // Create the test reporter and run the benchmarks. | |
419 | std::cout << "Running benchmarks...\n"; | |
420 | internal::TestReporter test_rep({&CR, &JR, &CSVR}); | |
421 | benchmark::RunSpecifiedBenchmarks(&test_rep); | |
422 | ||
423 | for (auto& rep_test : TestCases) { | |
424 | std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n"; | |
425 | std::string banner(msg.size() - 1, '-'); | |
426 | std::cout << banner << msg << banner << "\n"; | |
427 | ||
428 | std::cerr << rep_test.err_stream.str(); | |
429 | std::cout << rep_test.out_stream.str(); | |
430 | ||
431 | internal::CheckCases(rep_test.error_cases, rep_test.err_stream); | |
432 | internal::CheckCases(rep_test.output_cases, rep_test.out_stream); | |
433 | ||
434 | std::cout << "\n"; | |
435 | } | |
436 | ||
437 | // now that we know the output is as expected, we can dispatch | |
438 | // the checks to subscribees. | |
439 | auto& csv = TestCases[2]; | |
440 | // would use == but gcc spits a warning | |
441 | CHECK(std::strcmp(csv.name, "CSVReporter") == 0); | |
442 | internal::GetResultsChecker().CheckResults(csv.out_stream); | |
443 | } | |
444 | ||
445 | #ifdef __GNUC__ | |
446 | #pragma GCC diagnostic pop | |
447 | #endif | |
448 | ||
449 | int SubstrCnt(const std::string& haystack, const std::string& pat) { | |
450 | if (pat.length() == 0) return 0; | |
451 | int count = 0; | |
452 | for (size_t offset = haystack.find(pat); offset != std::string::npos; | |
453 | offset = haystack.find(pat, offset + pat.length())) | |
454 | ++count; | |
455 | return count; | |
456 | } | |
457 | ||
458 | static char ToHex(int ch) { | |
459 | return ch < 10 ? static_cast<char>('0' + ch) | |
460 | : static_cast<char>('a' + (ch - 10)); | |
461 | } | |
462 | ||
463 | static char RandomHexChar() { | |
464 | static std::mt19937 rd{std::random_device{}()}; | |
465 | static std::uniform_int_distribution<int> mrand{0, 15}; | |
466 | return ToHex(mrand(rd)); | |
467 | } | |
468 | ||
469 | static std::string GetRandomFileName() { | |
470 | std::string model = "test.%%%%%%"; | |
471 | for (auto & ch : model) { | |
472 | if (ch == '%') | |
473 | ch = RandomHexChar(); | |
474 | } | |
475 | return model; | |
476 | } | |
477 | ||
478 | static bool FileExists(std::string const& name) { | |
479 | std::ifstream in(name.c_str()); | |
480 | return in.good(); | |
481 | } | |
482 | ||
483 | static std::string GetTempFileName() { | |
484 | // This function attempts to avoid race conditions where two tests | |
485 | // create the same file at the same time. However, it still introduces races | |
486 | // similar to tmpnam. | |
487 | int retries = 3; | |
488 | while (--retries) { | |
489 | std::string name = GetRandomFileName(); | |
490 | if (!FileExists(name)) | |
491 | return name; | |
492 | } | |
493 | std::cerr << "Failed to create unique temporary file name" << std::endl; | |
494 | std::abort(); | |
495 | } | |
496 | ||
497 | std::string GetFileReporterOutput(int argc, char* argv[]) { | |
498 | std::vector<char*> new_argv(argv, argv + argc); | |
499 | assert(static_cast<decltype(new_argv)::size_type>(argc) == new_argv.size()); | |
500 | ||
501 | std::string tmp_file_name = GetTempFileName(); | |
502 | std::cout << "Will be using this as the tmp file: " << tmp_file_name << '\n'; | |
503 | ||
504 | std::string tmp = "--benchmark_out="; | |
505 | tmp += tmp_file_name; | |
506 | new_argv.emplace_back(const_cast<char*>(tmp.c_str())); | |
507 | ||
508 | argc = int(new_argv.size()); | |
509 | ||
510 | benchmark::Initialize(&argc, new_argv.data()); | |
511 | benchmark::RunSpecifiedBenchmarks(); | |
512 | ||
513 | // Read the output back from the file, and delete the file. | |
514 | std::ifstream tmp_stream(tmp_file_name); | |
515 | std::string output = std::string((std::istreambuf_iterator<char>(tmp_stream)), | |
516 | std::istreambuf_iterator<char>()); | |
517 | std::remove(tmp_file_name.c_str()); | |
518 | ||
519 | return output; | |
520 | } |