]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // (C) Copyright Raffi Enficiaud 2016. |
2 | // Distributed under the Boost Software License, Version 1.0. | |
3 | // (See accompanying file LICENSE_1_0.txt or copy at | |
4 | // http://www.boost.org/LICENSE_1_0.txt) | |
5 | ||
6 | // See http://www.boost.org/libs/test for the library home page. | |
7 | // | |
8 | // tests Unit Test Framework logging facilities against | |
9 | // pattern file | |
10 | // *************************************************************************** | |
11 | ||
12 | // Boost.Test | |
13 | #define BOOST_TEST_MAIN | |
b32b8144 | 14 | #include <boost/test/unit_test.hpp> |
7c673cae | 15 | #include <boost/test/unit_test_log.hpp> |
7c673cae FG |
16 | #include <boost/test/unit_test_suite.hpp> |
17 | #include <boost/test/framework.hpp> | |
18 | #include <boost/test/unit_test_parameters.hpp> | |
19 | #include <boost/test/utils/nullstream.hpp> | |
92f5a8d4 TL |
20 | #include <boost/test/execution_monitor.hpp> |
21 | ||
7c673cae | 22 | typedef boost::onullstream onullstream_type; |
7c673cae FG |
23 | |
24 | // BOOST | |
25 | #include <boost/lexical_cast.hpp> | |
26 | ||
b32b8144 FG |
27 | // our special logger for tests |
28 | #include "logger-for-tests.hpp" | |
29 | ||
7c673cae FG |
30 | // STL |
31 | #include <iostream> | |
32 | #include <ios> | |
33 | ||
34 | using boost::test_tools::output_test_stream; | |
35 | using namespace boost::unit_test; | |
11fdf7f2 | 36 | |
7c673cae FG |
37 | |
38 | ||
39 | //____________________________________________________________________________// | |
40 | ||
41 | // this one should generate a message as it does not execute any assertion | |
42 | void good_foo() {} | |
43 | ||
44 | void almost_good_foo() | |
45 | { | |
46 | BOOST_TEST_WARN( 2>3 ); | |
47 | } | |
48 | ||
49 | void bad_foo() { | |
50 | BOOST_ERROR( "" ); | |
51 | ||
52 | BOOST_TEST_MESSAGE("this is a message"); | |
53 | BOOST_CHECK(true); | |
54 | ||
55 | BOOST_TEST_INFO("Context value=something"); | |
56 | BOOST_TEST_INFO("Context value2=something different"); | |
57 | BOOST_ERROR( "with some message" ); | |
58 | ||
59 | BOOST_CHECK_MESSAGE( 1 == 2.3, "non sense" ); | |
60 | } | |
61 | ||
7c673cae FG |
62 | void very_bad_foo() { |
63 | BOOST_TEST_CONTEXT("some context") { | |
64 | BOOST_FAIL( "very_bad_foo is fatal" ); | |
65 | } | |
66 | } | |
67 | ||
68 | struct local_exception {}; | |
69 | ||
f67539c2 | 70 | void very_bad_exception() { |
7c673cae FG |
71 | BOOST_TEST_INFO("Context value=something"); |
72 | BOOST_TEST_INFO("Context value2=something different"); | |
73 | BOOST_ERROR( "with some message" ); | |
74 | ||
75 | BOOST_TEST_INFO("exception context should be shown"); | |
76 | throw local_exception(); | |
77 | } | |
78 | ||
11fdf7f2 | 79 | void bad_foo2() { bad_foo(); } // tests with clashing names |
7c673cae | 80 | |
92f5a8d4 TL |
81 | void timeout_foo() |
82 | { | |
83 | using boost::execution_exception; | |
84 | execution_exception::location dummy; | |
85 | throw execution_exception( | |
86 | execution_exception::timeout_error, | |
87 | "fake timeout", | |
88 | dummy); | |
89 | } | |
90 | ||
f67539c2 TL |
91 | void context_mixed_foo() { |
92 | BOOST_TEST_CONTEXT("some context") { | |
93 | BOOST_TEST( true ); | |
94 | } | |
95 | ||
96 | BOOST_TEST_INFO("other context"); | |
97 | throw local_exception(); | |
98 | } | |
99 | ||
7c673cae FG |
100 | //____________________________________________________________________________// |
101 | ||
b32b8144 FG |
102 | void check( output_test_stream& output, |
103 | output_format log_format, | |
104 | test_unit_id id, | |
f67539c2 TL |
105 | log_level ll = log_successful_tests, |
106 | output_format additional_format = OF_INVALID) | |
7c673cae | 107 | { |
f67539c2 TL |
108 | { |
109 | log_setup_teardown holder(output, log_format, ll, invalid_log_level, additional_format); | |
110 | output << "* " << log_format << "-format *******************************************************************"; | |
111 | output << std::endl; | |
112 | framework::finalize_setup_phase( id ); | |
113 | framework::run( id, false ); // do not continue the test tree to have the test_log_start/end | |
114 | output << std::endl; | |
115 | } | |
7c673cae FG |
116 | |
117 | BOOST_TEST( output.match_pattern(true) ); // flushes the stream at the end of the comparison. | |
118 | } | |
119 | ||
120 | //____________________________________________________________________________// | |
121 | ||
122 | void check( output_test_stream& output, test_suite* ts ) | |
123 | { | |
124 | ts->p_default_status.value = test_unit::RS_ENABLED; | |
125 | ||
126 | check( output, OF_CLF, ts->p_id ); | |
127 | check( output, OF_XML, ts->p_id ); | |
b32b8144 FG |
128 | check( output, OF_JUNIT, ts->p_id, log_successful_tests ); |
129 | check( output, OF_JUNIT, ts->p_id, log_cpp_exception_errors ); // should branch to the log log_all_errors | |
7c673cae FG |
130 | } |
131 | ||
132 | //____________________________________________________________________________// | |
133 | ||
134 | struct guard { | |
135 | ~guard() | |
136 | { | |
b32b8144 | 137 | boost::unit_test::unit_test_log.set_format( runtime_config::get<output_format>( runtime_config::btrt_log_format ) ); |
7c673cae FG |
138 | boost::unit_test::unit_test_log.set_stream( std::cout ); |
139 | } | |
140 | }; | |
141 | ||
7c673cae FG |
142 | //____________________________________________________________________________// |
143 | ||
f67539c2 TL |
144 | struct save_arguments { |
145 | static bool save_pattern() { | |
146 | if(m_first) { | |
147 | m_save_pattern = runtime_config::save_pattern(); | |
148 | m_first = false; | |
149 | } | |
150 | return m_save_pattern; | |
151 | } | |
152 | ||
153 | static const std::vector<std::string>& saved_argv() { | |
154 | if(argv.empty()) { | |
155 | for(size_t i = 0; i < static_cast<size_t>(framework::master_test_suite().argc); i++) { | |
156 | argv.push_back(framework::master_test_suite().argv[i]); | |
157 | } | |
158 | } | |
159 | return argv; | |
160 | } | |
161 | ||
162 | static std::string get_pattern_file(const std::string &to_retrieve) { | |
163 | if(argv.empty()) { | |
164 | for(size_t i = 0; i < static_cast<size_t>(framework::master_test_suite().argc); i++) { | |
165 | argv.push_back(framework::master_test_suite().argv[i]); | |
166 | } | |
167 | } | |
168 | ||
169 | for(size_t i = 0; i < static_cast<size_t>(framework::master_test_suite().argc); i++) { | |
170 | if(argv[i].find(to_retrieve) != std::string::npos) | |
171 | return argv[i]; | |
172 | } | |
173 | ||
174 | return ""; | |
175 | } | |
176 | ||
177 | static std::vector<std::string> argv; | |
178 | static bool m_first; | |
179 | static bool m_save_pattern; | |
180 | }; | |
181 | ||
182 | bool save_arguments::m_save_pattern = false; | |
183 | bool save_arguments::m_first = true; | |
184 | std::vector<std::string> save_arguments::argv = std::vector<std::string>(); | |
185 | ||
186 | //____________________________________________________________________________// | |
187 | ||
7c673cae | 188 | |
11fdf7f2 | 189 | BOOST_AUTO_TEST_CASE( test_logs ) |
7c673cae FG |
190 | { |
191 | guard G; | |
92f5a8d4 | 192 | boost::ignore_unused( G ); |
7c673cae FG |
193 | |
194 | #define PATTERN_FILE_NAME "log-formatter-test.pattern" | |
195 | ||
196 | std::string pattern_file_name( | |
f67539c2 TL |
197 | // we cannot use runtime_config::save_arguments() as one of the test is mutating |
198 | // the arguments for testing purposes | |
199 | // we have to inspect argv as b2 may run the program from an unknown location | |
200 | save_arguments::saved_argv().size() <= 1 | |
201 | ? (save_arguments::save_pattern() | |
202 | ? PATTERN_FILE_NAME | |
203 | : "./baseline-outputs/" PATTERN_FILE_NAME ) | |
204 | : save_arguments::get_pattern_file(PATTERN_FILE_NAME)); | |
7c673cae | 205 | |
b32b8144 | 206 | output_test_stream_for_loggers test_output( pattern_file_name, |
f67539c2 | 207 | !save_arguments::save_pattern(), |
b32b8144 FG |
208 | true, |
209 | __FILE__ ); | |
7c673cae | 210 | |
f67539c2 | 211 | #line 157 |
7c673cae FG |
212 | test_suite* ts_0 = BOOST_TEST_SUITE( "0 test cases inside" ); |
213 | ||
214 | test_suite* ts_1 = BOOST_TEST_SUITE( "1 test cases inside" ); | |
215 | ts_1->add( BOOST_TEST_CASE( good_foo ) ); | |
216 | ||
217 | test_suite* ts_1b = BOOST_TEST_SUITE( "1 bad test case inside" ); | |
218 | ts_1b->add( BOOST_TEST_CASE( bad_foo ), 1 ); | |
219 | ||
220 | test_suite* ts_1c = BOOST_TEST_SUITE( "1 almost good test case inside" ); | |
221 | ts_1c->add( BOOST_TEST_CASE( almost_good_foo ) ); | |
222 | ||
223 | test_suite* ts_2 = BOOST_TEST_SUITE( "2 test cases inside" ); | |
224 | ts_2->add( BOOST_TEST_CASE( good_foo ) ); | |
225 | ts_2->add( BOOST_TEST_CASE( bad_foo ), 1 ); | |
226 | ||
227 | test_suite* ts_3 = BOOST_TEST_SUITE( "3 test cases inside" ); | |
228 | ts_3->add( BOOST_TEST_CASE( bad_foo ) ); | |
229 | test_case* tc1 = BOOST_TEST_CASE( very_bad_foo ); | |
230 | ts_3->add( tc1 ); | |
f67539c2 | 231 | test_case* tc2 = BOOST_TEST_CASE_NAME( bad_foo2 , "bad_foo2<int>" ); // this is for skipped message GH-253 |
7c673cae FG |
232 | ts_3->add( tc2 ); |
233 | tc2->depends_on( tc1 ); | |
234 | ||
235 | test_suite* ts_4 = BOOST_TEST_SUITE( "4 test cases inside" ); | |
236 | ts_4->add( BOOST_TEST_CASE( bad_foo ) ); | |
237 | ts_4->add( BOOST_TEST_CASE( very_bad_foo ) ); | |
238 | ts_4->add( BOOST_TEST_CASE( very_bad_exception ) ); | |
11fdf7f2 | 239 | ts_4->add( BOOST_TEST_CASE( bad_foo2 ) ); |
7c673cae FG |
240 | |
241 | test_suite* ts_main = BOOST_TEST_SUITE( "Fake Test Suite Hierarchy" ); | |
242 | ts_main->add( ts_0 ); | |
243 | ts_main->add( ts_1 ); | |
244 | ts_main->add( ts_2 ); | |
245 | ts_main->add( ts_3 ); | |
246 | ts_main->add( ts_4 ); | |
247 | ||
92f5a8d4 TL |
248 | test_suite* ts_timeout = BOOST_TEST_SUITE( "Timeout" ); |
249 | ts_timeout->add( BOOST_TEST_CASE( good_foo ) ); | |
250 | test_case * tc_timeout = BOOST_TEST_CASE( timeout_foo ); | |
251 | ts_timeout->add( tc_timeout ); | |
252 | ||
253 | test_suite* ts_timeout_nested = BOOST_TEST_SUITE( "Timeout-nested" ); | |
254 | ts_timeout_nested->add( BOOST_TEST_CASE( good_foo ) ); | |
255 | test_suite* ts_timeout_internal = BOOST_TEST_SUITE( "Timeout" ); | |
256 | ts_timeout_internal->add( BOOST_TEST_CASE( good_foo ) ); | |
257 | test_case * tc_timeout_internal = BOOST_TEST_CASE( timeout_foo ); | |
258 | ts_timeout_internal->add( tc_timeout_internal ); | |
259 | ts_timeout_nested->add( ts_timeout_internal ); | |
260 | ts_timeout_nested->add( BOOST_TEST_CASE_NAME( good_foo, "good_foo2" ) ); | |
261 | ||
7c673cae FG |
262 | check( test_output, ts_1 ); |
263 | ||
264 | check( test_output, ts_1b ); | |
265 | ||
266 | check( test_output, ts_1c ); | |
267 | ||
268 | check( test_output, ts_2 ); | |
269 | ||
270 | check( test_output, ts_3 ); | |
271 | ||
272 | check( test_output, ts_4 ); | |
273 | ||
f67539c2 | 274 | ts_1->add( BOOST_TEST_CASE_NAME( bad_foo, "bad<bool>" ) ); |
7c673cae FG |
275 | ts_3->depends_on( ts_1 ); |
276 | ||
277 | check( test_output, ts_main ); | |
92f5a8d4 TL |
278 | |
279 | check( test_output, ts_timeout ); | |
280 | ||
281 | check( test_output, ts_timeout_nested ); | |
7c673cae FG |
282 | } |
283 | ||
7c673cae | 284 | |
11fdf7f2 TL |
285 | BOOST_AUTO_TEST_CASE( test_logs_junit_info_closing_tags ) |
286 | { | |
287 | guard G; | |
92f5a8d4 | 288 | boost::ignore_unused( G ); |
11fdf7f2 TL |
289 | |
290 | #define PATTERN_FILE_NAME_JUNIT "log-formatter-test.pattern.junit" | |
291 | ||
292 | std::string pattern_file_name( | |
f67539c2 TL |
293 | // we cannot use runtime_config::save_arguments() as one of the test is mutating |
294 | // the arguments for testing purposes | |
295 | // we have to inspect argv as b2 may run the program from an unknown location | |
296 | save_arguments::saved_argv().size() <= 1 | |
297 | ? (save_arguments::save_pattern() | |
298 | ? PATTERN_FILE_NAME_JUNIT | |
299 | : "./baseline-outputs/" PATTERN_FILE_NAME_JUNIT ) | |
300 | : save_arguments::get_pattern_file(PATTERN_FILE_NAME_JUNIT)); | |
11fdf7f2 TL |
301 | |
302 | output_test_stream_for_loggers test_output( pattern_file_name, | |
f67539c2 | 303 | !save_arguments::save_pattern(), |
11fdf7f2 TL |
304 | true, |
305 | __FILE__ ); | |
306 | ||
f67539c2 | 307 | #line 249 |
11fdf7f2 TL |
308 | test_suite* ts_main = BOOST_TEST_SUITE( "1 test cases inside" ); |
309 | ts_main->add( BOOST_TEST_CASE( almost_good_foo ) ); | |
310 | ||
311 | ts_main->p_default_status.value = test_unit::RS_ENABLED; | |
312 | char const* argv[] = { "a.exe", "--run_test=*", "--build_info" }; | |
313 | int argc = sizeof(argv)/sizeof(argv[0]); | |
314 | boost::unit_test::runtime_config::init( argc, (char**)argv ); | |
315 | boost::unit_test::framework::impl::setup_for_execution( *ts_main ); | |
316 | ||
317 | check( test_output, OF_JUNIT, ts_main->p_id, log_successful_tests ); | |
92f5a8d4 TL |
318 | |
319 | test_suite* ts_timeout = BOOST_TEST_SUITE( "Timeout" ); | |
320 | ts_timeout->add( BOOST_TEST_CASE( good_foo ) ); | |
321 | test_case * tc_timeout = BOOST_TEST_CASE( timeout_foo ); | |
322 | ts_timeout->add( tc_timeout ); | |
323 | ||
324 | check( test_output, OF_JUNIT, ts_timeout->p_id, log_successful_tests ); | |
325 | ||
326 | ||
327 | test_suite* ts_account_failures = BOOST_TEST_SUITE( "1 junit failure is not error" ); | |
328 | ts_account_failures->add( BOOST_TEST_CASE( bad_foo ) ); | |
329 | ts_account_failures->add( BOOST_TEST_CASE( very_bad_foo ) ); | |
330 | ts_account_failures->add( BOOST_TEST_CASE( good_foo ) ); | |
331 | ts_account_failures->add( BOOST_TEST_CASE( bad_foo2 ) ); | |
332 | ||
333 | char const* argv2[] = { "a.exe", "--run_test=*", "--build_info=false" }; | |
334 | int argc2 = sizeof(argv2)/sizeof(argv2[0]); | |
335 | boost::unit_test::runtime_config::init( argc2, (char**)argv2 ); | |
336 | boost::unit_test::framework::impl::setup_for_execution( *ts_account_failures ); | |
337 | ||
338 | check( test_output, OF_JUNIT, ts_account_failures->p_id, log_messages ); | |
11fdf7f2 TL |
339 | } |
340 | ||
f67539c2 TL |
341 | |
342 | BOOST_AUTO_TEST_CASE( test_logs_context ) | |
343 | { | |
344 | guard G; | |
345 | boost::ignore_unused( G ); | |
346 | ||
347 | #define PATTERN_FILE_NAME_CONTEXT "log-formatter-context-test.pattern" | |
348 | ||
349 | std::string pattern_file_name( | |
350 | // we cannot use runtime_config::save_arguments() as one of the test is mutating | |
351 | // the arguments for testing purposes | |
352 | // we have to inspect argv as b2 may run the program from an unknown location | |
353 | save_arguments::saved_argv().size() <= 1 | |
354 | ? (save_arguments::save_pattern() | |
355 | ? PATTERN_FILE_NAME_CONTEXT | |
356 | : "./baseline-outputs/" PATTERN_FILE_NAME_CONTEXT ) | |
357 | : save_arguments::get_pattern_file(PATTERN_FILE_NAME_CONTEXT)); | |
358 | ||
359 | output_test_stream_for_loggers test_output( pattern_file_name, | |
360 | !save_arguments::save_pattern(), | |
361 | true, | |
362 | __FILE__ ); | |
363 | ||
364 | #line 330 | |
365 | test_suite* ts_1 = BOOST_TEST_SUITE( "1 test cases inside" ); | |
366 | ts_1->add( BOOST_TEST_CASE( context_mixed_foo ) ); | |
367 | ||
368 | ts_1->p_default_status.value = test_unit::RS_ENABLED; | |
369 | check( test_output, OF_CLF, ts_1->p_id, log_successful_tests ); | |
370 | check( test_output, OF_CLF, ts_1->p_id, log_cpp_exception_errors ); | |
371 | check( test_output, OF_CLF, ts_1->p_id, log_successful_tests, OF_JUNIT ); | |
372 | check( test_output, OF_CLF, ts_1->p_id, log_cpp_exception_errors, OF_JUNIT ); | |
373 | ||
374 | check( test_output, OF_XML, ts_1->p_id, log_successful_tests ); | |
375 | check( test_output, OF_XML, ts_1->p_id, log_cpp_exception_errors ); | |
376 | check( test_output, OF_JUNIT, ts_1->p_id, log_successful_tests ); | |
377 | check( test_output, OF_JUNIT, ts_1->p_id, log_cpp_exception_errors ); | |
378 | ||
379 | } | |
380 | ||
381 | ||
7c673cae | 382 | // EOF |