1 // (C) Copyright 2016 Raffi Enficiaud.
2 // Distributed under the Boost Software License, Version 1.0.
3 // (See accompanying file LICENSE_1_0.txt or copy at
4 // http://www.boost.org/LICENSE_1_0.txt)
6 // See http://www.boost.org/libs/test for the library home page.
9 ///@brief Contains the implementatoin of the Junit log formatter (OF_JUNIT)
10 // ***************************************************************************
12 #ifndef BOOST_TEST_JUNIT_LOG_FORMATTER_IPP__
13 #define BOOST_TEST_JUNIT_LOG_FORMATTER_IPP__
16 #include <boost/test/output/junit_log_formatter.hpp>
17 #include <boost/test/execution_monitor.hpp>
18 #include <boost/test/framework.hpp>
19 #include <boost/test/tree/test_unit.hpp>
20 #include <boost/test/utils/basic_cstring/io.hpp>
21 #include <boost/test/utils/xml_printer.hpp>
22 #include <boost/test/utils/string_cast.hpp>
23 #include <boost/test/framework.hpp>
25 #include <boost/test/tree/visitor.hpp>
26 #include <boost/test/tree/test_case_counter.hpp>
27 #include <boost/test/tree/traverse.hpp>
28 #include <boost/test/results_collector.hpp>
30 #include <boost/test/utils/algorithm.hpp>
31 #include <boost/test/utils/string_cast.hpp>
33 //#include <boost/test/results_reporter.hpp>
37 #include <boost/version.hpp>
44 #include <boost/test/detail/suppress_warnings.hpp>
47 //____________________________________________________________________________//
54 struct s_replace_chars {
56 void operator()(T& to_replace)
60 else if(to_replace == ' ')
65 inline std::string tu_name_normalize(std::string full_name)
67 // maybe directly using normalize_test_case_name instead?
68 std::for_each(full_name.begin(), full_name.end(), s_replace_chars());
72 inline std::string tu_name_remove_newlines(std::string full_name)
74 full_name.erase(std::remove(full_name.begin(), full_name.end(), '\n'), full_name.end());
78 const_string file_basename(const_string filename) {
80 const_string path_sep( "\\/" );
81 const_string::iterator it = unit_test::utils::find_last_of( filename.begin(), filename.end(),
82 path_sep.begin(), path_sep.end() );
83 if( it != filename.end() )
84 filename.trim_left( it + 1 );
90 // ************************************************************************** //
91 // ************** junit_log_formatter ************** //
92 // ************************************************************************** //
95 junit_log_formatter::log_start( std::ostream& /*ostr*/, counter_t /*test_cases_amount*/)
98 list_path_to_root.clear();
99 runner_log_entry.clear();
102 //____________________________________________________________________________//
104 class junit_result_helper : public test_tree_visitor {
106 typedef junit_impl::junit_log_helper::assertion_entry assertion_entry;
107 typedef std::vector< assertion_entry >::const_iterator vect_assertion_entry_citerator;
108 typedef std::list<std::string>::const_iterator list_str_citerator;
111 explicit junit_result_helper(
112 std::ostream& stream,
114 junit_log_formatter::map_trace_t const& mt,
115 junit_impl::junit_log_helper const& runner_log_,
116 bool display_build_info )
120 , runner_log( runner_log_ )
122 , m_display_build_info(display_build_info)
125 void add_log_entry(assertion_entry const& log) const
127 std::string entry_type;
128 if( log.log_entry == assertion_entry::log_entry_failure ) {
129 entry_type = "failure";
131 else if( log.log_entry == assertion_entry::log_entry_error ) {
132 entry_type = "error";
140 << " message" << utils::attr_value() << log.logentry_message
141 << " type" << utils::attr_value() << log.logentry_type
144 if(!log.output.empty()) {
145 m_stream << utils::cdata() << "\n" + log.output;
148 m_stream << "</" << entry_type << ">";
151 struct conditional_cdata_helper {
153 std::string const field;
156 conditional_cdata_helper(std::ostream &ostr_, std::string field_)
162 ~conditional_cdata_helper() {
164 ostr << BOOST_TEST_L( "]]>" ) << "</" << field << '>' << std::endl;
168 void operator()(const std::string& s) {
169 bool current_empty = s.empty();
173 ostr << '<' << field << '>' << BOOST_TEST_L( "<![CDATA[" );
182 std::list<std::string> build_skipping_chain(test_unit const & tu) const
184 // we enter here because we know that the tu has been skipped.
185 // either junit has not seen this tu, or it is indicated as disabled
186 assert(m_map_test.count(tu.p_id) == 0 || results_collector.results( tu.p_id ).p_skipped);
188 std::list<std::string> out;
190 test_unit_id id(tu.p_id);
191 while( id != m_ts.p_id && id != INV_TEST_UNIT_ID) {
192 test_unit const& tu_hierarchy = boost::unit_test::framework::get( id, TUT_ANY );
193 out.push_back("- disabled test unit: '" + tu_name_remove_newlines(tu_hierarchy.full_name()) + "'\n");
194 if(m_map_test.count(id) > 0)
196 // junit has seen the reason: this is enough for constructing the chain
199 id = tu_hierarchy.p_parent_id;
201 junit_log_formatter::map_trace_t::const_iterator it_element_stack(m_map_test.find(id));
202 if( it_element_stack != m_map_test.end() )
204 out.push_back("- reason: '" + it_element_stack->second.skipping_reason + "'");
205 out.push_front("Test case disabled because of the following chain of decision:\n");
211 std::string get_class_name(test_unit const & tu_class) const {
212 std::string classname;
213 test_unit_id id(tu_class.p_parent_id);
214 while( id != m_ts.p_id && id != INV_TEST_UNIT_ID ) {
215 test_unit const& tu = boost::unit_test::framework::get( id, TUT_ANY );
216 classname = tu_name_normalize(tu.p_name) + "." + classname;
220 // removes the trailing dot
221 if(!classname.empty() && *classname.rbegin() == '.') {
222 classname.erase(classname.size()-1);
228 void write_testcase_header(test_unit const & tu,
229 test_results const *tr,
230 int nb_assertions) const
233 std::string classname;
235 if(tu.p_id == m_ts.p_id ) {
239 classname = get_class_name(tu);
240 name = tu_name_normalize(tu.p_name);
243 if( tu.p_type == TUT_SUITE ) {
244 name += "-setup-teardown";
247 m_stream << "<testcase assertions" << utils::attr_value() << nb_assertions;
248 if(!classname.empty())
249 m_stream << " classname" << utils::attr_value() << classname;
251 // test case name and time taken
253 << " name" << utils::attr_value() << name
254 << " time" << utils::attr_value() << double(tr->p_duration_microseconds) * 1E-6
258 void write_testcase_system_out(junit_impl::junit_log_helper const &detailed_log,
259 test_unit const * tu,
262 // system-out + all info/messages, the object skips the empty entries
263 conditional_cdata_helper system_out_helper(m_stream, "system-out");
265 // indicate why the test has been skipped first
267 std::list<std::string> skipping_decision_chain = build_skipping_chain(*tu);
268 for(list_str_citerator it(skipping_decision_chain.begin()), ite(skipping_decision_chain.end());
272 system_out_helper(*it);
277 for(list_str_citerator it(detailed_log.system_out.begin()), ite(detailed_log.system_out.end());
281 system_out_helper(*it);
284 // warning/info message last
285 for(vect_assertion_entry_citerator it(detailed_log.assertion_entries.begin());
286 it != detailed_log.assertion_entries.end();
289 if(it->log_entry != assertion_entry::log_entry_info)
291 system_out_helper(it->output);
295 void write_testcase_system_err(junit_impl::junit_log_helper const &detailed_log,
296 test_unit const * tu,
297 test_results const *tr) const
299 // system-err output + test case informations
300 bool has_failed = (tr != 0) ? !tr->p_skipped && !tr->passed() : false;
301 if(!detailed_log.system_err.empty() || has_failed)
303 std::ostringstream o;
305 o << "Failures detected in:" << std::endl;
308 o << "ERROR STREAM:" << std::endl;
311 if(tu->p_type == TUT_SUITE) {
312 if( tu->p_id == m_ts.p_id ) {
313 o << " boost.test global setup/teardown" << std::endl;
315 o << "- test suite: " << tu_name_remove_newlines(tu->full_name()) << std::endl;
319 o << "- test case: " << tu_name_remove_newlines(tu->full_name());
320 if(!tu->p_description.value.empty())
321 o << " '" << tu->p_description << "'";
324 << "- file: " << file_basename(tu->p_file_name) << std::endl
325 << "- line: " << tu->p_line_num << std::endl
329 if(!detailed_log.system_err.empty())
330 o << std::endl << "STDERR BEGIN: ------------" << std::endl;
332 for(list_str_citerator it(detailed_log.system_err.begin()), ite(detailed_log.system_err.end());
339 if(!detailed_log.system_err.empty())
340 o << std::endl << "STDERR END ------------" << std::endl;
342 conditional_cdata_helper system_err_helper(m_stream, "system-err");
343 system_err_helper(o.str());
347 int get_nb_assertions(junit_impl::junit_log_helper const &detailed_log,
348 test_unit const & tu,
349 test_results const *tr) const {
350 int nb_assertions(-1);
351 if( tu.p_type == TUT_SUITE ) {
353 for(vect_assertion_entry_citerator it(detailed_log.assertion_entries.begin());
354 it != detailed_log.assertion_entries.end();
357 if(it->log_entry != assertion_entry::log_entry_info)
362 nb_assertions = tr->p_assertions_passed + tr->p_assertions_failed;
365 return nb_assertions;
368 void output_detailed_logs(junit_impl::junit_log_helper const &detailed_log,
369 test_unit const & tu,
371 test_results const *tr) const
373 int nb_assertions = get_nb_assertions(detailed_log, tu, tr);
374 if(!nb_assertions && tu.p_type == TUT_SUITE)
377 write_testcase_header(tu, tr, nb_assertions);
380 m_stream << "<skipped/>" << std::endl;
384 for(vect_assertion_entry_citerator it(detailed_log.assertion_entries.begin());
385 it != detailed_log.assertion_entries.end();
392 write_testcase_system_out(detailed_log, &tu, skipped);
393 write_testcase_system_err(detailed_log, &tu, tr);
394 m_stream << "</testcase>" << std::endl;
397 void visit( test_case const& tc )
400 test_results const& tr = results_collector.results( tc.p_id );
401 junit_log_formatter::map_trace_t::const_iterator it_find = m_map_test.find(tc.p_id);
402 if(it_find == m_map_test.end())
404 // test has been skipped and not seen by the logger
405 output_detailed_logs(junit_impl::junit_log_helper(), tc, true, &tr);
408 output_detailed_logs(it_find->second, tc, tr.p_skipped, &tr);
412 bool test_suite_start( test_suite const& ts )
414 test_results const& tr = results_collector.results( ts.p_id );
416 // unique test suite, without s, nesting not supported in CI
417 if( m_ts.p_id == ts.p_id ) {
418 m_stream << "<testsuite";
421 // << "disabled=\"" << tr.p_test_cases_skipped << "\" "
422 << " tests" << utils::attr_value() << tr.p_test_cases_passed
423 << " skipped" << utils::attr_value() << tr.p_test_cases_skipped
424 << " errors" << utils::attr_value() << tr.p_test_cases_aborted
425 << " failures" << utils::attr_value() << tr.p_test_cases_failed
426 << " id" << utils::attr_value() << m_id++
427 << " name" << utils::attr_value() << tu_name_normalize(ts.p_name)
428 << " time" << utils::attr_value() << (tr.p_duration_microseconds * 1E-6)
431 if(m_display_build_info)
433 m_stream << "<properties>" << std::endl;
434 m_stream << "<property name=\"platform\" value" << utils::attr_value() << BOOST_PLATFORM << std::endl;
435 m_stream << "<property name=\"compiler\" value" << utils::attr_value() << BOOST_COMPILER << std::endl;
436 m_stream << "<property name=\"stl\" value" << utils::attr_value() << BOOST_STDLIB << std::endl;
438 std::ostringstream o;
439 o << BOOST_VERSION/100000 << "." << BOOST_VERSION/100 % 1000 << "." << BOOST_VERSION % 100;
440 m_stream << "<property name=\"boost\" value" << utils::attr_value() << o.str() << std::endl;
441 m_stream << "</properties>" << std::endl;
445 if( !tr.p_skipped ) {
446 // if we land here, then this is a chance that we are logging the fixture setup/teardown of a test-suite.
447 // the setup/teardown logging of a test-case is part of the test case.
448 // we do not care about the test-suite that were skipped (really??)
449 junit_log_formatter::map_trace_t::const_iterator it_find = m_map_test.find(ts.p_id);
450 if(it_find != m_map_test.end()) {
451 output_detailed_logs(it_find->second, ts, false, &tr);
455 return true; // indicates that the children should also be parsed
458 virtual void test_suite_finish( test_suite const& ts )
460 if( m_ts.p_id == ts.p_id ) {
461 write_testcase_system_out(runner_log, 0, false);
462 write_testcase_system_err(runner_log, 0, 0);
464 m_stream << "</testsuite>";
471 std::ostream& m_stream;
472 test_unit const& m_ts;
473 junit_log_formatter::map_trace_t const& m_map_test;
474 junit_impl::junit_log_helper const& runner_log;
476 bool m_display_build_info;
482 junit_log_formatter::log_finish( std::ostream& ostr )
484 ostr << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" << std::endl;
486 // getting the root test suite
487 if(!map_tests.empty()) {
488 test_unit* root = &boost::unit_test::framework::get( map_tests.begin()->first, TUT_ANY );
490 // looking for the root of the SUBtree (we stay in the subtree)
491 while(root->p_parent_id != INV_TEST_UNIT_ID && map_tests.count(root->p_parent_id) > 0) {
492 root = &boost::unit_test::framework::get( root->p_parent_id, TUT_ANY );
494 junit_result_helper ch( ostr, *root, map_tests, this->runner_log_entry, m_display_build_info );
495 traverse_test_tree( root->p_id, ch, true ); // last is to ignore disabled suite special handling
498 ostr << "<testsuites errors=\"1\">";
499 ostr << "<testsuite errors=\"1\" name=\"boost-test-framework\">";
500 ostr << "<testcase assertions=\"1\" name=\"test-setup\">";
501 ostr << "<system-out>Incorrect setup: no test case executed</system-out>";
502 ostr << "</testcase></testsuite></testsuites>";
507 //____________________________________________________________________________//
510 junit_log_formatter::log_build_info( std::ostream& /*ostr*/ )
512 m_display_build_info = true;
515 //____________________________________________________________________________//
518 junit_log_formatter::test_unit_start( std::ostream& /*ostr*/, test_unit const& tu )
520 list_path_to_root.push_back( tu.p_id );
521 map_tests.insert(std::make_pair(tu.p_id, junit_impl::junit_log_helper())); // current_test_case_id not working here
526 //____________________________________________________________________________//
529 junit_log_formatter::test_unit_finish( std::ostream& /*ostr*/, test_unit const& tu, unsigned long /*elapsed*/ )
531 // the time is already stored in the result_reporter
532 assert( tu.p_id == list_path_to_root.back() );
533 list_path_to_root.pop_back();
537 junit_log_formatter::test_unit_aborted( std::ostream& /*ostr*/, test_unit const& tu )
539 assert( tu.p_id == list_path_to_root.back() );
540 //list_path_to_root.pop_back();
543 //____________________________________________________________________________//
546 junit_log_formatter::test_unit_skipped( std::ostream& /*ostr*/, test_unit const& tu, const_string reason )
548 // if a test unit is skipped, then the start of this TU has not been called yet.
549 // we cannot use get_current_log_entry here, but the TU id should appear in the map.
550 // The "skip" boolean is given by the boost.test framework
551 junit_impl::junit_log_helper& v = map_tests[tu.p_id]; // not sure if we can use get_current_log_entry()
552 v.skipping_reason.assign(reason.begin(), reason.end());
555 //____________________________________________________________________________//
558 junit_log_formatter::log_exception_start( std::ostream& /*ostr*/, log_checkpoint_data const& checkpoint_data, execution_exception const& ex )
560 std::ostringstream o;
561 execution_exception::location const& loc = ex.where();
563 m_is_last_assertion_or_error = false;
565 junit_impl::junit_log_helper& last_entry = get_current_log_entry();
567 junit_impl::junit_log_helper::assertion_entry entry;
569 entry.logentry_message = "unexpected exception";
570 entry.log_entry = junit_impl::junit_log_helper::assertion_entry::log_entry_error;
574 case execution_exception::cpp_exception_error:
575 entry.logentry_type = "uncaught exception";
577 case execution_exception::timeout_error:
578 entry.logentry_type = "execution timeout";
580 case execution_exception::user_error:
581 entry.logentry_type = "user, assert() or CRT error";
583 case execution_exception::user_fatal_error:
584 // Looks like never used
585 entry.logentry_type = "user fatal error";
587 case execution_exception::system_error:
588 entry.logentry_type = "system error";
590 case execution_exception::system_fatal_error:
591 entry.logentry_type = "system fatal error";
594 entry.logentry_type = "no error"; // not sure how to handle this one
598 o << "UNCAUGHT EXCEPTION:" << std::endl;
599 if( !loc.m_function.is_empty() )
600 o << "- function: \"" << loc.m_function << "\"" << std::endl;
602 o << "- file: " << file_basename(loc.m_file_name) << std::endl
603 << "- line: " << loc.m_line_num << std::endl
606 o << "\nEXCEPTION STACK TRACE: --------------\n" << ex.what()
607 << "\n-------------------------------------";
609 if( !checkpoint_data.m_file_name.is_empty() ) {
610 o << std::endl << std::endl
611 << "Last checkpoint:" << std::endl
612 << "- message: \"" << checkpoint_data.m_message << "\"" << std::endl
613 << "- file: " << file_basename(checkpoint_data.m_file_name) << std::endl
614 << "- line: " << checkpoint_data.m_line_num << std::endl
618 entry.output = o.str();
620 last_entry.assertion_entries.push_back(entry);
623 //____________________________________________________________________________//
626 junit_log_formatter::log_exception_finish( std::ostream& /*ostr*/ )
628 // sealing the last entry
629 assert(!get_current_log_entry().assertion_entries.back().sealed);
630 get_current_log_entry().assertion_entries.back().sealed = true;
633 //____________________________________________________________________________//
636 junit_log_formatter::log_entry_start( std::ostream& /*ostr*/, log_entry_data const& entry_data, log_entry_types let )
638 junit_impl::junit_log_helper& last_entry = get_current_log_entry();
639 last_entry.skipping = false;
640 m_is_last_assertion_or_error = true;
643 case unit_test_log_formatter::BOOST_UTL_ET_INFO:
645 if(m_log_level_internal > log_successful_tests) {
646 last_entry.skipping = true;
649 // no break on purpose
651 case unit_test_log_formatter::BOOST_UTL_ET_MESSAGE:
653 if(m_log_level_internal > log_messages) {
654 last_entry.skipping = true;
657 // no break on purpose
659 case unit_test_log_formatter::BOOST_UTL_ET_WARNING:
661 if(m_log_level_internal > log_warnings) {
662 last_entry.skipping = true;
665 std::ostringstream o;
666 junit_impl::junit_log_helper::assertion_entry entry;
668 entry.log_entry = junit_impl::junit_log_helper::assertion_entry::log_entry_info;
669 entry.logentry_message = "info";
670 entry.logentry_type = "message";
672 o << (let == unit_test_log_formatter::BOOST_UTL_ET_WARNING ?
673 "WARNING:" : (let == unit_test_log_formatter::BOOST_UTL_ET_MESSAGE ?
674 "MESSAGE:" : "INFO:"))
676 << "- file : " << file_basename(entry_data.m_file_name) << std::endl
677 << "- line : " << entry_data.m_line_num << std::endl
678 << "- message: "; // no CR
680 entry.output += o.str();
681 last_entry.assertion_entries.push_back(entry);
685 case unit_test_log_formatter::BOOST_UTL_ET_ERROR:
686 case unit_test_log_formatter::BOOST_UTL_ET_FATAL_ERROR:
688 std::ostringstream o;
689 junit_impl::junit_log_helper::assertion_entry entry;
690 entry.log_entry = junit_impl::junit_log_helper::assertion_entry::log_entry_failure;
691 entry.logentry_message = "failure";
692 entry.logentry_type = (let == unit_test_log_formatter::BOOST_UTL_ET_ERROR ? "assertion error" : "fatal error");
694 o << "ASSERTION FAILURE:" << std::endl
695 << "- file : " << file_basename(entry_data.m_file_name) << std::endl
696 << "- line : " << entry_data.m_line_num << std::endl
697 << "- message: " ; // no CR
699 entry.output += o.str();
700 last_entry.assertion_entries.push_back(entry);
706 //____________________________________________________________________________//
709 junit_log_formatter::log_entry_value( std::ostream& /*ostr*/, const_string value )
711 junit_impl::junit_log_helper& last_entry = get_current_log_entry();
712 if(last_entry.skipping)
715 assert(last_entry.assertion_entries.empty() || !last_entry.assertion_entries.back().sealed);
717 if(!last_entry.assertion_entries.empty())
719 junit_impl::junit_log_helper::assertion_entry& log_entry = last_entry.assertion_entries.back();
720 log_entry.output += value;
724 // this may be a message coming from another observer
725 // the prefix is set in the log_entry_start
726 last_entry.system_out.push_back(std::string(value.begin(), value.end()));
730 //____________________________________________________________________________//
733 junit_log_formatter::log_entry_finish( std::ostream& /*ostr*/ )
735 junit_impl::junit_log_helper& last_entry = get_current_log_entry();
736 if(!last_entry.skipping)
738 assert(last_entry.assertion_entries.empty() || !last_entry.assertion_entries.back().sealed);
740 if(!last_entry.assertion_entries.empty()) {
741 junit_impl::junit_log_helper::assertion_entry& log_entry = last_entry.assertion_entries.back();
742 log_entry.output += "\n\n"; // quote end, CR
743 log_entry.sealed = true;
746 last_entry.system_out.push_back("\n\n"); // quote end, CR
750 last_entry.skipping = false;
753 //____________________________________________________________________________//
756 junit_log_formatter::entry_context_start( std::ostream& /*ostr*/, log_level )
758 junit_impl::junit_log_helper& last_entry = get_current_log_entry();
759 if(last_entry.skipping)
762 std::vector< junit_impl::junit_log_helper::assertion_entry > &v_failure_or_error = last_entry.assertion_entries;
763 assert(!v_failure_or_error.back().sealed);
765 junit_impl::junit_log_helper::assertion_entry& last_log_entry = v_failure_or_error.back();
766 if(m_is_last_assertion_or_error)
768 last_log_entry.output += "\n- context:\n";
772 last_log_entry.output += "\n\nCONTEXT:\n";
776 //____________________________________________________________________________//
779 junit_log_formatter::entry_context_finish( std::ostream& /*ostr*/, log_level )
781 // no op, may be removed
782 junit_impl::junit_log_helper& last_entry = get_current_log_entry();
783 if(last_entry.skipping)
785 assert(!get_current_log_entry().assertion_entries.back().sealed);
788 //____________________________________________________________________________//
791 junit_log_formatter::log_entry_context( std::ostream& /*ostr*/, log_level , const_string context_descr )
793 junit_impl::junit_log_helper& last_entry = get_current_log_entry();
794 if(last_entry.skipping)
797 assert(!last_entry.assertion_entries.back().sealed);
798 junit_impl::junit_log_helper::assertion_entry& last_log_entry = get_current_log_entry().assertion_entries.back();
800 last_log_entry.output +=
801 (m_is_last_assertion_or_error ? " - '": "- '") + std::string(context_descr.begin(), context_descr.end()) + "'\n"; // quote end
804 //____________________________________________________________________________//
808 junit_log_formatter::get_default_stream_description() const {
809 std::string name = framework::master_test_suite().p_name.value;
811 static const std::string to_replace[] = { " ", "\"", "/", "\\", ":"};
812 static const std::string replacement[] = { "_", "_" , "_", "_" , "_"};
814 name = unit_test::utils::replace_all_occurrences_of(
816 to_replace, to_replace + sizeof(to_replace)/sizeof(to_replace[0]),
817 replacement, replacement + sizeof(replacement)/sizeof(replacement[0]));
819 std::ifstream check_init((name + ".xml").c_str());
821 return name + ".xml";
824 for(; index < 100; index++) {
825 std::string candidate = name + "_" + utils::string_cast(index) + ".xml";
826 std::ifstream file(candidate.c_str());
831 return name + ".xml";
834 } // namespace output
835 } // namespace unit_test
838 #include <boost/test/detail/enable_warnings.hpp>
840 #endif // BOOST_TEST_junit_log_formatter_IPP_020105GER