ceph/src/googletest/googletest/test/googletest-json-output-unittest.py

   1 #!/usr/bin/env python
   2 # Copyright 2018, Google Inc.
   3 # All rights reserved.
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 #     * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #     * Redistributions in binary form must reproduce the above
  12 # copyright notice, this list of conditions and the following disclaimer
  13 # in the documentation and/or other materials provided with the
  14 # distribution.
  15 #     * Neither the name of Google Inc. nor the names of its
  16 # contributors may be used to endorse or promote products derived from
  17 # this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31 """Unit test for the gtest_json_output module."""
  32
  33 import datetime
  34 import errno
  35 import json
  36 import os
  37 import re
  38 import sys
  39
  40 import gtest_json_test_utils
  41 import gtest_test_utils
  42
  43 GTEST_FILTER_FLAG = '--gtest_filter'
  44 GTEST_LIST_TESTS_FLAG = '--gtest_list_tests'
  45 GTEST_OUTPUT_FLAG = '--gtest_output'
  46 GTEST_DEFAULT_OUTPUT_FILE = 'test_detail.json'
  47 GTEST_PROGRAM_NAME = 'gtest_xml_output_unittest_'
  48
  49 # The flag indicating stacktraces are not supported
  50 NO_STACKTRACE_SUPPORT_FLAG = '--no_stacktrace_support'
  51
  52 SUPPORTS_STACK_TRACES = NO_STACKTRACE_SUPPORT_FLAG not in sys.argv
  53
  54 if SUPPORTS_STACK_TRACES:
  55   STACK_TRACE_TEMPLATE = '\nStack trace:\n*'
  56 else:
  57   STACK_TRACE_TEMPLATE = ''
  58
  59 EXPECTED_NON_EMPTY = {
  60     u'tests':
  61         26,
  62     u'failures':
  63         5,
  64     u'disabled':
  65         2,
  66     u'errors':
  67         0,
  68     u'timestamp':
  69         u'*',
  70     u'time':
  71         u'*',
  72     u'ad_hoc_property':
  73         u'42',
  74     u'name':
  75         u'AllTests',
  76     u'testsuites': [{
  77         u'name':
  78             u'SuccessfulTest',
  79         u'tests':
  80             1,
  81         u'failures':
  82             0,
  83         u'disabled':
  84             0,
  85         u'errors':
  86             0,
  87         u'time':
  88             u'*',
  89         u'timestamp':
  90             u'*',
  91         u'testsuite': [{
  92             u'name': u'Succeeds',
  93             u'status': u'RUN',
  94             u'result': u'COMPLETED',
  95             u'time': u'*',
  96             u'timestamp': u'*',
  97             u'classname': u'SuccessfulTest'
  98         }]
  99     }, {
 100         u'name':
 101             u'FailedTest',
 102         u'tests':
 103             1,
 104         u'failures':
 105             1,
 106         u'disabled':
 107             0,
 108         u'errors':
 109             0,
 110         u'time':
 111             u'*',
 112         u'timestamp':
 113             u'*',
 114         u'testsuite': [{
 115             u'name':
 116                 u'Fails',
 117             u'status':
 118                 u'RUN',
 119             u'result':
 120                 u'COMPLETED',
 121             u'time':
 122                 u'*',
 123             u'timestamp':
 124                 u'*',
 125             u'classname':
 126                 u'FailedTest',
 127             u'failures': [{
 128                 u'failure': u'gtest_xml_output_unittest_.cc:*\n'
 129                             u'Expected equality of these values:\n'
 130                             u'  1\n  2' + STACK_TRACE_TEMPLATE,
 131                 u'type': u''
 132             }]
 133         }]
 134     }, {
 135         u'name':
 136             u'DisabledTest',
 137         u'tests':
 138             1,
 139         u'failures':
 140             0,
 141         u'disabled':
 142             1,
 143         u'errors':
 144             0,
 145         u'time':
 146             u'*',
 147         u'timestamp':
 148             u'*',
 149         u'testsuite': [{
 150             u'name': u'DISABLED_test_not_run',
 151             u'status': u'NOTRUN',
 152             u'result': u'SUPPRESSED',
 153             u'time': u'*',
 154             u'timestamp': u'*',
 155             u'classname': u'DisabledTest'
 156         }]
 157     }, {
 158         u'name':
 159             u'SkippedTest',
 160         u'tests':
 161             3,
 162         u'failures':
 163             1,
 164         u'disabled':
 165             0,
 166         u'errors':
 167             0,
 168         u'time':
 169             u'*',
 170         u'timestamp':
 171             u'*',
 172         u'testsuite': [{
 173             u'name': u'Skipped',
 174             u'status': u'RUN',
 175             u'result': u'SKIPPED',
 176             u'time': u'*',
 177             u'timestamp': u'*',
 178             u'classname': u'SkippedTest'
 179         }, {
 180             u'name': u'SkippedWithMessage',
 181             u'status': u'RUN',
 182             u'result': u'SKIPPED',
 183             u'time': u'*',
 184             u'timestamp': u'*',
 185             u'classname': u'SkippedTest'
 186         }, {
 187             u'name':
 188                 u'SkippedAfterFailure',
 189             u'status':
 190                 u'RUN',
 191             u'result':
 192                 u'COMPLETED',
 193             u'time':
 194                 u'*',
 195             u'timestamp':
 196                 u'*',
 197             u'classname':
 198                 u'SkippedTest',
 199             u'failures': [{
 200                 u'failure': u'gtest_xml_output_unittest_.cc:*\n'
 201                             u'Expected equality of these values:\n'
 202                             u'  1\n  2' + STACK_TRACE_TEMPLATE,
 203                 u'type': u''
 204             }]
 205         }]
 206     }, {
 207         u'name':
 208             u'MixedResultTest',
 209         u'tests':
 210             3,
 211         u'failures':
 212             1,
 213         u'disabled':
 214             1,
 215         u'errors':
 216             0,
 217         u'time':
 218             u'*',
 219         u'timestamp':
 220             u'*',
 221         u'testsuite': [{
 222             u'name': u'Succeeds',
 223             u'status': u'RUN',
 224             u'result': u'COMPLETED',
 225             u'time': u'*',
 226             u'timestamp': u'*',
 227             u'classname': u'MixedResultTest'
 228         }, {
 229             u'name':
 230                 u'Fails',
 231             u'status':
 232                 u'RUN',
 233             u'result':
 234                 u'COMPLETED',
 235             u'time':
 236                 u'*',
 237             u'timestamp':
 238                 u'*',
 239             u'classname':
 240                 u'MixedResultTest',
 241             u'failures': [{
 242                 u'failure': u'gtest_xml_output_unittest_.cc:*\n'
 243                             u'Expected equality of these values:\n'
 244                             u'  1\n  2' + STACK_TRACE_TEMPLATE,
 245                 u'type': u''
 246             }, {
 247                 u'failure': u'gtest_xml_output_unittest_.cc:*\n'
 248                             u'Expected equality of these values:\n'
 249                             u'  2\n  3' + STACK_TRACE_TEMPLATE,
 250                 u'type': u''
 251             }]
 252         }, {
 253             u'name': u'DISABLED_test',
 254             u'status': u'NOTRUN',
 255             u'result': u'SUPPRESSED',
 256             u'time': u'*',
 257             u'timestamp': u'*',
 258             u'classname': u'MixedResultTest'
 259         }]
 260     }, {
 261         u'name':
 262             u'XmlQuotingTest',
 263         u'tests':
 264             1,
 265         u'failures':
 266             1,
 267         u'disabled':
 268             0,
 269         u'errors':
 270             0,
 271         u'time':
 272             u'*',
 273         u'timestamp':
 274             u'*',
 275         u'testsuite': [{
 276             u'name':
 277                 u'OutputsCData',
 278             u'status':
 279                 u'RUN',
 280             u'result':
 281                 u'COMPLETED',
 282             u'time':
 283                 u'*',
 284             u'timestamp':
 285                 u'*',
 286             u'classname':
 287                 u'XmlQuotingTest',
 288             u'failures': [{
 289                 u'failure': u'gtest_xml_output_unittest_.cc:*\n'
 290                             u'Failed\nXML output: <?xml encoding="utf-8">'
 291                             u'<top><![CDATA[cdata text]]></top>' +
 292                             STACK_TRACE_TEMPLATE,
 293                 u'type': u''
 294             }]
 295         }]
 296     }, {
 297         u'name':
 298             u'InvalidCharactersTest',
 299         u'tests':
 300             1,
 301         u'failures':
 302             1,
 303         u'disabled':
 304             0,
 305         u'errors':
 306             0,
 307         u'time':
 308             u'*',
 309         u'timestamp':
 310             u'*',
 311         u'testsuite': [{
 312             u'name':
 313                 u'InvalidCharactersInMessage',
 314             u'status':
 315                 u'RUN',
 316             u'result':
 317                 u'COMPLETED',
 318             u'time':
 319                 u'*',
 320             u'timestamp':
 321                 u'*',
 322             u'classname':
 323                 u'InvalidCharactersTest',
 324             u'failures': [{
 325                 u'failure': u'gtest_xml_output_unittest_.cc:*\n'
 326                             u'Failed\nInvalid characters in brackets'
 327                             u' [\x01\x02]' + STACK_TRACE_TEMPLATE,
 328                 u'type': u''
 329             }]
 330         }]
 331     }, {
 332         u'name':
 333             u'PropertyRecordingTest',
 334         u'tests':
 335             4,
 336         u'failures':
 337             0,
 338         u'disabled':
 339             0,
 340         u'errors':
 341             0,
 342         u'time':
 343             u'*',
 344         u'timestamp':
 345             u'*',
 346         u'SetUpTestSuite':
 347             u'yes',
 348         u'TearDownTestSuite':
 349             u'aye',
 350         u'testsuite': [{
 351             u'name': u'OneProperty',
 352             u'status': u'RUN',
 353             u'result': u'COMPLETED',
 354             u'time': u'*',
 355             u'timestamp': u'*',
 356             u'classname': u'PropertyRecordingTest',
 357             u'key_1': u'1'
 358         }, {
 359             u'name': u'IntValuedProperty',
 360             u'status': u'RUN',
 361             u'result': u'COMPLETED',
 362             u'time': u'*',
 363             u'timestamp': u'*',
 364             u'classname': u'PropertyRecordingTest',
 365             u'key_int': u'1'
 366         }, {
 367             u'name': u'ThreeProperties',
 368             u'status': u'RUN',
 369             u'result': u'COMPLETED',
 370             u'time': u'*',
 371             u'timestamp': u'*',
 372             u'classname': u'PropertyRecordingTest',
 373             u'key_1': u'1',
 374             u'key_2': u'2',
 375             u'key_3': u'3'
 376         }, {
 377             u'name': u'TwoValuesForOneKeyUsesLastValue',
 378             u'status': u'RUN',
 379             u'result': u'COMPLETED',
 380             u'time': u'*',
 381             u'timestamp': u'*',
 382             u'classname': u'PropertyRecordingTest',
 383             u'key_1': u'2'
 384         }]
 385     }, {
 386         u'name':
 387             u'NoFixtureTest',
 388         u'tests':
 389             3,
 390         u'failures':
 391             0,
 392         u'disabled':
 393             0,
 394         u'errors':
 395             0,
 396         u'time':
 397             u'*',
 398         u'timestamp':
 399             u'*',
 400         u'testsuite': [{
 401             u'name': u'RecordProperty',
 402             u'status': u'RUN',
 403             u'result': u'COMPLETED',
 404             u'time': u'*',
 405             u'timestamp': u'*',
 406             u'classname': u'NoFixtureTest',
 407             u'key': u'1'
 408         }, {
 409             u'name': u'ExternalUtilityThatCallsRecordIntValuedProperty',
 410             u'status': u'RUN',
 411             u'result': u'COMPLETED',
 412             u'time': u'*',
 413             u'timestamp': u'*',
 414             u'classname': u'NoFixtureTest',
 415             u'key_for_utility_int': u'1'
 416         }, {
 417             u'name': u'ExternalUtilityThatCallsRecordStringValuedProperty',
 418             u'status': u'RUN',
 419             u'result': u'COMPLETED',
 420             u'time': u'*',
 421             u'timestamp': u'*',
 422             u'classname': u'NoFixtureTest',
 423             u'key_for_utility_string': u'1'
 424         }]
 425     }, {
 426         u'name':
 427             u'TypedTest/0',
 428         u'tests':
 429             1,
 430         u'failures':
 431             0,
 432         u'disabled':
 433             0,
 434         u'errors':
 435             0,
 436         u'time':
 437             u'*',
 438         u'timestamp':
 439             u'*',
 440         u'testsuite': [{
 441             u'name': u'HasTypeParamAttribute',
 442             u'type_param': u'int',
 443             u'status': u'RUN',
 444             u'result': u'COMPLETED',
 445             u'time': u'*',
 446             u'timestamp': u'*',
 447             u'classname': u'TypedTest/0'
 448         }]
 449     }, {
 450         u'name':
 451             u'TypedTest/1',
 452         u'tests':
 453             1,
 454         u'failures':
 455             0,
 456         u'disabled':
 457             0,
 458         u'errors':
 459             0,
 460         u'time':
 461             u'*',
 462         u'timestamp':
 463             u'*',
 464         u'testsuite': [{
 465             u'name': u'HasTypeParamAttribute',
 466             u'type_param': u'long',
 467             u'status': u'RUN',
 468             u'result': u'COMPLETED',
 469             u'time': u'*',
 470             u'timestamp': u'*',
 471             u'classname': u'TypedTest/1'
 472         }]
 473     }, {
 474         u'name':
 475             u'Single/TypeParameterizedTestSuite/0',
 476         u'tests':
 477             1,
 478         u'failures':
 479             0,
 480         u'disabled':
 481             0,
 482         u'errors':
 483             0,
 484         u'time':
 485             u'*',
 486         u'timestamp':
 487             u'*',
 488         u'testsuite': [{
 489             u'name': u'HasTypeParamAttribute',
 490             u'type_param': u'int',
 491             u'status': u'RUN',
 492             u'result': u'COMPLETED',
 493             u'time': u'*',
 494             u'timestamp': u'*',
 495             u'classname': u'Single/TypeParameterizedTestSuite/0'
 496         }]
 497     }, {
 498         u'name':
 499             u'Single/TypeParameterizedTestSuite/1',
 500         u'tests':
 501             1,
 502         u'failures':
 503             0,
 504         u'disabled':
 505             0,
 506         u'errors':
 507             0,
 508         u'time':
 509             u'*',
 510         u'timestamp':
 511             u'*',
 512         u'testsuite': [{
 513             u'name': u'HasTypeParamAttribute',
 514             u'type_param': u'long',
 515             u'status': u'RUN',
 516             u'result': u'COMPLETED',
 517             u'time': u'*',
 518             u'timestamp': u'*',
 519             u'classname': u'Single/TypeParameterizedTestSuite/1'
 520         }]
 521     }, {
 522         u'name':
 523             u'Single/ValueParamTest',
 524         u'tests':
 525             4,
 526         u'failures':
 527             0,
 528         u'disabled':
 529             0,
 530         u'errors':
 531             0,
 532         u'time':
 533             u'*',
 534         u'timestamp':
 535             u'*',
 536         u'testsuite': [{
 537             u'name': u'HasValueParamAttribute/0',
 538             u'value_param': u'33',
 539             u'status': u'RUN',
 540             u'result': u'COMPLETED',
 541             u'time': u'*',
 542             u'timestamp': u'*',
 543             u'classname': u'Single/ValueParamTest'
 544         }, {
 545             u'name': u'HasValueParamAttribute/1',
 546             u'value_param': u'42',
 547             u'status': u'RUN',
 548             u'result': u'COMPLETED',
 549             u'time': u'*',
 550             u'timestamp': u'*',
 551             u'classname': u'Single/ValueParamTest'
 552         }, {
 553             u'name': u'AnotherTestThatHasValueParamAttribute/0',
 554             u'value_param': u'33',
 555             u'status': u'RUN',
 556             u'result': u'COMPLETED',
 557             u'time': u'*',
 558             u'timestamp': u'*',
 559             u'classname': u'Single/ValueParamTest'
 560         }, {
 561             u'name': u'AnotherTestThatHasValueParamAttribute/1',
 562             u'value_param': u'42',
 563             u'status': u'RUN',
 564             u'result': u'COMPLETED',
 565             u'time': u'*',
 566             u'timestamp': u'*',
 567             u'classname': u'Single/ValueParamTest'
 568         }]
 569     }]
 570 }
 571
 572 EXPECTED_FILTERED = {
 573     u'tests':
 574         1,
 575     u'failures':
 576         0,
 577     u'disabled':
 578         0,
 579     u'errors':
 580         0,
 581     u'time':
 582         u'*',
 583     u'timestamp':
 584         u'*',
 585     u'name':
 586         u'AllTests',
 587     u'ad_hoc_property':
 588         u'42',
 589     u'testsuites': [{
 590         u'name':
 591             u'SuccessfulTest',
 592         u'tests':
 593             1,
 594         u'failures':
 595             0,
 596         u'disabled':
 597             0,
 598         u'errors':
 599             0,
 600         u'time':
 601             u'*',
 602         u'timestamp':
 603             u'*',
 604         u'testsuite': [{
 605             u'name': u'Succeeds',
 606             u'status': u'RUN',
 607             u'result': u'COMPLETED',
 608             u'time': u'*',
 609             u'timestamp': u'*',
 610             u'classname': u'SuccessfulTest',
 611         }]
 612     }],
 613 }
 614
 615 EXPECTED_EMPTY = {
 616     u'tests': 0,
 617     u'failures': 0,
 618     u'disabled': 0,
 619     u'errors': 0,
 620     u'time': u'*',
 621     u'timestamp': u'*',
 622     u'name': u'AllTests',
 623     u'testsuites': [],
 624 }
 625
 626 GTEST_PROGRAM_PATH = gtest_test_utils.GetTestExecutablePath(GTEST_PROGRAM_NAME)
 627
 628 SUPPORTS_TYPED_TESTS = 'TypedTest' in gtest_test_utils.Subprocess(
 629     [GTEST_PROGRAM_PATH, GTEST_LIST_TESTS_FLAG], capture_stderr=False).output
 630
 631
 632 class GTestJsonOutputUnitTest(gtest_test_utils.TestCase):
 633   """Unit test for Google Test's JSON output functionality.
 634   """
 635
 636   # This test currently breaks on platforms that do not support typed and
 637   # type-parameterized tests, so we don't run it under them.
 638   if SUPPORTS_TYPED_TESTS:
 639
 640     def testNonEmptyJsonOutput(self):
 641       """Verifies JSON output for a Google Test binary with non-empty output.
 642
 643       Runs a test program that generates a non-empty JSON output, and
 644       tests that the JSON output is expected.
 645       """
 646       self._TestJsonOutput(GTEST_PROGRAM_NAME, EXPECTED_NON_EMPTY, 1)
 647
 648   def testEmptyJsonOutput(self):
 649     """Verifies JSON output for a Google Test binary without actual tests.
 650
 651     Runs a test program that generates an empty JSON output, and
 652     tests that the JSON output is expected.
 653     """
 654
 655     self._TestJsonOutput('gtest_no_test_unittest', EXPECTED_EMPTY, 0)
 656
 657   def testTimestampValue(self):
 658     """Checks whether the timestamp attribute in the JSON output is valid.
 659
 660     Runs a test program that generates an empty JSON output, and checks if
 661     the timestamp attribute in the testsuites tag is valid.
 662     """
 663     actual = self._GetJsonOutput('gtest_no_test_unittest', [], 0)
 664     date_time_str = actual['timestamp']
 665     # datetime.strptime() is only available in Python 2.5+ so we have to
 666     # parse the expected datetime manually.
 667     match = re.match(r'(\d+)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)', date_time_str)
 668     self.assertTrue(
 669         re.match,
 670         'JSON datettime string %s has incorrect format' % date_time_str)
 671     date_time_from_json = datetime.datetime(
 672         year=int(match.group(1)), month=int(match.group(2)),
 673         day=int(match.group(3)), hour=int(match.group(4)),
 674         minute=int(match.group(5)), second=int(match.group(6)))
 675
 676     time_delta = abs(datetime.datetime.now() - date_time_from_json)
 677     # timestamp value should be near the current local time
 678     self.assertTrue(time_delta < datetime.timedelta(seconds=600),
 679                     'time_delta is %s' % time_delta)
 680
 681   def testDefaultOutputFile(self):
 682     """Verifies the default output file name.
 683
 684     Confirms that Google Test produces an JSON output file with the expected
 685     default name if no name is explicitly specified.
 686     """
 687     output_file = os.path.join(gtest_test_utils.GetTempDir(),
 688                                GTEST_DEFAULT_OUTPUT_FILE)
 689     gtest_prog_path = gtest_test_utils.GetTestExecutablePath(
 690         'gtest_no_test_unittest')
 691     try:
 692       os.remove(output_file)
 693     except OSError:
 694       e = sys.exc_info()[1]
 695       if e.errno != errno.ENOENT:
 696         raise
 697
 698     p = gtest_test_utils.Subprocess(
 699         [gtest_prog_path, '%s=json' % GTEST_OUTPUT_FLAG],
 700         working_dir=gtest_test_utils.GetTempDir())
 701     self.assert_(p.exited)
 702     self.assertEquals(0, p.exit_code)
 703     self.assert_(os.path.isfile(output_file))
 704
 705   def testSuppressedJsonOutput(self):
 706     """Verifies that no JSON output is generated.
 707
 708     Tests that no JSON file is generated if the default JSON listener is
 709     shut down before RUN_ALL_TESTS is invoked.
 710     """
 711
 712     json_path = os.path.join(gtest_test_utils.GetTempDir(),
 713                              GTEST_PROGRAM_NAME + 'out.json')
 714     if os.path.isfile(json_path):
 715       os.remove(json_path)
 716
 717     command = [GTEST_PROGRAM_PATH,
 718                '%s=json:%s' % (GTEST_OUTPUT_FLAG, json_path),
 719                '--shut_down_xml']
 720     p = gtest_test_utils.Subprocess(command)
 721     if p.terminated_by_signal:
 722       # p.signal is available only if p.terminated_by_signal is True.
 723       self.assertFalse(
 724           p.terminated_by_signal,
 725           '%s was killed by signal %d' % (GTEST_PROGRAM_NAME, p.signal))
 726     else:
 727       self.assert_(p.exited)
 728       self.assertEquals(1, p.exit_code,
 729                         "'%s' exited with code %s, which doesn't match "
 730                         'the expected exit code %s.'
 731                         % (command, p.exit_code, 1))
 732
 733     self.assert_(not os.path.isfile(json_path))
 734
 735   def testFilteredTestJsonOutput(self):
 736     """Verifies JSON output when a filter is applied.
 737
 738     Runs a test program that executes only some tests and verifies that
 739     non-selected tests do not show up in the JSON output.
 740     """
 741
 742     self._TestJsonOutput(GTEST_PROGRAM_NAME, EXPECTED_FILTERED, 0,
 743                          extra_args=['%s=SuccessfulTest.*' % GTEST_FILTER_FLAG])
 744
 745   def _GetJsonOutput(self, gtest_prog_name, extra_args, expected_exit_code):
 746     """Returns the JSON output generated by running the program gtest_prog_name.
 747
 748     Furthermore, the program's exit code must be expected_exit_code.
 749
 750     Args:
 751       gtest_prog_name: Google Test binary name.
 752       extra_args: extra arguments to binary invocation.
 753       expected_exit_code: program's exit code.
 754     """
 755     json_path = os.path.join(gtest_test_utils.GetTempDir(),
 756                              gtest_prog_name + 'out.json')
 757     gtest_prog_path = gtest_test_utils.GetTestExecutablePath(gtest_prog_name)
 758
 759     command = (
 760         [gtest_prog_path, '%s=json:%s' % (GTEST_OUTPUT_FLAG, json_path)] +
 761         extra_args
 762     )
 763     p = gtest_test_utils.Subprocess(command)
 764     if p.terminated_by_signal:
 765       self.assert_(False,
 766                    '%s was killed by signal %d' % (gtest_prog_name, p.signal))
 767     else:
 768       self.assert_(p.exited)
 769       self.assertEquals(expected_exit_code, p.exit_code,
 770                         "'%s' exited with code %s, which doesn't match "
 771                         'the expected exit code %s.'
 772                         % (command, p.exit_code, expected_exit_code))
 773     with open(json_path) as f:
 774       actual = json.load(f)
 775     return actual
 776
 777   def _TestJsonOutput(self, gtest_prog_name, expected,
 778                       expected_exit_code, extra_args=None):
 779     """Checks the JSON output generated by the Google Test binary.
 780
 781     Asserts that the JSON document generated by running the program
 782     gtest_prog_name matches expected_json, a string containing another
 783     JSON document.  Furthermore, the program's exit code must be
 784     expected_exit_code.
 785
 786     Args:
 787       gtest_prog_name: Google Test binary name.
 788       expected: expected output.
 789       expected_exit_code: program's exit code.
 790       extra_args: extra arguments to binary invocation.
 791     """
 792
 793     actual = self._GetJsonOutput(gtest_prog_name, extra_args or [],
 794                                  expected_exit_code)
 795     self.assertEqual(expected, gtest_json_test_utils.normalize(actual))
 796
 797
 798 if __name__ == '__main__':
 799   if NO_STACKTRACE_SUPPORT_FLAG in sys.argv:
 800     # unittest.main() can't handle unknown flags
 801     sys.argv.remove(NO_STACKTRACE_SUPPORT_FLAG)
 802
 803   os.environ['GTEST_STACK_TRACE_DEPTH'] = '1'
 804   gtest_test_utils.Main()