ceph/src/s3select/rapidjson/thirdparty/gtest/googletest/test/gtest_json_output_unittest.py

   1 #!/usr/bin/env python
   2 # Copyright 2018, Google Inc.
   3 # All rights reserved.
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 #     * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #     * Redistributions in binary form must reproduce the above
  12 # copyright notice, this list of conditions and the following disclaimer
  13 # in the documentation and/or other materials provided with the
  14 # distribution.
  15 #     * Neither the name of Google Inc. nor the names of its
  16 # contributors may be used to endorse or promote products derived from
  17 # this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31 """Unit test for the gtest_json_output module."""
  32
  33 import datetime
  34 import errno
  35 import json
  36 import os
  37 import re
  38 import sys
  39
  40 import gtest_json_test_utils
  41 import gtest_test_utils
  42
  43 GTEST_FILTER_FLAG = '--gtest_filter'
  44 GTEST_LIST_TESTS_FLAG = '--gtest_list_tests'
  45 GTEST_OUTPUT_FLAG = '--gtest_output'
  46 GTEST_DEFAULT_OUTPUT_FILE = 'test_detail.json'
  47 GTEST_PROGRAM_NAME = 'gtest_xml_output_unittest_'
  48
  49 SUPPORTS_STACK_TRACES = False
  50
  51 if SUPPORTS_STACK_TRACES:
  52   STACK_TRACE_TEMPLATE = '\nStack trace:\n*'
  53 else:
  54   STACK_TRACE_TEMPLATE = ''
  55
  56 EXPECTED_NON_EMPTY = {
  57     u'tests': 23,
  58     u'failures': 4,
  59     u'disabled': 2,
  60     u'errors': 0,
  61     u'timestamp': u'*',
  62     u'time': u'*',
  63     u'ad_hoc_property': u'42',
  64     u'name': u'AllTests',
  65     u'testsuites': [
  66         {
  67             u'name': u'SuccessfulTest',
  68             u'tests': 1,
  69             u'failures': 0,
  70             u'disabled': 0,
  71             u'errors': 0,
  72             u'time': u'*',
  73             u'testsuite': [
  74                 {
  75                     u'name': u'Succeeds',
  76                     u'status': u'RUN',
  77                     u'time': u'*',
  78                     u'classname': u'SuccessfulTest'
  79                 }
  80             ]
  81         },
  82         {
  83             u'name': u'FailedTest',
  84             u'tests': 1,
  85             u'failures': 1,
  86             u'disabled': 0,
  87             u'errors': 0,
  88             u'time': u'*',
  89             u'testsuite': [
  90                 {
  91                     u'name': u'Fails',
  92                     u'status': u'RUN',
  93                     u'time': u'*',
  94                     u'classname': u'FailedTest',
  95                     u'failures': [
  96                         {
  97                             u'failure':
  98                                 u'gtest_xml_output_unittest_.cc:*\n'
  99                                 u'Expected equality of these values:\n'
 100                                 u'  1\n  2' + STACK_TRACE_TEMPLATE,
 101                             u'type': u''
 102                         }
 103                     ]
 104                 }
 105             ]
 106         },
 107         {
 108             u'name': u'DisabledTest',
 109             u'tests': 1,
 110             u'failures': 0,
 111             u'disabled': 1,
 112             u'errors': 0,
 113             u'time': u'*',
 114             u'testsuite': [
 115                 {
 116                     u'name': u'DISABLED_test_not_run',
 117                     u'status': u'NOTRUN',
 118                     u'time': u'*',
 119                     u'classname': u'DisabledTest'
 120                 }
 121             ]
 122         },
 123         {
 124             u'name': u'MixedResultTest',
 125             u'tests': 3,
 126             u'failures': 1,
 127             u'disabled': 1,
 128             u'errors': 0,
 129             u'time': u'*',
 130             u'testsuite': [
 131                 {
 132                     u'name': u'Succeeds',
 133                     u'status': u'RUN',
 134                     u'time': u'*',
 135                     u'classname': u'MixedResultTest'
 136                 },
 137                 {
 138                     u'name': u'Fails',
 139                     u'status': u'RUN',
 140                     u'time': u'*',
 141                     u'classname': u'MixedResultTest',
 142                     u'failures': [
 143                         {
 144                             u'failure':
 145                                 u'gtest_xml_output_unittest_.cc:*\n'
 146                                 u'Expected equality of these values:\n'
 147                                 u'  1\n  2' + STACK_TRACE_TEMPLATE,
 148                             u'type': u''
 149                         },
 150                         {
 151                             u'failure':
 152                                 u'gtest_xml_output_unittest_.cc:*\n'
 153                                 u'Expected equality of these values:\n'
 154                                 u'  2\n  3' + STACK_TRACE_TEMPLATE,
 155                             u'type': u''
 156                         }
 157                     ]
 158                 },
 159                 {
 160                     u'name': u'DISABLED_test',
 161                     u'status': u'NOTRUN',
 162                     u'time': u'*',
 163                     u'classname': u'MixedResultTest'
 164                 }
 165             ]
 166         },
 167         {
 168             u'name': u'XmlQuotingTest',
 169             u'tests': 1,
 170             u'failures': 1,
 171             u'disabled': 0,
 172             u'errors': 0,
 173             u'time': u'*',
 174             u'testsuite': [
 175                 {
 176                     u'name': u'OutputsCData',
 177                     u'status': u'RUN',
 178                     u'time': u'*',
 179                     u'classname': u'XmlQuotingTest',
 180                     u'failures': [
 181                         {
 182                             u'failure':
 183                                 u'gtest_xml_output_unittest_.cc:*\n'
 184                                 u'Failed\nXML output: <?xml encoding="utf-8">'
 185                                 u'<top><![CDATA[cdata text]]></top>' +
 186                                 STACK_TRACE_TEMPLATE,
 187                             u'type': u''
 188                         }
 189                     ]
 190                 }
 191             ]
 192         },
 193         {
 194             u'name': u'InvalidCharactersTest',
 195             u'tests': 1,
 196             u'failures': 1,
 197             u'disabled': 0,
 198             u'errors': 0,
 199             u'time': u'*',
 200             u'testsuite': [
 201                 {
 202                     u'name': u'InvalidCharactersInMessage',
 203                     u'status': u'RUN',
 204                     u'time': u'*',
 205                     u'classname': u'InvalidCharactersTest',
 206                     u'failures': [
 207                         {
 208                             u'failure':
 209                                 u'gtest_xml_output_unittest_.cc:*\n'
 210                                 u'Failed\nInvalid characters in brackets'
 211                                 u' [\x01\x02]' + STACK_TRACE_TEMPLATE,
 212                             u'type': u''
 213                         }
 214                     ]
 215                 }
 216             ]
 217         },
 218         {
 219             u'name': u'PropertyRecordingTest',
 220             u'tests': 4,
 221             u'failures': 0,
 222             u'disabled': 0,
 223             u'errors': 0,
 224             u'time': u'*',
 225             u'SetUpTestCase': u'yes',
 226             u'TearDownTestCase': u'aye',
 227             u'testsuite': [
 228                 {
 229                     u'name': u'OneProperty',
 230                     u'status': u'RUN',
 231                     u'time': u'*',
 232                     u'classname': u'PropertyRecordingTest',
 233                     u'key_1': u'1'
 234                 },
 235                 {
 236                     u'name': u'IntValuedProperty',
 237                     u'status': u'RUN',
 238                     u'time': u'*',
 239                     u'classname': u'PropertyRecordingTest',
 240                     u'key_int': u'1'
 241                 },
 242                 {
 243                     u'name': u'ThreeProperties',
 244                     u'status': u'RUN',
 245                     u'time': u'*',
 246                     u'classname': u'PropertyRecordingTest',
 247                     u'key_1': u'1',
 248                     u'key_2': u'2',
 249                     u'key_3': u'3'
 250                 },
 251                 {
 252                     u'name': u'TwoValuesForOneKeyUsesLastValue',
 253                     u'status': u'RUN',
 254                     u'time': u'*',
 255                     u'classname': u'PropertyRecordingTest',
 256                     u'key_1': u'2'
 257                 }
 258             ]
 259         },
 260         {
 261             u'name': u'NoFixtureTest',
 262             u'tests': 3,
 263             u'failures': 0,
 264             u'disabled': 0,
 265             u'errors': 0,
 266             u'time': u'*',
 267             u'testsuite': [
 268                 {
 269                     u'name': u'RecordProperty',
 270                     u'status': u'RUN',
 271                     u'time': u'*',
 272                     u'classname': u'NoFixtureTest',
 273                     u'key': u'1'
 274                 },
 275                 {
 276                     u'name': u'ExternalUtilityThatCallsRecordIntValuedProperty',
 277                     u'status': u'RUN',
 278                     u'time': u'*',
 279                     u'classname': u'NoFixtureTest',
 280                     u'key_for_utility_int': u'1'
 281                 },
 282                 {
 283                     u'name':
 284                         u'ExternalUtilityThatCallsRecordStringValuedProperty',
 285                     u'status': u'RUN',
 286                     u'time': u'*',
 287                     u'classname': u'NoFixtureTest',
 288                     u'key_for_utility_string': u'1'
 289                 }
 290             ]
 291         },
 292         {
 293             u'name': u'TypedTest/0',
 294             u'tests': 1,
 295             u'failures': 0,
 296             u'disabled': 0,
 297             u'errors': 0,
 298             u'time': u'*',
 299             u'testsuite': [
 300                 {
 301                     u'name': u'HasTypeParamAttribute',
 302                     u'type_param': u'int',
 303                     u'status': u'RUN',
 304                     u'time': u'*',
 305                     u'classname': u'TypedTest/0'
 306                 }
 307             ]
 308         },
 309         {
 310             u'name': u'TypedTest/1',
 311             u'tests': 1,
 312             u'failures': 0,
 313             u'disabled': 0,
 314             u'errors': 0,
 315             u'time': u'*',
 316             u'testsuite': [
 317                 {
 318                     u'name': u'HasTypeParamAttribute',
 319                     u'type_param': u'long',
 320                     u'status': u'RUN',
 321                     u'time': u'*',
 322                     u'classname': u'TypedTest/1'
 323                 }
 324             ]
 325         },
 326         {
 327             u'name': u'Single/TypeParameterizedTestCase/0',
 328             u'tests': 1,
 329             u'failures': 0,
 330             u'disabled': 0,
 331             u'errors': 0,
 332             u'time': u'*',
 333             u'testsuite': [
 334                 {
 335                     u'name': u'HasTypeParamAttribute',
 336                     u'type_param': u'int',
 337                     u'status': u'RUN',
 338                     u'time': u'*',
 339                     u'classname': u'Single/TypeParameterizedTestCase/0'
 340                 }
 341             ]
 342         },
 343         {
 344             u'name': u'Single/TypeParameterizedTestCase/1',
 345             u'tests': 1,
 346             u'failures': 0,
 347             u'disabled': 0,
 348             u'errors': 0,
 349             u'time': u'*',
 350             u'testsuite': [
 351                 {
 352                     u'name': u'HasTypeParamAttribute',
 353                     u'type_param': u'long',
 354                     u'status': u'RUN',
 355                     u'time': u'*',
 356                     u'classname': u'Single/TypeParameterizedTestCase/1'
 357                 }
 358             ]
 359         },
 360         {
 361             u'name': u'Single/ValueParamTest',
 362             u'tests': 4,
 363             u'failures': 0,
 364             u'disabled': 0,
 365             u'errors': 0,
 366             u'time': u'*',
 367             u'testsuite': [
 368                 {
 369                     u'name': u'HasValueParamAttribute/0',
 370                     u'value_param': u'33',
 371                     u'status': u'RUN',
 372                     u'time': u'*',
 373                     u'classname': u'Single/ValueParamTest'
 374                 },
 375                 {
 376                     u'name': u'HasValueParamAttribute/1',
 377                     u'value_param': u'42',
 378                     u'status': u'RUN',
 379                     u'time': u'*',
 380                     u'classname': u'Single/ValueParamTest'
 381                 },
 382                 {
 383                     u'name': u'AnotherTestThatHasValueParamAttribute/0',
 384                     u'value_param': u'33',
 385                     u'status': u'RUN',
 386                     u'time': u'*',
 387                     u'classname': u'Single/ValueParamTest'
 388                 },
 389                 {
 390                     u'name': u'AnotherTestThatHasValueParamAttribute/1',
 391                     u'value_param': u'42',
 392                     u'status': u'RUN',
 393                     u'time': u'*',
 394                     u'classname': u'Single/ValueParamTest'
 395                 }
 396             ]
 397         }
 398     ]
 399 }
 400
 401 EXPECTED_FILTERED = {
 402     u'tests': 1,
 403     u'failures': 0,
 404     u'disabled': 0,
 405     u'errors': 0,
 406     u'time': u'*',
 407     u'timestamp': u'*',
 408     u'name': u'AllTests',
 409     u'ad_hoc_property': u'42',
 410     u'testsuites': [{
 411         u'name': u'SuccessfulTest',
 412         u'tests': 1,
 413         u'failures': 0,
 414         u'disabled': 0,
 415         u'errors': 0,
 416         u'time': u'*',
 417         u'testsuite': [{
 418             u'name': u'Succeeds',
 419             u'status': u'RUN',
 420             u'time': u'*',
 421             u'classname': u'SuccessfulTest',
 422         }]
 423     }],
 424 }
 425
 426 EXPECTED_EMPTY = {
 427     u'tests': 0,
 428     u'failures': 0,
 429     u'disabled': 0,
 430     u'errors': 0,
 431     u'time': u'*',
 432     u'timestamp': u'*',
 433     u'name': u'AllTests',
 434     u'testsuites': [],
 435 }
 436
 437 GTEST_PROGRAM_PATH = gtest_test_utils.GetTestExecutablePath(GTEST_PROGRAM_NAME)
 438
 439 SUPPORTS_TYPED_TESTS = 'TypedTest' in gtest_test_utils.Subprocess(
 440     [GTEST_PROGRAM_PATH, GTEST_LIST_TESTS_FLAG], capture_stderr=False).output
 441
 442
 443 class GTestJsonOutputUnitTest(gtest_test_utils.TestCase):
 444   """Unit test for Google Test's JSON output functionality.
 445   """
 446
 447   # This test currently breaks on platforms that do not support typed and
 448   # type-parameterized tests, so we don't run it under them.
 449   if SUPPORTS_TYPED_TESTS:
 450
 451     def testNonEmptyJsonOutput(self):
 452       """Verifies JSON output for a Google Test binary with non-empty output.
 453
 454       Runs a test program that generates a non-empty JSON output, and
 455       tests that the JSON output is expected.
 456       """
 457       self._TestJsonOutput(GTEST_PROGRAM_NAME, EXPECTED_NON_EMPTY, 1)
 458
 459   def testEmptyJsonOutput(self):
 460     """Verifies JSON output for a Google Test binary without actual tests.
 461
 462     Runs a test program that generates an empty JSON output, and
 463     tests that the JSON output is expected.
 464     """
 465
 466     self._TestJsonOutput('gtest_no_test_unittest', EXPECTED_EMPTY, 0)
 467
 468   def testTimestampValue(self):
 469     """Checks whether the timestamp attribute in the JSON output is valid.
 470
 471     Runs a test program that generates an empty JSON output, and checks if
 472     the timestamp attribute in the testsuites tag is valid.
 473     """
 474     actual = self._GetJsonOutput('gtest_no_test_unittest', [], 0)
 475     date_time_str = actual['timestamp']
 476     # datetime.strptime() is only available in Python 2.5+ so we have to
 477     # parse the expected datetime manually.
 478     match = re.match(r'(\d+)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)', date_time_str)
 479     self.assertTrue(
 480         re.match,
 481         'JSON datettime string %s has incorrect format' % date_time_str)
 482     date_time_from_json = datetime.datetime(
 483         year=int(match.group(1)), month=int(match.group(2)),
 484         day=int(match.group(3)), hour=int(match.group(4)),
 485         minute=int(match.group(5)), second=int(match.group(6)))
 486
 487     time_delta = abs(datetime.datetime.now() - date_time_from_json)
 488     # timestamp value should be near the current local time
 489     self.assertTrue(time_delta < datetime.timedelta(seconds=600),
 490                     'time_delta is %s' % time_delta)
 491
 492   def testDefaultOutputFile(self):
 493     """Verifies the default output file name.
 494
 495     Confirms that Google Test produces an JSON output file with the expected
 496     default name if no name is explicitly specified.
 497     """
 498     output_file = os.path.join(gtest_test_utils.GetTempDir(),
 499                                GTEST_DEFAULT_OUTPUT_FILE)
 500     gtest_prog_path = gtest_test_utils.GetTestExecutablePath(
 501         'gtest_no_test_unittest')
 502     try:
 503       os.remove(output_file)
 504     except OSError:
 505       e = sys.exc_info()[1]
 506       if e.errno != errno.ENOENT:
 507         raise
 508
 509     p = gtest_test_utils.Subprocess(
 510         [gtest_prog_path, '%s=json' % GTEST_OUTPUT_FLAG],
 511         working_dir=gtest_test_utils.GetTempDir())
 512     self.assert_(p.exited)
 513     self.assertEquals(0, p.exit_code)
 514     self.assert_(os.path.isfile(output_file))
 515
 516   def testSuppressedJsonOutput(self):
 517     """Verifies that no JSON output is generated.
 518
 519     Tests that no JSON file is generated if the default JSON listener is
 520     shut down before RUN_ALL_TESTS is invoked.
 521     """
 522
 523     json_path = os.path.join(gtest_test_utils.GetTempDir(),
 524                              GTEST_PROGRAM_NAME + 'out.json')
 525     if os.path.isfile(json_path):
 526       os.remove(json_path)
 527
 528     command = [GTEST_PROGRAM_PATH,
 529                '%s=json:%s' % (GTEST_OUTPUT_FLAG, json_path),
 530                '--shut_down_xml']
 531     p = gtest_test_utils.Subprocess(command)
 532     if p.terminated_by_signal:
 533       # p.signal is available only if p.terminated_by_signal is True.
 534       self.assertFalse(
 535           p.terminated_by_signal,
 536           '%s was killed by signal %d' % (GTEST_PROGRAM_NAME, p.signal))
 537     else:
 538       self.assert_(p.exited)
 539       self.assertEquals(1, p.exit_code,
 540                         "'%s' exited with code %s, which doesn't match "
 541                         'the expected exit code %s.'
 542                         % (command, p.exit_code, 1))
 543
 544     self.assert_(not os.path.isfile(json_path))
 545
 546   def testFilteredTestJsonOutput(self):
 547     """Verifies JSON output when a filter is applied.
 548
 549     Runs a test program that executes only some tests and verifies that
 550     non-selected tests do not show up in the JSON output.
 551     """
 552
 553     self._TestJsonOutput(GTEST_PROGRAM_NAME, EXPECTED_FILTERED, 0,
 554                          extra_args=['%s=SuccessfulTest.*' % GTEST_FILTER_FLAG])
 555
 556   def _GetJsonOutput(self, gtest_prog_name, extra_args, expected_exit_code):
 557     """Returns the JSON output generated by running the program gtest_prog_name.
 558
 559     Furthermore, the program's exit code must be expected_exit_code.
 560
 561     Args:
 562       gtest_prog_name: Google Test binary name.
 563       extra_args: extra arguments to binary invocation.
 564       expected_exit_code: program's exit code.
 565     """
 566     json_path = os.path.join(gtest_test_utils.GetTempDir(),
 567                              gtest_prog_name + 'out.json')
 568     gtest_prog_path = gtest_test_utils.GetTestExecutablePath(gtest_prog_name)
 569
 570     command = (
 571         [gtest_prog_path, '%s=json:%s' % (GTEST_OUTPUT_FLAG, json_path)] +
 572         extra_args
 573     )
 574     p = gtest_test_utils.Subprocess(command)
 575     if p.terminated_by_signal:
 576       self.assert_(False,
 577                    '%s was killed by signal %d' % (gtest_prog_name, p.signal))
 578     else:
 579       self.assert_(p.exited)
 580       self.assertEquals(expected_exit_code, p.exit_code,
 581                         "'%s' exited with code %s, which doesn't match "
 582                         'the expected exit code %s.'
 583                         % (command, p.exit_code, expected_exit_code))
 584     with open(json_path) as f:
 585       actual = json.load(f)
 586     return actual
 587
 588   def _TestJsonOutput(self, gtest_prog_name, expected,
 589                       expected_exit_code, extra_args=None):
 590     """Checks the JSON output generated by the Google Test binary.
 591
 592     Asserts that the JSON document generated by running the program
 593     gtest_prog_name matches expected_json, a string containing another
 594     JSON document.  Furthermore, the program's exit code must be
 595     expected_exit_code.
 596
 597     Args:
 598       gtest_prog_name: Google Test binary name.
 599       expected: expected output.
 600       expected_exit_code: program's exit code.
 601       extra_args: extra arguments to binary invocation.
 602     """
 603
 604     actual = self._GetJsonOutput(gtest_prog_name, extra_args or [],
 605                                  expected_exit_code)
 606     self.assertEqual(expected, gtest_json_test_utils.normalize(actual))
 607
 608
 609 if __name__ == '__main__':
 610   os.environ['GTEST_STACK_TRACE_DEPTH'] = '1'
 611   gtest_test_utils.Main()