tests/test-runner/bin/zts-report.py

   1 #!/usr/bin/python
   2
   3 #
   4 # This file and its contents are supplied under the terms of the
   5 # Common Development and Distribution License ("CDDL"), version 1.0.
   6 # You may only use this file in accordance with the terms of version
   7 # 1.0 of the CDDL.
   8 #
   9 # A full copy of the text of the CDDL should have accompanied this
  10 # source.  A copy of the CDDL is also available via the Internet at
  11 # http://www.illumos.org/license/CDDL.
  12 #
  13
  14 #
  15 # Copyright (c) 2017 by Delphix. All rights reserved.
  16 # Copyright (c) 2018 by Lawrence Livermore National Security, LLC.
  17 #
  18
  19 import os
  20 import re
  21 import sys
  22
  23 #
  24 # This script parses the stdout of zfstest, which has this format:
  25 #
  26 # Test: /path/to/testa (run as root) [00:00] [PASS]
  27 # Test: /path/to/testb (run as jkennedy) [00:00] [PASS]
  28 # Test: /path/to/testc (run as root) [00:00] [FAIL]
  29 # [...many more results...]
  30 #
  31 # Results Summary
  32 # FAIL      22
  33 # SKIP      32
  34 # PASS    1156
  35 #
  36 # Running Time:   02:50:31
  37 # Percent passed: 95.5%
  38 # Log directory:  /var/tmp/test_results/20180615T205926
  39 #
  40
  41 #
  42 # Common generic reasons for a test or test group to be skipped.
  43 #
  44 # Some test cases are known to fail in ways which are not harmful or dangerous.
  45 # In these cases simply mark the test as a known failure until it can be
  46 # updated and the issue resolved.  Note that it's preferable to open a unique
  47 # issue on the GitHub issue tracker for each test case failure.
  48 #
  49 known_reason = 'Known issue'
  50
  51 #
  52 # Some tests require that a test user be able to execute the zfs utilities.
  53 # This may not be possible when testing in-tree due to the default permissions
  54 # on the user's home directory.  When testing this can be resolved by granting
  55 # group read access.
  56 #
  57 # chmod 0750 $HOME
  58 #
  59 exec_reason = 'Test user execute permissions required for utilities'
  60
  61 #
  62 # Some tests require that the DISKS provided can be partitioned.  This is
  63 # normally not an issue because loop back devices are used for DISKS and they
  64 # can be partition.  There is one notable exception, the CentOS 6.x kernel is
  65 # old enough that it does not support partitioning loop back devices.
  66 #
  67 disk_reason = 'Partitionable DISKS required'
  68
  69 #
  70 # Some tests require a minimum python version of 3.5 and will be skipped when
  71 # the default system version is too old.  There may also be tests which require
  72 # additional python modules be installed, for example python-cffi is required
  73 # by the pyzfs tests.
  74 #
  75 python_reason = 'Python v3.5 or newer required'
  76 python_deps_reason = 'Python modules missing: python-cffi'
  77
  78 #
  79 # Some tests require the O_TMPFILE flag which was first introduced in the
  80 # 3.11 kernel.
  81 #
  82 tmpfile_reason = 'Kernel O_TMPFILE support required'
  83
  84 #
  85 # Some tests may depend on udev change events being generated when block
  86 # devices change capacity.  This functionality wasn't available until the
  87 # 2.6.38 kernel.
  88 #
  89 udev_reason = 'Kernel block device udev change events required'
  90
  91 #
  92 # Some tests require that the NFS client and server utilities be installed.
  93 #
  94 share_reason = 'NFS client and server utilities required'
  95
  96 #
  97 # Some tests require that the lsattr utility support the project id feature.
  98 #
  99 project_id_reason = 'lsattr with set/show project ID required'
 100
 101 #
 102 # Some tests require that the kernel support user namespaces.
 103 #
 104 user_ns_reason = 'Kernel user namespace support required'
 105
 106 #
 107 # Some rewind tests can fail since nothing guarantees that old MOS blocks
 108 # are not overwritten.  Snapshots protect datasets and data files but not
 109 # the MOS.  Reasonable efforts are made in the test case to increase the
 110 # odds that some txgs will have their MOS data left untouched, but it is
 111 # never a sure thing.
 112 #
 113 rewind_reason = 'Arbitrary pool rewind is not guaranteed'
 114
 115 #
 116 # Some tests may by structured in a way that relies on exact knowledge
 117 # of how much free space in available in a pool.  These tests cannot be
 118 # made completely reliable because the internal details of how free space
 119 # is managed are not exposed to user space.
 120 #
 121 enospc_reason = 'Exact free space reporting is not guaranteed'
 122
 123 #
 124 # Some tests require a minimum version of the fio benchmark utility.
 125 # Older distributions such as CentOS 6.x only provide fio-2.0.13.
 126 #
 127 fio_reason = 'Fio v2.3 or newer required'
 128
 129 #
 130 # Some tests are not applicable to Linux or need to be updated to operate
 131 # in the manor required by Linux.  Any tests which are skipped for this
 132 # reason will be suppressed in the final analysis output.
 133 #
 134 na_reason = "N/A on Linux"
 135
 136 summary = {
 137     'total': float(0),
 138     'passed': float(0),
 139     'logfile': "Could not determine logfile location."
 140 }
 141
 142 #
 143 # These tests are known to fail, thus we use this list to prevent these
 144 # failures from failing the job as a whole; only unexpected failures
 145 # bubble up to cause this script to exit with a non-zero exit status.
 146 #
 147 # Format: { 'test-name': ['expected result', 'issue-number | reason'] }
 148 #
 149 # For each known failure it is recommended to link to a GitHub issue by
 150 # setting the reason to the issue number.  Alternately, one of the generic
 151 # reasons listed above can be used.
 152 #
 153 known = {
 154     'casenorm/sensitive_none_lookup': ['FAIL', '7633'],
 155     'casenorm/sensitive_none_delete': ['FAIL', '7633'],
 156     'casenorm/sensitive_formd_lookup': ['FAIL', '7633'],
 157     'casenorm/sensitive_formd_delete': ['FAIL', '7633'],
 158     'casenorm/insensitive_none_lookup': ['FAIL', '7633'],
 159     'casenorm/insensitive_none_delete': ['FAIL', '7633'],
 160     'casenorm/insensitive_formd_lookup': ['FAIL', '7633'],
 161     'casenorm/insensitive_formd_delete': ['FAIL', '7633'],
 162     'casenorm/mixed_none_lookup': ['FAIL', '7633'],
 163     'casenorm/mixed_none_lookup_ci': ['FAIL', '7633'],
 164     'casenorm/mixed_none_delete': ['FAIL', '7633'],
 165     'casenorm/mixed_formd_lookup': ['FAIL', '7633'],
 166     'casenorm/mixed_formd_lookup_ci': ['FAIL', '7633'],
 167     'casenorm/mixed_formd_delete': ['FAIL', '7633'],
 168     'cli_root/zfs_receive/zfs_receive_004_neg': ['FAIL', known_reason],
 169     'cli_root/zfs_unshare/zfs_unshare_002_pos': ['SKIP', na_reason],
 170     'cli_root/zfs_unshare/zfs_unshare_006_pos': ['SKIP', na_reason],
 171     'cli_root/zpool_create/zpool_create_016_pos': ['SKIP', na_reason],
 172     'cli_user/misc/zfs_share_001_neg': ['SKIP', na_reason],
 173     'cli_user/misc/zfs_unshare_001_neg': ['SKIP', na_reason],
 174     'inuse/inuse_001_pos': ['SKIP', na_reason],
 175     'inuse/inuse_003_pos': ['SKIP', na_reason],
 176     'inuse/inuse_006_pos': ['SKIP', na_reason],
 177     'inuse/inuse_007_pos': ['SKIP', na_reason],
 178     'privilege/setup': ['SKIP', na_reason],
 179     'refreserv/refreserv_004_pos': ['FAIL', known_reason],
 180     'removal/removal_condense_export': ['SKIP', known_reason],
 181     'removal/removal_with_zdb': ['SKIP', known_reason],
 182     'rootpool/setup': ['SKIP', na_reason],
 183     'rsend/rsend_008_pos': ['SKIP', '6066'],
 184     'snapshot/rollback_003_pos': ['SKIP', '6143'],
 185     'vdev_zaps/vdev_zaps_007_pos': ['FAIL', known_reason],
 186     'xattr/xattr_008_pos': ['SKIP', na_reason],
 187     'xattr/xattr_009_neg': ['SKIP', na_reason],
 188     'xattr/xattr_010_neg': ['SKIP', na_reason],
 189     'zvol/zvol_misc/zvol_misc_001_neg': ['SKIP', na_reason],
 190     'zvol/zvol_misc/zvol_misc_003_neg': ['SKIP', na_reason],
 191     'zvol/zvol_misc/zvol_misc_004_pos': ['SKIP', na_reason],
 192     'zvol/zvol_misc/zvol_misc_005_neg': ['SKIP', na_reason],
 193     'zvol/zvol_misc/zvol_misc_006_pos': ['SKIP', na_reason],
 194     'zvol/zvol_swap/zvol_swap_003_pos': ['SKIP', na_reason],
 195     'zvol/zvol_swap/zvol_swap_005_pos': ['SKIP', na_reason],
 196     'zvol/zvol_swap/zvol_swap_006_pos': ['SKIP', na_reason],
 197 }
 198
 199 #
 200 # These tests may occasionally fail or be skipped.  We want there failures
 201 # to be reported but only unexpected failures should bubble up to cause
 202 # this script to exit with a non-zero exit status.
 203 #
 204 # Format: { 'test-name': ['expected result', 'issue-number | reason'] }
 205 #
 206 # For each known failure it is recommended to link to a GitHub issue by
 207 # setting the reason to the issue number.  Alternately, one of the generic
 208 # reasons listed above can be used.
 209 #
 210 maybe = {
 211     'cache/setup': ['SKIP', disk_reason],
 212     'cache/cache_010_neg': ['FAIL', known_reason],
 213     'chattr/setup': ['SKIP', exec_reason],
 214     'cli_root/zdb/zdb_006_pos': ['FAIL', known_reason],
 215     'cli_root/zfs_get/zfs_get_004_pos': ['FAIL', known_reason],
 216     'cli_root/zfs_get/zfs_get_009_pos': ['SKIP', '5479'],
 217     'cli_root/zfs_rollback/zfs_rollback_001_pos': ['FAIL', '6415'],
 218     'cli_root/zfs_rollback/zfs_rollback_002_pos': ['FAIL', '6416'],
 219     'cli_root/zfs_share/setup': ['SKIP', share_reason],
 220     'cli_root/zfs_snapshot/zfs_snapshot_002_neg': ['FAIL', known_reason],
 221     'cli_root/zfs_unshare/setup': ['SKIP', share_reason],
 222     'cli_root/zpool_add/setup': ['SKIP', disk_reason],
 223     'cli_root/zpool_add/zpool_add_004_pos': ['FAIL', known_reason],
 224     'cli_root/zpool_create/setup': ['SKIP', disk_reason],
 225     'cli_root/zpool_create/zpool_create_008_pos': ['FAIL', known_reason],
 226     'cli_root/zpool_destroy/zpool_destroy_001_pos': ['SKIP', '6145'],
 227     'cli_root/zpool_expand/setup': ['SKIP', udev_reason],
 228     'cli_root/zpool_export/setup': ['SKIP', disk_reason],
 229     'cli_root/zpool_import/setup': ['SKIP', disk_reason],
 230     'cli_root/zpool_import/import_rewind_device_replaced':
 231         ['FAIL', rewind_reason],
 232     'cli_root/zpool_import/import_rewind_config_changed':
 233         ['FAIL', rewind_reason],
 234     'cli_root/zpool_import/zpool_import_missing_003_pos': ['SKIP', '6839'],
 235     'cli_root/zpool_remove/setup': ['SKIP', disk_reason],
 236     'cli_root/zpool_upgrade/zpool_upgrade_004_pos': ['FAIL', '6141'],
 237     'cli_user/misc/arc_summary3_001_pos': ['SKIP', python_reason],
 238     'delegate/setup': ['SKIP', exec_reason],
 239     'fault/auto_online_001_pos': ['SKIP', disk_reason],
 240     'fault/auto_replace_001_pos': ['SKIP', disk_reason],
 241     'history/history_004_pos': ['FAIL', '7026'],
 242     'history/history_005_neg': ['FAIL', '6680'],
 243     'history/history_006_neg': ['FAIL', '5657'],
 244     'history/history_008_pos': ['FAIL', known_reason],
 245     'history/history_010_pos': ['SKIP', exec_reason],
 246     'inuse/inuse_005_pos': ['SKIP', disk_reason],
 247     'inuse/inuse_008_pos': ['SKIP', disk_reason],
 248     'inuse/inuse_009_pos': ['SKIP', disk_reason],
 249     'io/mmap': ['SKIP', fio_reason],
 250     'largest_pool/largest_pool_001_pos': ['FAIL', known_reason],
 251     'pyzfs/pyzfs_unittest': ['SKIP', python_deps_reason],
 252     'no_space/enospc_002_pos': ['FAIL', enospc_reason],
 253     'projectquota/setup': ['SKIP', exec_reason],
 254     'redundancy/redundancy_004_neg': ['FAIL', '7290'],
 255     'reservation/reservation_008_pos': ['FAIL', '7741'],
 256     'reservation/reservation_018_pos': ['FAIL', '5642'],
 257     'rsend/rsend_019_pos': ['FAIL', '6086'],
 258     'rsend/rsend_020_pos': ['FAIL', '6446'],
 259     'rsend/rsend_021_pos': ['FAIL', '6446'],
 260     'rsend/rsend_024_pos': ['FAIL', '5665'],
 261     'rsend/send-c_volume': ['FAIL', '6087'],
 262     'snapshot/clone_001_pos': ['FAIL', known_reason],
 263     'snapshot/snapshot_009_pos': ['FAIL', '7961'],
 264     'snapshot/snapshot_010_pos': ['FAIL', '7961'],
 265     'snapused/snapused_004_pos': ['FAIL', '5513'],
 266     'tmpfile/setup': ['SKIP', tmpfile_reason],
 267     'threadsappend/threadsappend_001_pos': ['FAIL', '6136'],
 268     'upgrade/upgrade_projectquota_001_pos': ['SKIP', project_id_reason],
 269     'user_namespace/setup': ['SKIP', user_ns_reason],
 270     'userquota/setup': ['SKIP', exec_reason],
 271     'vdev_zaps/vdev_zaps_004_pos': ['FAIL', '6935'],
 272     'write_dirs/setup': ['SKIP', disk_reason],
 273     'zvol/zvol_ENOSPC/zvol_ENOSPC_001_pos': ['FAIL', '5848'],
 274 }
 275
 276
 277 def usage(s):
 278     print(s)
 279     sys.exit(1)
 280
 281
 282 def process_results(pathname):
 283     try:
 284         f = open(pathname)
 285     except IOError as e:
 286         print('Error opening file: %s' % e)
 287         sys.exit(1)
 288
 289     prefix = '/zfs-tests/tests/functional/'
 290     pattern = \
 291         r'^Test:\s*\S*%s(\S+)\s*\(run as (\S+)\)\s*\[(\S+)\]\s*\[(\S+)\]' \
 292         % prefix
 293     pattern_log = r'^\s*Log directory:\s*(\S*)'
 294
 295     d = {}
 296     for l in f.readlines():
 297         m = re.match(pattern, l)
 298         if m and len(m.groups()) == 4:
 299             summary['total'] += 1
 300             if m.group(4) == "PASS":
 301                 summary['passed'] += 1
 302             d[m.group(1)] = m.group(4)
 303             continue
 304
 305         m = re.match(pattern_log, l)
 306         if m:
 307             summary['logfile'] = m.group(1)
 308
 309     return d
 310
 311
 312 if __name__ == "__main__":
 313     if len(sys.argv) is not 2:
 314         usage('usage: %s <pathname>' % sys.argv[0])
 315     results = process_results(sys.argv[1])
 316
 317     if summary['total'] == 0:
 318         print("\n\nNo test results were found.")
 319         print("Log directory:  %s" % summary['logfile'])
 320         sys.exit(0)
 321
 322     expected = []
 323     unexpected = []
 324
 325     for test in list(results.keys()):
 326         if results[test] == "PASS":
 327             continue
 328
 329         setup = test.replace(os.path.basename(test), "setup")
 330         if results[test] == "SKIP" and test != setup:
 331             if setup in known and known[setup][0] == "SKIP":
 332                 continue
 333             if setup in maybe and maybe[setup][0] == "SKIP":
 334                 continue
 335
 336         if ((test not in known or results[test] not in known[test][0]) and
 337                 (test not in maybe or results[test] not in maybe[test][0])):
 338             unexpected.append(test)
 339         else:
 340             expected.append(test)
 341
 342     print("\nTests with results other than PASS that are expected:")
 343     for test in sorted(expected):
 344         issue_url = 'https://github.com/zfsonlinux/zfs/issues/'
 345
 346         # Include the reason why the result is expected, given the following:
 347         # 1. Suppress test results which set the "N/A on Linux" reason.
 348         # 2. Numerical reasons are assumed to be GitHub issue numbers.
 349         # 3. When an entire test group is skipped only report the setup reason.
 350         if test in known:
 351             if known[test][1] == na_reason:
 352                 continue
 353             elif known[test][1].isdigit():
 354                 expect = issue_url + known[test][1]
 355             else:
 356                 expect = known[test][1]
 357         elif test in maybe:
 358             if maybe[test][1].isdigit():
 359                 expect = issue_url + maybe[test][1]
 360             else:
 361                 expect = maybe[test][1]
 362         elif setup in known and known[setup][0] == "SKIP" and setup != test:
 363             continue
 364         elif setup in maybe and maybe[setup][0] == "SKIP" and setup != test:
 365             continue
 366         else:
 367             expect = "UNKNOWN REASON"
 368         print("    %s %s (%s)" % (results[test], test, expect))
 369
 370     print("\nTests with result of PASS that are unexpected:")
 371     for test in sorted(known.keys()):
 372         # We probably should not be silently ignoring the case
 373         # where "test" is not in "results".
 374         if test not in results or results[test] != "PASS":
 375             continue
 376         print("    %s %s (expected %s)" % (results[test], test,
 377                                            known[test][0]))
 378
 379     print("\nTests with results other than PASS that are unexpected:")
 380     for test in sorted(unexpected):
 381         expect = "PASS" if test not in known else known[test][0]
 382         print("    %s %s (expected %s)" % (results[test], test, expect))
 383
 384     if len(unexpected) == 0:
 385         sys.exit(0)
 386     else:
 387         sys.exit(1)