tests/test-runner/bin/zts-report.py

   1 #!/usr/bin/python
   2
   3 #
   4 # This file and its contents are supplied under the terms of the
   5 # Common Development and Distribution License ("CDDL"), version 1.0.
   6 # You may only use this file in accordance with the terms of version
   7 # 1.0 of the CDDL.
   8 #
   9 # A full copy of the text of the CDDL should have accompanied this
  10 # source.  A copy of the CDDL is also available via the Internet at
  11 # http://www.illumos.org/license/CDDL.
  12 #
  13
  14 #
  15 # Copyright (c) 2017 by Delphix. All rights reserved.
  16 # Copyright (c) 2018 by Lawrence Livermore National Security, LLC.
  17 #
  18
  19 import os
  20 import re
  21 import sys
  22
  23 #
  24 # This script parses the stdout of zfstest, which has this format:
  25 #
  26 # Test: /path/to/testa (run as root) [00:00] [PASS]
  27 # Test: /path/to/testb (run as jkennedy) [00:00] [PASS]
  28 # Test: /path/to/testc (run as root) [00:00] [FAIL]
  29 # [...many more results...]
  30 #
  31 # Results Summary
  32 # FAIL      22
  33 # SKIP      32
  34 # PASS    1156
  35 #
  36 # Running Time:   02:50:31
  37 # Percent passed: 95.5%
  38 # Log directory:  /var/tmp/test_results/20180615T205926
  39 #
  40
  41 #
  42 # Common generic reasons for a test or test group to be skipped.
  43 #
  44 # Some test cases are known to fail in ways which are not harmful or dangerous.
  45 # In these cases simply mark the test as a known failure until it can be
  46 # updated and the issue resolved.  Note that it's preferable to open a unique
  47 # issue on the GitHub issue tracker for each test case failure.
  48 #
  49 known_reason = 'Known issue'
  50
  51 #
  52 # Some tests require that a test user be able to execute the zfs utilities.
  53 # This may not be possible when testing in-tree due to the default permissions
  54 # on the user's home directory.  When testing this can be resolved by granting
  55 # group read access.
  56 #
  57 # chmod 0750 $HOME
  58 #
  59 exec_reason = 'Test user execute permissions required for utilities'
  60
  61 #
  62 # Some tests require that the DISKS provided can be partitioned.  This is
  63 # normally not an issue because loop back devices are used for DISKS and they
  64 # can be partition.  There is one notable exception, the CentOS 6.x kernel is
  65 # old enough that it does not support partitioning loop back devices.
  66 #
  67 disk_reason = 'Partitionable DISKS required'
  68
  69 #
  70 # Some tests require a minimum python version of 3.5 and will be skipped when
  71 # the default system version is too old.  There may also be tests which require
  72 # additional python modules be installed, for example python-cffi is required
  73 # by the pyzfs tests.
  74 #
  75 python_reason = 'Python v3.5 or newer required'
  76 python_deps_reason = 'Python modules missing: python-cffi'
  77
  78 #
  79 # Some tests require the O_TMPFILE flag which was first introduced in the
  80 # 3.11 kernel.
  81 #
  82 tmpfile_reason = 'Kernel O_TMPFILE support required'
  83
  84 #
  85 # Some tests may depend on udev change events being generated when block
  86 # devices change capacity.  This functionality wasn't available until the
  87 # 2.6.38 kernel.
  88 #
  89 udev_reason = 'Kernel block device udev change events required'
  90
  91 #
  92 # Some tests require that the NFS client and server utilities be installed.
  93 #
  94 share_reason = 'NFS client and server utilities required'
  95
  96 #
  97 # Some tests require that the lsattr utility support the project id feature.
  98 #
  99 project_id_reason = 'lsattr with set/show project ID required'
 100
 101 #
 102 # Some tests require that the kernel support user namespaces.
 103 #
 104 user_ns_reason = 'Kernel user namespace support required'
 105
 106 #
 107 # Some rewind tests can fail since nothing guarantees that old MOS blocks
 108 # are not overwritten.  Snapshots protect datasets and data files but not
 109 # the MOS.  Reasonable efforts are made in the test case to increase the
 110 # odds that some txgs will have their MOS data left untouched, but it is
 111 # never a sure thing.
 112 #
 113 rewind_reason = 'Arbitrary pool rewind is not guaranteed'
 114
 115 #
 116 # Some tests may by structured in a way that relies on exact knowledge
 117 # of how much free space in available in a pool.  These tests cannot be
 118 # made completely reliable because the internal details of how free space
 119 # is managed are not exposed to user space.
 120 #
 121 enospc_reason = 'Exact free space reporting is not guaranteed'
 122
 123 #
 124 # Some tests require a minimum version of the fio benchmark utility.
 125 # Older distributions such as CentOS 6.x only provide fio-2.0.13.
 126 #
 127 fio_reason = 'Fio v2.3 or newer required'
 128
 129 #
 130 # Some tests are not applicable to Linux or need to be updated to operate
 131 # in the manor required by Linux.  Any tests which are skipped for this
 132 # reason will be suppressed in the final analysis output.
 133 #
 134 na_reason = "N/A on Linux"
 135
 136 summary = {
 137     'total': float(0),
 138     'passed': float(0),
 139     'logfile': "Could not determine logfile location."
 140 }
 141
 142 #
 143 # These tests are known to fail, thus we use this list to prevent these
 144 # failures from failing the job as a whole; only unexpected failures
 145 # bubble up to cause this script to exit with a non-zero exit status.
 146 #
 147 # Format: { 'test-name': ['expected result', 'issue-number | reason'] }
 148 #
 149 # For each known failure it is recommended to link to a GitHub issue by
 150 # setting the reason to the issue number.  Alternately, one of the generic
 151 # reasons listed above can be used.
 152 #
 153 known = {
 154     'acl/posix/posix_001_pos': ['FAIL', known_reason],
 155     'acl/posix/posix_002_pos': ['FAIL', known_reason],
 156     'casenorm/sensitive_none_lookup': ['FAIL', '7633'],
 157     'casenorm/sensitive_none_delete': ['FAIL', '7633'],
 158     'casenorm/sensitive_formd_lookup': ['FAIL', '7633'],
 159     'casenorm/sensitive_formd_delete': ['FAIL', '7633'],
 160     'casenorm/insensitive_none_lookup': ['FAIL', '7633'],
 161     'casenorm/insensitive_none_delete': ['FAIL', '7633'],
 162     'casenorm/insensitive_formd_lookup': ['FAIL', '7633'],
 163     'casenorm/insensitive_formd_delete': ['FAIL', '7633'],
 164     'casenorm/mixed_none_lookup': ['FAIL', '7633'],
 165     'casenorm/mixed_none_lookup_ci': ['FAIL', '7633'],
 166     'casenorm/mixed_none_delete': ['FAIL', '7633'],
 167     'casenorm/mixed_formd_lookup': ['FAIL', '7633'],
 168     'casenorm/mixed_formd_lookup_ci': ['FAIL', '7633'],
 169     'casenorm/mixed_formd_delete': ['FAIL', '7633'],
 170     'cli_root/zfs_mount/zfs_mount_006_pos': ['SKIP', '4990'],
 171     'cli_root/zfs_receive/zfs_receive_004_neg': ['FAIL', known_reason],
 172     'cli_root/zfs_unshare/zfs_unshare_002_pos': ['SKIP', na_reason],
 173     'cli_root/zfs_unshare/zfs_unshare_006_pos': ['SKIP', na_reason],
 174     'cli_root/zpool_create/zpool_create_016_pos': ['SKIP', na_reason],
 175     'cli_user/misc/zfs_share_001_neg': ['SKIP', na_reason],
 176     'cli_user/misc/zfs_unshare_001_neg': ['SKIP', na_reason],
 177     'inuse/inuse_001_pos': ['SKIP', na_reason],
 178     'inuse/inuse_003_pos': ['SKIP', na_reason],
 179     'inuse/inuse_006_pos': ['SKIP', na_reason],
 180     'inuse/inuse_007_pos': ['SKIP', na_reason],
 181     'privilege/setup': ['SKIP', na_reason],
 182     'refreserv/refreserv_004_pos': ['FAIL', known_reason],
 183     'removal/removal_condense_export': ['SKIP', known_reason],
 184     'removal/removal_with_zdb': ['SKIP', known_reason],
 185     'rootpool/setup': ['SKIP', na_reason],
 186     'rsend/rsend_008_pos': ['SKIP', '6066'],
 187     'vdev_zaps/vdev_zaps_007_pos': ['FAIL', known_reason],
 188     'xattr/xattr_008_pos': ['SKIP', na_reason],
 189     'xattr/xattr_009_neg': ['SKIP', na_reason],
 190     'xattr/xattr_010_neg': ['SKIP', na_reason],
 191     'zvol/zvol_misc/zvol_misc_001_neg': ['SKIP', na_reason],
 192     'zvol/zvol_misc/zvol_misc_003_neg': ['SKIP', na_reason],
 193     'zvol/zvol_misc/zvol_misc_004_pos': ['SKIP', na_reason],
 194     'zvol/zvol_misc/zvol_misc_005_neg': ['SKIP', na_reason],
 195     'zvol/zvol_misc/zvol_misc_006_pos': ['SKIP', na_reason],
 196     'zvol/zvol_swap/zvol_swap_003_pos': ['SKIP', na_reason],
 197     'zvol/zvol_swap/zvol_swap_005_pos': ['SKIP', na_reason],
 198     'zvol/zvol_swap/zvol_swap_006_pos': ['SKIP', na_reason],
 199 }
 200
 201 #
 202 # These tests may occasionally fail or be skipped.  We want there failures
 203 # to be reported but only unexpected failures should bubble up to cause
 204 # this script to exit with a non-zero exit status.
 205 #
 206 # Format: { 'test-name': ['expected result', 'issue-number | reason'] }
 207 #
 208 # For each known failure it is recommended to link to a GitHub issue by
 209 # setting the reason to the issue number.  Alternately, one of the generic
 210 # reasons listed above can be used.
 211 #
 212 maybe = {
 213     'cache/setup': ['SKIP', disk_reason],
 214     'cache/cache_010_neg': ['FAIL', known_reason],
 215     'chattr/setup': ['SKIP', exec_reason],
 216     'cli_root/zdb/zdb_006_pos': ['FAIL', known_reason],
 217     'cli_root/zfs_get/zfs_get_004_pos': ['FAIL', known_reason],
 218     'cli_root/zfs_get/zfs_get_009_pos': ['SKIP', '5479'],
 219     'cli_root/zfs_rename/zfs_rename_006_pos': ['FAIL', '5647'],
 220     'cli_root/zfs_rename/zfs_rename_009_neg': ['FAIL', '5648'],
 221     'cli_root/zfs_rollback/zfs_rollback_001_pos': ['FAIL', '6415'],
 222     'cli_root/zfs_rollback/zfs_rollback_002_pos': ['FAIL', '6416'],
 223     'cli_root/zfs_share/setup': ['SKIP', share_reason],
 224     'cli_root/zfs_snapshot/zfs_snapshot_002_neg': ['FAIL', known_reason],
 225     'cli_root/zfs_unshare/setup': ['SKIP', share_reason],
 226     'cli_root/zpool_add/setup': ['SKIP', disk_reason],
 227     'cli_root/zpool_add/zpool_add_004_pos': ['FAIL', known_reason],
 228     'cli_root/zpool_create/setup': ['SKIP', disk_reason],
 229     'cli_root/zpool_create/zpool_create_008_pos': ['FAIL', known_reason],
 230     'cli_root/zpool_destroy/zpool_destroy_001_pos': ['SKIP', '6145'],
 231     'cli_root/zpool_expand/setup': ['SKIP', udev_reason],
 232     'cli_root/zpool_export/setup': ['SKIP', disk_reason],
 233     'cli_root/zpool_import/setup': ['SKIP', disk_reason],
 234     'cli_root/zpool_import/import_rewind_device_replaced':
 235         ['FAIL', rewind_reason],
 236     'cli_root/zpool_import/import_rewind_config_changed':
 237         ['FAIL', rewind_reason],
 238     'cli_root/zpool_import/zpool_import_missing_003_pos': ['SKIP', '6839'],
 239     'cli_root/zpool_remove/setup': ['SKIP', disk_reason],
 240     'cli_root/zpool_upgrade/zpool_upgrade_004_pos': ['FAIL', '6141'],
 241     'cli_user/misc/arc_summary3_001_pos': ['SKIP', python_reason],
 242     'delegate/setup': ['SKIP', exec_reason],
 243     'fault/auto_online_001_pos': ['SKIP', disk_reason],
 244     'fault/auto_replace_001_pos': ['SKIP', disk_reason],
 245     'history/history_004_pos': ['FAIL', '7026'],
 246     'history/history_005_neg': ['FAIL', '6680'],
 247     'history/history_006_neg': ['FAIL', '5657'],
 248     'history/history_008_pos': ['FAIL', known_reason],
 249     'history/history_010_pos': ['SKIP', exec_reason],
 250     'inuse/inuse_005_pos': ['SKIP', disk_reason],
 251     'inuse/inuse_008_pos': ['SKIP', disk_reason],
 252     'inuse/inuse_009_pos': ['SKIP', disk_reason],
 253     'io/mmap': ['SKIP', fio_reason],
 254     'largest_pool/largest_pool_001_pos': ['FAIL', known_reason],
 255     'pyzfs/pyzfs_unittest': ['SKIP', python_deps_reason],
 256     'no_space/enospc_002_pos': ['FAIL', enospc_reason],
 257     'projectquota/setup': ['SKIP', exec_reason],
 258     'redundancy/redundancy_004_neg': ['FAIL', '7290'],
 259     'reservation/reservation_008_pos': ['FAIL', '7741'],
 260     'reservation/reservation_018_pos': ['FAIL', '5642'],
 261     'rsend/rsend_019_pos': ['FAIL', '6086'],
 262     'rsend/rsend_020_pos': ['FAIL', '6446'],
 263     'rsend/rsend_021_pos': ['FAIL', '6446'],
 264     'rsend/rsend_024_pos': ['FAIL', '5665'],
 265     'rsend/send-c_volume': ['FAIL', '6087'],
 266     'snapshot/clone_001_pos': ['FAIL', known_reason],
 267     'snapused/snapused_004_pos': ['FAIL', '5513'],
 268     'tmpfile/setup': ['SKIP', tmpfile_reason],
 269     'threadsappend/threadsappend_001_pos': ['FAIL', '6136'],
 270     'upgrade/upgrade_projectquota_001_pos': ['SKIP', project_id_reason],
 271     'user_namespace/setup': ['SKIP', user_ns_reason],
 272     'userquota/setup': ['SKIP', exec_reason],
 273     'vdev_zaps/vdev_zaps_004_pos': ['FAIL', '6935'],
 274     'write_dirs/setup': ['SKIP', disk_reason],
 275     'zvol/zvol_ENOSPC/zvol_ENOSPC_001_pos': ['FAIL', '5848'],
 276 }
 277
 278
 279 def usage(s):
 280     print(s)
 281     sys.exit(1)
 282
 283
 284 def process_results(pathname):
 285     try:
 286         f = open(pathname)
 287     except IOError as e:
 288         print('Error opening file: %s' % e)
 289         sys.exit(1)
 290
 291     prefix = '/zfs-tests/tests/functional/'
 292     pattern = '^Test:\s*\S*%s(\S+)\s*\(run as (\S+)\)\s*\[(\S+)\]\s*\[(\S+)\]'\
 293         % prefix
 294     pattern_log = '^\s*Log directory:\s*(\S*)'
 295
 296     d = {}
 297     for l in f.readlines():
 298         m = re.match(pattern, l)
 299         if m and len(m.groups()) == 4:
 300             summary['total'] += 1
 301             if m.group(4) == "PASS":
 302                 summary['passed'] += 1
 303             d[m.group(1)] = m.group(4)
 304             continue
 305
 306         m = re.match(pattern_log, l)
 307         if m:
 308             summary['logfile'] = m.group(1)
 309
 310     return d
 311
 312
 313 if __name__ == "__main__":
 314     if len(sys.argv) is not 2:
 315         usage('usage: %s <pathname>' % sys.argv[0])
 316     results = process_results(sys.argv[1])
 317
 318     if summary['total'] == 0:
 319         print("\n\nNo test results were found.")
 320         print("Log directory:  %s" % summary['logfile'])
 321         sys.exit(0)
 322
 323     expected = []
 324     unexpected = []
 325
 326     for test in list(results.keys()):
 327         if results[test] == "PASS":
 328             continue
 329
 330         setup = test.replace(os.path.basename(test), "setup")
 331         if results[test] == "SKIP" and test != setup:
 332             if setup in known and known[setup][0] == "SKIP":
 333                 continue
 334             if setup in maybe and maybe[setup][0] == "SKIP":
 335                 continue
 336
 337         if ((test not in known or results[test] not in known[test][0]) and
 338                 (test not in maybe or results[test] not in maybe[test][0])):
 339             unexpected.append(test)
 340         else:
 341             expected.append(test)
 342
 343     print("\nTests with results other than PASS that are expected:")
 344     for test in sorted(expected):
 345         issue_url = 'https://github.com/zfsonlinux/zfs/issues/'
 346
 347         # Include the reason why the result is expected, given the following:
 348         # 1. Suppress test results which set the "N/A on Linux" reason.
 349         # 2. Numerical reasons are assumed to be GitHub issue numbers.
 350         # 3. When an entire test group is skipped only report the setup reason.
 351         if test in known:
 352             if known[test][1] == na_reason:
 353                 continue
 354             elif known[test][1].isdigit():
 355                 expect = issue_url + known[test][1]
 356             else:
 357                 expect = known[test][1]
 358         elif test in maybe:
 359             if maybe[test][1].isdigit():
 360                 expect = issue_url + maybe[test][1]
 361             else:
 362                 expect = maybe[test][1]
 363         elif setup in known and known[setup][0] == "SKIP" and setup != test:
 364             continue
 365         elif setup in maybe and maybe[setup][0] == "SKIP" and setup != test:
 366             continue
 367         else:
 368             expect = "UNKNOWN REASON"
 369         print("    %s %s (%s)" % (results[test], test, expect))
 370
 371     print("\nTests with result of PASS that are unexpected:")
 372     for test in sorted(known.keys()):
 373         # We probably should not be silently ignoring the case
 374         # where "test" is not in "results".
 375         if test not in results or results[test] != "PASS":
 376             continue
 377         print("    %s %s (expected %s)" % (results[test], test,
 378                                            known[test][0]))
 379
 380     print("\nTests with results other than PASS that are unexpected:")
 381     for test in sorted(unexpected):
 382         expect = "PASS" if test not in known else known[test][0]
 383         print("    %s %s (expected %s)" % (results[test], test, expect))
 384
 385     if len(unexpected) == 0:
 386         sys.exit(0)
 387     else:
 388         sys.exit(1)