ZTS: Retry in import_rewind_config_changed.ksh

As explained by the disclaimer in the test case,

    "This test can fail since nothing guarantees that old
    MOS blocks aren't overwritten."

This behavior is expected and correct, but results in a
flaky test case which is problematic for the CI.  The best
we can do to resolve this is to retry the sub-test which
failed when the MOS blocks have clearly been overwritten.

When testing failures were rare enough that a single retry
should normally be sufficient.  However, we allow up to
five for good measure.

Reviewed by: George Melikov <mail@gmelikov.ru>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes #13119

diff --git a/tests/test-runner/bin/zts-report.py.in b/tests/test-runner/bin/zts-report.py.in
index 74a6a1b4d348200d031044d7418b8b0e6a595038..cdb4fc343e46629e6772ca9ceec37a74117b0d1a 100755 (executable)
--- a/tests/test-runner/bin/zts-report.py.in
+++ b/tests/test-runner/bin/zts-report.py.in
@@ -222,8 +222,6 @@ maybe = {
     'cli_root/zfs_unshare/setup': ['SKIP', share_reason],
     'cli_root/zpool_add/zpool_add_004_pos': ['FAIL', known_reason],
     'cli_root/zpool_destroy/zpool_destroy_001_pos': ['SKIP', '6145'],
-    'cli_root/zpool_import/import_rewind_config_changed':
-        ['FAIL', rewind_reason],
     'cli_root/zpool_import/zpool_import_missing_003_pos': ['SKIP', '6839'],
     'cli_root/zpool_initialize/zpool_initialize_import_export':
         ['FAIL', '11948'],

diff --git a/tests/zfs-tests/tests/functional/cli_root/zpool_import/import_rewind_config_changed.ksh b/tests/zfs-tests/tests/functional/cli_root/zpool_import/import_rewind_config_changed.ksh
index 3ac8c104f1cacb7524ba81749dcbbd4e7016e888..d79c757d24060a88ec1db6454f2319ec34da1af9 100755 (executable)
--- a/tests/zfs-tests/tests/functional/cli_root/zpool_import/import_rewind_config_changed.ksh
+++ b/tests/zfs-tests/tests/functional/cli_root/zpool_import/import_rewind_config_changed.ksh
@@ -61,6 +61,7 @@ function test_common
        typeset detachvdev="${4:-}"
        typeset removevdev="${5:-}"
        typeset finalpool="${6:-}"
+       typeset retval=1
 
        typeset poolcheck="$poolcreate"
 
@@ -120,19 +121,30 @@ function test_common
        # while having a checkpoint, we take it after the
        # operation that changes the config.
        #
+       # However, it is possible the MOS data was overwritten
+       # in which case the pool will either be unimportable, or
+       # may have been rewound prior to the data being written.
+       # In which case an error is returned and test_common()
+       # is retried by the caller to minimize false positives.
+       #
        log_must zpool checkpoint $TESTPOOL1
 
        log_must overwrite_data $TESTPOOL1 ""
 
        log_must zpool export $TESTPOOL1
 
-       log_must zpool import -d $DEVICE_DIR -T $txg $TESTPOOL1
-       log_must check_pool_config $TESTPOOL1 "$poolcheck"
+       zpool import -d $DEVICE_DIR -T $txg $TESTPOOL1
+       if (( $? == 0 )); then
+               verify_data_md5sums $MD5FILE
+               if (( $? == 0 )); then
+                       retval=0
+               fi
 
-       log_must verify_data_md5sums $MD5FILE
+               log_must check_pool_config $TESTPOOL1 "$poolcheck"
+               log_must zpool destroy $TESTPOOL1
+       fi
 
        # Cleanup
-       log_must zpool destroy $TESTPOOL1
        if [[ -n $pathstochange ]]; then
                for dev in $pathstochange; do
                        log_must mv "${dev}_new" $dev
@@ -143,6 +155,7 @@ function test_common
        log_must zpool destroy $TESTPOOL2
 
        log_note ""
+       return $retval
 }
 
 function test_add_vdevs
@@ -152,7 +165,12 @@ function test_add_vdevs
 
        log_note "$0: pool '$poolcreate', add $addvdevs."
 
-       test_common "$poolcreate" "$addvdevs"
+       for retry in $(seq 1 5); do
+               test_common "$poolcreate" "$addvdevs" && return
+               log_note "Retry $retry / 5 for test_add_vdevs()"
+       done
+
+       log_fail "Exhausted all 5 retries for test_add_vdevs()"
 }
 
 function test_attach_vdev
@@ -163,7 +181,12 @@ function test_attach_vdev
 
        log_note "$0: pool '$poolcreate', attach $attachvdev to $attachto."
 
-       test_common "$poolcreate" "" "$attachto $attachvdev"
+       for retry in $(seq 1 5); do
+               test_common "$poolcreate" "" "$attachto $attachvdev" && return
+               log_note "Retry $retry / 5 for test_attach_vdev()"
+       done
+
+       log_fail "Exhausted all 5 retries for test_attach_vdev()"
 }
 
 function test_detach_vdev
@@ -173,7 +196,12 @@ function test_detach_vdev
 
        log_note "$0: pool '$poolcreate', detach $detachvdev."
 
-       test_common "$poolcreate" "" "" "$detachvdev"
+       for retry in $(seq 1 5); do
+               test_common "$poolcreate" "" "" "$detachvdev" && return
+               log_note "Retry $retry / 5 for test_detach_vdev()"
+       done
+
+       log_fail "Exhausted all 5 retries for test_detach_vdev()"
 }
 
 function test_attach_detach_vdev
@@ -186,7 +214,13 @@ function test_attach_detach_vdev
        log_note "$0: pool '$poolcreate', attach $attachvdev to $attachto," \
            "then detach $detachvdev."
 
-       test_common "$poolcreate" "" "$attachto $attachvdev" "$detachvdev"
+       for retry in $(seq 1 5); do
+               test_common "$poolcreate" "" "$attachto $attachvdev" \
+                   "$detachvdev" && return
+               log_note "Retry $retry / 5 for test_attach_detach_vdev()"
+       done
+
+       log_fail "Exhausted all 5 retries for test_attach_detach_vdev()"
 }
 
 function test_remove_vdev
@@ -197,7 +231,13 @@ function test_remove_vdev
 
        log_note "$0: pool '$poolcreate', remove $removevdev."
 
-       test_common "$poolcreate" "" "" "" "$removevdev" "$finalpool"
+       for retry in $(seq 1 5); do
+               test_common "$poolcreate" "" "" "" "$removevdev" \
+                   "$finalpool" && return
+               log_note "Retry $retry / 5 for test_remove_vdev()"
+       done
+
+       log_fail "Exhausted all 5 retries for test_remove_vdev()"
 }
 
 # Record txg history