#include <sys/zil_impl.h>
#include <sys/vdev_impl.h>
#include <sys/vdev_file.h>
+#include <sys/vdev_initialize.h>
+#include <sys/vdev_trim.h>
#include <sys/spa_impl.h>
#include <sys/metaslab_impl.h>
#include <sys/dsl_prop.h>
#include <sys/fs/zfs.h>
#include <zfs_fletcher.h>
#include <libnvpair.h>
-#include <libzfs.h>
+#include <libzutil.h>
#include <sys/crypto/icp.h>
#ifdef __GLIBC__
#include <execinfo.h> /* for backtrace() */
uint64_t zo_metaslab_force_ganging;
int zo_mmp_test;
int zo_special_vdevs;
+ int zo_dump_dbgmsg;
} ztest_shared_opts_t;
static const ztest_shared_opts_t ztest_opts_defaults = {
- .zo_pool = { 'z', 't', 'e', 's', 't', '\0' },
- .zo_dir = { '/', 't', 'm', 'p', '\0' },
+ .zo_pool = "ztest",
+ .zo_dir = "/tmp",
.zo_alt_ztest = { '\0' },
.zo_alt_libpath = { '\0' },
.zo_vdevs = 5,
.zo_init = 1,
.zo_time = 300, /* 5 minutes */
.zo_maxloops = 50, /* max loops during spa_freeze() */
- .zo_metaslab_force_ganging = 32 << 10,
+ .zo_metaslab_force_ganging = 64 << 10,
.zo_special_vdevs = ZTEST_VDEV_CLASS_RND,
};
extern unsigned long zio_decompress_fail_fraction;
extern unsigned long zfs_reconstruct_indirect_damage_fraction;
+
static ztest_shared_opts_t *ztest_shared_opts;
static ztest_shared_opts_t ztest_opts;
static char *ztest_wkeydata = "abcdefghijklmnopqrstuvwxyz012345";
ztest_func_t ztest_reguid;
ztest_func_t ztest_spa_upgrade;
ztest_func_t ztest_device_removal;
-ztest_func_t ztest_remap_blocks;
ztest_func_t ztest_spa_checkpoint_create_discard;
+ztest_func_t ztest_initialize;
+ztest_func_t ztest_trim;
ztest_func_t ztest_fletcher;
ztest_func_t ztest_fletcher_incr;
ztest_func_t ztest_verify_dnode_bt;
ZTI_INIT(ztest_vdev_class_add, 1, &ztest_opts.zo_vdevtime),
ZTI_INIT(ztest_vdev_aux_add_remove, 1, &ztest_opts.zo_vdevtime),
ZTI_INIT(ztest_device_removal, 1, &zopt_sometimes),
- ZTI_INIT(ztest_remap_blocks, 1, &zopt_sometimes),
ZTI_INIT(ztest_spa_checkpoint_create_discard, 1, &zopt_rarely),
+ ZTI_INIT(ztest_initialize, 1, &zopt_sometimes),
+ ZTI_INIT(ztest_trim, 1, &zopt_sometimes),
ZTI_INIT(ztest_fletcher, 1, &zopt_rarely),
ZTI_INIT(ztest_fletcher_incr, 1, &zopt_rarely),
ZTI_INIT(ztest_verify_dnode_bt, 1, &zopt_sometimes),
static kmutex_t ztest_vdev_lock;
static boolean_t ztest_device_removal_active = B_FALSE;
+static boolean_t ztest_pool_scrubbed = B_FALSE;
static kmutex_t ztest_checkpoint_lock;
/*
static pthread_rwlock_t ztest_name_lock;
static boolean_t ztest_dump_core = B_TRUE;
-static boolean_t ztest_dump_debug_buffer = B_FALSE;
static boolean_t ztest_exiting;
/* Global commit callback list */
};
static void usage(boolean_t) __NORETURN;
+static int ztest_scrub_impl(spa_t *spa);
/*
* These libumem hooks provide a reasonable set of defaults for the allocator's
static void
dump_debug_buffer(void)
{
- if (!ztest_dump_debug_buffer)
+ ssize_t ret __attribute__((unused));
+
+ if (!ztest_opts.zo_dump_dbgmsg)
return;
- (void) printf("\n");
+ /*
+ * We use write() instead of printf() so that this function
+ * is safe to call from a signal handler.
+ */
+ ret = write(STDOUT_FILENO, "\n", 1);
zfs_dbgmsg_print("ztest");
}
(void) fprintf(stderr, "%s\n", buf);
fatal_msg = buf; /* to ease debugging */
- dump_debug_buffer();
-
if (ztest_dump_core)
abort();
+ else
+ dump_debug_buffer();
+
exit(3);
}
usage(B_FALSE);
break;
case 'G':
- ztest_dump_debug_buffer = B_TRUE;
+ zo->zo_dump_dbgmsg = 1;
break;
case 'h':
usage(B_TRUE);
boolean_t readonly, boolean_t decrypt, void *tag, objset_t **osp)
{
int err;
+ char *cp = NULL;
+ char ddname[ZFS_MAX_DATASET_NAME_LEN];
+
+ strcpy(ddname, name);
+ cp = strchr(ddname, '@');
+ if (cp != NULL)
+ *cp = '\0';
err = dmu_objset_own(name, type, readonly, decrypt, tag, osp);
- if (decrypt && err == EACCES) {
- char ddname[ZFS_MAX_DATASET_NAME_LEN];
+ while (decrypt && err == EACCES) {
dsl_crypto_params_t *dcp;
nvlist_t *crypto_args = fnvlist_alloc();
- char *cp = NULL;
-
- /* spa_keystore_load_wkey() expects a dsl dir name */
- strcpy(ddname, name);
- cp = strchr(ddname, '@');
- if (cp != NULL)
- *cp = '\0';
fnvlist_add_uint8_array(crypto_args, "wkeydata",
(uint8_t *)ztest_wkeydata, WRAPPING_KEY_LEN);
dsl_crypto_params_free(dcp, B_FALSE);
fnvlist_free(crypto_args);
- if (err != 0)
- return (err);
+ if (err == EINVAL) {
+ /*
+ * We couldn't load a key for this dataset so try
+ * the parent. This loop will eventually hit the
+ * encryption root since ztest only makes clones
+ * as children of their origin datasets.
+ */
+ cp = strrchr(ddname, '/');
+ if (cp == NULL)
+ return (err);
+
+ *cp = '\0';
+ err = EACCES;
+ continue;
+ } else if (err != 0) {
+ break;
+ }
err = dmu_objset_own(name, type, readonly, decrypt, tag, osp);
+ break;
}
return (err);
* ZIL get_data callbacks
*/
+/* ARGSUSED */
static void
ztest_get_done(zgd_t *zgd, int error)
{
ztest_range_unlock((rl_t *)zgd->zgd_lr);
ztest_object_unlock(zd, object);
- if (error == 0 && zgd->zgd_bp)
- zil_lwb_add_block(zgd->zgd_lwb, zgd->zgd_bp);
-
umem_free(zgd, sizeof (*zgd));
}
{
objset_t *os = zd->zd_os;
+ /*
+ * We hold the ztest_vdev_lock so we don't cause problems with
+ * other threads that wish to remove a log device, such as
+ * ztest_device_removal().
+ */
+ mutex_enter(&ztest_vdev_lock);
+
/*
* We grab the zd_dirobj_lock to ensure that no other thread is
* updating the zil (i.e. adding in-memory log records) and the
(void) pthread_rwlock_unlock(&zd->zd_zilog_lock);
mutex_exit(&zd->zd_dirobj_lock);
+ mutex_exit(&ztest_vdev_lock);
}
/*
if (zo->zo_mmp_test)
return;
+ /*
+ * Since enabling MMP involves setting a property, it could not be done
+ * while the pool is suspended.
+ */
+ if (spa_suspended(spa))
+ return;
+
spa_config_enter(spa, SCL_CONFIG, FTAG, RW_READER);
mutex_enter(&spa->spa_props_lock);
/*
* If we're configuring a RAIDZ device then make sure that the
- * the initial version is capable of supporting that feature.
+ * initial version is capable of supporting that feature.
*/
switch (ztest_opts.zo_raidz_parity) {
case 0:
mutex_enter(&ztest_vdev_lock);
- /* ensure we have a useable config; mirrors of raidz aren't supported */
+ /* ensure we have a usable config; mirrors of raidz aren't supported */
if (zs->zs_mirrors < 3 || ztest_opts.zo_raidz > 1) {
mutex_exit(&ztest_vdev_lock);
return;
pguid = pvd->vdev_guid;
/*
- * If oldvd has siblings, then half of the time, detach it.
+ * If oldvd has siblings, then half of the time, detach it. Prior
+ * to the detach the pool is scrubbed in order to prevent creating
+ * unrepairable blocks as a result of the data corruption injection.
*/
if (oldvd_has_siblings && ztest_random(2) == 0) {
spa_config_exit(spa, SCL_ALL, FTAG);
+
+ error = ztest_scrub_impl(spa);
+ if (error)
+ goto out;
+
error = spa_vdev_detach(spa, oldguid, pguid, B_FALSE);
if (error != 0 && error != ENODEV && error != EBUSY &&
error != ENOTSUP && error != ZFS_ERR_CHECKPOINT_EXISTS &&
ztest_device_removal_active = B_TRUE;
mutex_exit(&ztest_vdev_lock);
- while (spa->spa_vdev_removal != NULL)
+ /*
+ * spa->spa_vdev_removal is created in a sync task that
+ * is initiated via dsl_sync_task_nowait(). Since the
+ * task may not run before spa_vdev_remove() returns, we
+ * must wait at least 1 txg to ensure that the removal
+ * struct has been created.
+ */
+ txg_wait_synced(spa_get_dsl(spa), 0);
+
+ while (spa->spa_removing_phys.sr_state == DSS_SCANNING)
txg_wait_synced(spa_get_dsl(spa), 0);
} else {
mutex_exit(&ztest_vdev_lock);
VERIFY0(dsl_destroy_snapshot(name, B_TRUE));
} else {
error = dsl_destroy_head(name);
- /* There could be a hold on this dataset */
- if (error != EBUSY)
+ if (error == ENOSPC) {
+ /* There could be checkpoint or insufficient slop */
+ ztest_record_enospc(FTAG);
+ } else if (error != EBUSY) {
+ /* There could be a hold on this dataset */
ASSERT0(error);
+ }
}
return (0);
}
FTAG, &dbt, DMU_READ_NO_PREFETCH) == 0);
}
if (i != 5 || chunksize < (SPA_MINBLOCKSIZE * 2)) {
- dmu_assign_arcbuf_by_dbuf(bonus_db, off,
- bigbuf_arcbufs[j], tx);
+ VERIFY0(dmu_assign_arcbuf_by_dbuf(bonus_db,
+ off, bigbuf_arcbufs[j], tx));
} else {
- dmu_assign_arcbuf_by_dbuf(bonus_db, off,
- bigbuf_arcbufs[2 * j], tx);
- dmu_assign_arcbuf_by_dbuf(bonus_db,
+ VERIFY0(dmu_assign_arcbuf_by_dbuf(bonus_db,
+ off, bigbuf_arcbufs[2 * j], tx));
+ VERIFY0(dmu_assign_arcbuf_by_dbuf(bonus_db,
off + chunksize / 2,
- bigbuf_arcbufs[2 * j + 1], tx);
+ bigbuf_arcbufs[2 * j + 1], tx));
}
if (i == 1) {
dmu_buf_rele(dbt, FTAG);
umem_free(bigcheck, bigsize);
}
if (i == 2) {
- txg_wait_open(dmu_objset_pool(os), 0);
+ txg_wait_open(dmu_objset_pool(os), 0, B_TRUE);
} else if (i == 3) {
txg_wait_synced(dmu_objset_pool(os), 0);
}
dmu_tx_commit(tx);
/*
- * Generate a buch of random entries.
+ * Generate a bunch of random entries.
*/
ints = MAX(ZTEST_ZAP_MIN_INTS, object % ZTEST_ZAP_MAX_INTS);
}
/*
- * Testcase to test the upgrading of a microzap to fatzap.
+ * Test case to test the upgrading of a microzap to fatzap.
*/
void
ztest_fzap(ztest_ds_t *zd, uint64_t id)
(void) pthread_rwlock_unlock(&ztest_name_lock);
}
-/* ARGSUSED */
-void
-ztest_remap_blocks(ztest_ds_t *zd, uint64_t id)
-{
- (void) pthread_rwlock_rdlock(&ztest_name_lock);
-
- int error = dmu_objset_remap_indirects(zd->zd_name);
- if (error == ENOSPC)
- error = 0;
- ASSERT0(error);
-
- (void) pthread_rwlock_unlock(&ztest_name_lock);
-}
-
/* ARGSUSED */
void
ztest_spa_prop_get_set(ztest_ds_t *zd, uint64_t id)
(void) ztest_spa_prop_set_uint64(ZPOOL_PROP_DEDUPDITTO,
ZIO_DEDUPDITTO_MIN + ztest_random(ZIO_DEDUPDITTO_MIN));
+ (void) ztest_spa_prop_set_uint64(ZPOOL_PROP_AUTOTRIM, ztest_random(2));
+
VERIFY0(spa_prop_get(ztest_spa, &props));
if (ztest_opts.zo_verbose >= 6)
ASSERT(leaves >= 1);
+ /*
+ * While ztest is running the number of leaves will not change. This
+ * is critical for the fault injection logic as it determines where
+ * errors can be safely injected such that they are always repairable.
+ *
+ * When restarting ztest a different number of leaves may be requested
+ * which will shift the regions to be damaged. This is fine as long
+ * as the pool has been scrubbed prior to using the new mapping.
+ * Failure to do can result in non-repairable damage being injected.
+ */
+ if (ztest_pool_scrubbed == B_FALSE)
+ goto out;
+
/*
* Grab the name lock as reader. There are some operations
* which don't like to have their vdevs changed while
spa_t *spa = ztest_spa;
objset_t *os = zd->zd_os;
ztest_od_t *od;
- uint64_t object, blocksize, txg, pattern, psize;
+ uint64_t object, blocksize, txg, pattern;
enum zio_checksum checksum = spa_dedup_checksum(spa);
dmu_buf_t *db;
dmu_tx_t *tx;
- abd_t *abd;
- blkptr_t blk;
- int copies = 2 * ZIO_DEDUPDITTO_MIN;
- int i;
-
- blocksize = ztest_random_blocksize();
- blocksize = MIN(blocksize, 2048); /* because we write so many */
od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL);
- ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, blocksize, 0, 0);
+ ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, 0, 0, 0);
if (ztest_object_init(zd, od, sizeof (ztest_od_t), B_FALSE) != 0) {
umem_free(od, sizeof (ztest_od_t));
/*
* Take the name lock as writer to prevent anyone else from changing
- * the pool and dataset properies we need to maintain during this test.
+ * the pool and dataset properties we need to maintain during this test.
*/
(void) pthread_rwlock_wrlock(&ztest_name_lock);
blocksize = od[0].od_blocksize;
pattern = zs->zs_guid ^ dds.dds_guid;
+ /*
+ * The numbers of copies written must always be greater than or
+ * equal to the threshold set by the dedupditto property. This
+ * is initialized in ztest_run() and then randomly changed by
+ * ztest_spa_prop_get_set(), these function will never set it
+ * larger than 2 * ZIO_DEDUPDITTO_MIN.
+ */
+ int copies = 2 * ZIO_DEDUPDITTO_MIN;
+
+ /*
+ * The block size is limited by DMU_MAX_ACCESS (64MB) which
+ * caps the maximum transaction size. A block size of up to
+ * SPA_OLD_MAXBLOCKSIZE is allowed which results in a maximum
+ * transaction size of: 128K * 200 (copies) = ~25MB
+ *
+ * The actual block size is checked here, rather than requested
+ * above, because the way ztest_od_init() is implemented it does
+ * not guarantee the block size requested will be used.
+ */
+ if (blocksize > SPA_OLD_MAXBLOCKSIZE) {
+ (void) pthread_rwlock_unlock(&ztest_name_lock);
+ umem_free(od, sizeof (ztest_od_t));
+ return;
+ }
+
ASSERT(object != 0);
tx = dmu_tx_create(os);
/*
* Write all the copies of our block.
*/
- for (i = 0; i < copies; i++) {
+ for (int i = 0; i < copies; i++) {
uint64_t offset = i * blocksize;
int error = dmu_buf_hold(os, object, offset, FTAG, &db,
DMU_READ_NO_PREFETCH);
/*
* Find out what block we got.
*/
- VERIFY0(dmu_buf_hold(os, object, 0, FTAG, &db,
- DMU_READ_NO_PREFETCH));
- blk = *((dmu_buf_impl_t *)db)->db_blkptr;
+ VERIFY0(dmu_buf_hold(os, object, 0, FTAG, &db, DMU_READ_NO_PREFETCH));
+ blkptr_t blk = *((dmu_buf_impl_t *)db)->db_blkptr;
dmu_buf_rele(db, FTAG);
/*
* Damage the block. Dedup-ditto will save us when we read it later.
*/
- psize = BP_GET_PSIZE(&blk);
- abd = abd_alloc_linear(psize, B_TRUE);
+ uint64_t psize = BP_GET_PSIZE(&blk);
+ abd_t *abd = abd_alloc_linear(psize, B_TRUE);
ztest_pattern_set(abd_to_buf(abd), psize, ~pattern);
(void) zio_wait(zio_rewrite(NULL, spa, 0, &blk,
umem_free(od, sizeof (ztest_od_t));
}
+/*
+ * By design ztest will never inject uncorrectable damage in to the pool.
+ * Issue a scrub, wait for it to complete, and verify there is never any
+ * any persistent damage.
+ *
+ * Only after a full scrub has been completed is it safe to start injecting
+ * data corruption. See the comment in zfs_fault_inject().
+ */
+static int
+ztest_scrub_impl(spa_t *spa)
+{
+ int error = spa_scan(spa, POOL_SCAN_SCRUB);
+ if (error)
+ return (error);
+
+ while (dsl_scan_scrubbing(spa_get_dsl(spa)))
+ txg_wait_synced(spa_get_dsl(spa), 0);
+
+ if (spa_get_errlog_size(spa) > 0)
+ return (ECKSUM);
+
+ ztest_pool_scrubbed = B_TRUE;
+
+ return (0);
+}
+
/*
* Scrub the pool.
*/
ztest_scrub(ztest_ds_t *zd, uint64_t id)
{
spa_t *spa = ztest_spa;
+ int error;
/*
* Scrub in progress by device removal.
if (ztest_device_removal_active)
return;
+ /*
+ * Start a scrub, wait a moment, then force a restart.
+ */
(void) spa_scan(spa, POOL_SCAN_SCRUB);
- (void) poll(NULL, 0, 100); /* wait a moment, then force a restart */
- (void) spa_scan(spa, POOL_SCAN_SCRUB);
+ (void) poll(NULL, 0, 100);
+
+ error = ztest_scrub_impl(spa);
+ if (error == EBUSY)
+ error = 0;
+ ASSERT0(error);
}
/*
strcpy(bin, "zdb");
}
+static vdev_t *
+ztest_random_concrete_vdev_leaf(vdev_t *vd)
+{
+ if (vd == NULL)
+ return (NULL);
+
+ if (vd->vdev_children == 0)
+ return (vd);
+
+ vdev_t *eligible[vd->vdev_children];
+ int eligible_idx = 0, i;
+ for (i = 0; i < vd->vdev_children; i++) {
+ vdev_t *cvd = vd->vdev_child[i];
+ if (cvd->vdev_top->vdev_removing)
+ continue;
+ if (cvd->vdev_children > 0 ||
+ (vdev_is_concrete(cvd) && !cvd->vdev_detached)) {
+ eligible[eligible_idx++] = cvd;
+ }
+ }
+ VERIFY(eligible_idx > 0);
+
+ uint64_t child_no = ztest_random(eligible_idx);
+ return (ztest_random_concrete_vdev_leaf(eligible[child_no]));
+}
+
+/* ARGSUSED */
+void
+ztest_initialize(ztest_ds_t *zd, uint64_t id)
+{
+ spa_t *spa = ztest_spa;
+ int error = 0;
+
+ mutex_enter(&ztest_vdev_lock);
+
+ spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
+
+ /* Random leaf vdev */
+ vdev_t *rand_vd = ztest_random_concrete_vdev_leaf(spa->spa_root_vdev);
+ if (rand_vd == NULL) {
+ spa_config_exit(spa, SCL_VDEV, FTAG);
+ mutex_exit(&ztest_vdev_lock);
+ return;
+ }
+
+ /*
+ * The random vdev we've selected may change as soon as we
+ * drop the spa_config_lock. We create local copies of things
+ * we're interested in.
+ */
+ uint64_t guid = rand_vd->vdev_guid;
+ char *path = strdup(rand_vd->vdev_path);
+ boolean_t active = rand_vd->vdev_initialize_thread != NULL;
+
+ zfs_dbgmsg("vd %px, guid %llu", rand_vd, guid);
+ spa_config_exit(spa, SCL_VDEV, FTAG);
+
+ uint64_t cmd = ztest_random(POOL_INITIALIZE_FUNCS);
+
+ nvlist_t *vdev_guids = fnvlist_alloc();
+ nvlist_t *vdev_errlist = fnvlist_alloc();
+ fnvlist_add_uint64(vdev_guids, path, guid);
+ error = spa_vdev_initialize(spa, vdev_guids, cmd, vdev_errlist);
+ fnvlist_free(vdev_guids);
+ fnvlist_free(vdev_errlist);
+
+ switch (cmd) {
+ case POOL_INITIALIZE_CANCEL:
+ if (ztest_opts.zo_verbose >= 4) {
+ (void) printf("Cancel initialize %s", path);
+ if (!active)
+ (void) printf(" failed (no initialize active)");
+ (void) printf("\n");
+ }
+ break;
+ case POOL_INITIALIZE_START:
+ if (ztest_opts.zo_verbose >= 4) {
+ (void) printf("Start initialize %s", path);
+ if (active && error == 0)
+ (void) printf(" failed (already active)");
+ else if (error != 0)
+ (void) printf(" failed (error %d)", error);
+ (void) printf("\n");
+ }
+ break;
+ case POOL_INITIALIZE_SUSPEND:
+ if (ztest_opts.zo_verbose >= 4) {
+ (void) printf("Suspend initialize %s", path);
+ if (!active)
+ (void) printf(" failed (no initialize active)");
+ (void) printf("\n");
+ }
+ break;
+ }
+ free(path);
+ mutex_exit(&ztest_vdev_lock);
+}
+
+/* ARGSUSED */
+void
+ztest_trim(ztest_ds_t *zd, uint64_t id)
+{
+ spa_t *spa = ztest_spa;
+ int error = 0;
+
+ mutex_enter(&ztest_vdev_lock);
+
+ spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
+
+ /* Random leaf vdev */
+ vdev_t *rand_vd = ztest_random_concrete_vdev_leaf(spa->spa_root_vdev);
+ if (rand_vd == NULL) {
+ spa_config_exit(spa, SCL_VDEV, FTAG);
+ mutex_exit(&ztest_vdev_lock);
+ return;
+ }
+
+ /*
+ * The random vdev we've selected may change as soon as we
+ * drop the spa_config_lock. We create local copies of things
+ * we're interested in.
+ */
+ uint64_t guid = rand_vd->vdev_guid;
+ char *path = strdup(rand_vd->vdev_path);
+ boolean_t active = rand_vd->vdev_trim_thread != NULL;
+
+ zfs_dbgmsg("vd %p, guid %llu", rand_vd, guid);
+ spa_config_exit(spa, SCL_VDEV, FTAG);
+
+ uint64_t cmd = ztest_random(POOL_TRIM_FUNCS);
+ uint64_t rate = 1 << ztest_random(30);
+ boolean_t partial = (ztest_random(5) > 0);
+ boolean_t secure = (ztest_random(5) > 0);
+
+ nvlist_t *vdev_guids = fnvlist_alloc();
+ nvlist_t *vdev_errlist = fnvlist_alloc();
+ fnvlist_add_uint64(vdev_guids, path, guid);
+ error = spa_vdev_trim(spa, vdev_guids, cmd, rate, partial,
+ secure, vdev_errlist);
+ fnvlist_free(vdev_guids);
+ fnvlist_free(vdev_errlist);
+
+ switch (cmd) {
+ case POOL_TRIM_CANCEL:
+ if (ztest_opts.zo_verbose >= 4) {
+ (void) printf("Cancel TRIM %s", path);
+ if (!active)
+ (void) printf(" failed (no TRIM active)");
+ (void) printf("\n");
+ }
+ break;
+ case POOL_TRIM_START:
+ if (ztest_opts.zo_verbose >= 4) {
+ (void) printf("Start TRIM %s", path);
+ if (active && error == 0)
+ (void) printf(" failed (already active)");
+ else if (error != 0)
+ (void) printf(" failed (error %d)", error);
+ (void) printf("\n");
+ }
+ break;
+ case POOL_TRIM_SUSPEND:
+ if (ztest_opts.zo_verbose >= 4) {
+ (void) printf("Suspend TRIM %s", path);
+ if (!active)
+ (void) printf(" failed (no TRIM active)");
+ (void) printf("\n");
+ }
+ break;
+ }
+ free(path);
+ mutex_exit(&ztest_vdev_lock);
+}
+
/*
* Verify pool integrity by running zdb.
*/
ztest_get_zdb_bin(bin, len);
(void) sprintf(zdb,
- "%s -bcc%s%s -G -d -U %s "
- "-o zfs_reconstruct_indirect_combinations_max=65536 %s",
+ "%s -bcc%s%s -G -d -Y -U %s %s",
bin,
ztest_opts.zo_verbose >= 3 ? "s" : "",
ztest_opts.zo_verbose >= 4 ? "v" : "",
{
ztest_shared_t *zs = arg;
spa_t *spa = ztest_spa;
- hrtime_t delta, overdue, total = 0;
+ hrtime_t delay, overdue, last_run = gethrtime();
- for (;;) {
- delta = zs->zs_thread_stop - zs->zs_thread_start +
- MSEC2NSEC(zfs_deadman_synctime_ms);
+ delay = (zs->zs_thread_stop - zs->zs_thread_start) +
+ MSEC2NSEC(zfs_deadman_synctime_ms);
- (void) poll(NULL, 0, (int)NSEC2MSEC(delta));
+ while (!ztest_exiting) {
+ /*
+ * Wait for the delay timer while checking occasionally
+ * if we should stop.
+ */
+ if (gethrtime() < last_run + delay) {
+ (void) poll(NULL, 0, 1000);
+ continue;
+ }
/*
* If the pool is suspended then fail immediately. Otherwise,
* then it may be hung and is terminated.
*/
overdue = zs->zs_proc_stop + MSEC2NSEC(zfs_deadman_synctime_ms);
- total += zfs_deadman_synctime_ms / 1000;
if (gethrtime() > overdue) {
fatal(0, "aborting test after %llu seconds because "
- "the process is overdue for termination.", total);
+ "the process is overdue for termination.",
+ (gethrtime() - zs->zs_proc_start) / NANOSEC);
}
(void) printf("ztest has been running for %lld seconds\n",
- total);
+ (gethrtime() - zs->zs_proc_start) / NANOSEC);
+
+ last_run = gethrtime();
+ delay = MSEC2NSEC(zfs_deadman_checktime_ms);
}
+
+ thread_exit();
}
static void
ztest_zd_fini(zd);
}
+/* ARGSUSED */
+static int
+ztest_replay_zil_cb(const char *name, void *arg)
+{
+ objset_t *os;
+ ztest_ds_t *zdtmp;
+
+ VERIFY0(ztest_dmu_objset_own(name, DMU_OST_ANY, B_TRUE,
+ B_TRUE, FTAG, &os));
+
+ zdtmp = umem_alloc(sizeof (ztest_ds_t), UMEM_NOFAIL);
+
+ ztest_zd_init(zdtmp, NULL, os);
+ zil_replay(os, zdtmp, ztest_replay_vector);
+ ztest_zd_fini(zdtmp);
+
+ if (dmu_objset_zil(os)->zl_parse_lr_count != 0 &&
+ ztest_opts.zo_verbose >= 6) {
+ zilog_t *zilog = dmu_objset_zil(os);
+
+ (void) printf("%s replay %llu blocks, %llu records, seq %llu\n",
+ name,
+ (u_longlong_t)zilog->zl_parse_blk_count,
+ (u_longlong_t)zilog->zl_parse_lr_count,
+ (u_longlong_t)zilog->zl_replaying_seq);
+ }
+
+ umem_free(zdtmp, sizeof (ztest_ds_t));
+
+ dmu_objset_disown(os, B_TRUE, FTAG);
+ return (0);
+}
+
/*
* Kick off threads to run tests on all datasets in parallel.
*/
{
spa_t *spa;
objset_t *os;
- kthread_t *resume_thread;
+ kthread_t *resume_thread, *deadman_thread;
kthread_t **run_threads;
uint64_t object;
int error;
/*
* Create a deadman thread and set to panic if we hang.
*/
- (void) thread_create(NULL, 0, ztest_deadman_thread,
+ deadman_thread = thread_create(NULL, 0, ztest_deadman_thread,
zs, 0, NULL, TS_RUN | TS_JOINABLE, defclsyspri);
spa->spa_deadman_failmode = ZIO_FAILURE_MODE_PANIC;
}
zs->zs_enospc_count = 0;
+ /*
+ * If we were in the middle of ztest_device_removal() and were killed
+ * we need to ensure the removal and scrub complete before running
+ * any tests that check ztest_device_removal_active. The removal will
+ * be restarted automatically when the spa is opened, but we need to
+ * initiate the scrub manually if it is not already in progress. Note
+ * that we always run the scrub whenever an indirect vdev exists
+ * because we have no way of knowing for sure if ztest_device_removal()
+ * fully completed its scrub before the pool was reimported.
+ */
+ if (spa->spa_removing_phys.sr_state == DSS_SCANNING ||
+ spa->spa_removing_phys.sr_prev_indirect_vdev != -1) {
+ while (spa->spa_removing_phys.sr_state == DSS_SCANNING)
+ txg_wait_synced(spa_get_dsl(spa), 0);
+
+ error = ztest_scrub_impl(spa);
+ if (error == EBUSY)
+ error = 0;
+ ASSERT0(error);
+ }
+
run_threads = umem_zalloc(ztest_opts.zo_threads * sizeof (kthread_t *),
UMEM_NOFAIL);
if (ztest_opts.zo_verbose >= 4)
(void) printf("starting main threads...\n");
+ /*
+ * Replay all logs of all datasets in the pool. This is primarily for
+ * temporary datasets which wouldn't otherwise get replayed, which
+ * can trigger failures when attempting to offline a SLOG in
+ * ztest_fault_inject().
+ */
+ (void) dmu_objset_find(ztest_opts.zo_pool, ztest_replay_zil_cb,
+ NULL, DS_FIND_CHILDREN);
+
/*
* Kick off all the tests that run in parallel.
*/
}
/*
- * Wait for all of the tests to complete. We go in reverse order
- * so we don't close datasets while threads are still using them.
+ * Wait for all of the tests to complete.
*/
- for (t = ztest_opts.zo_threads - 1; t >= 0; t--) {
+ for (t = 0; t < ztest_opts.zo_threads; t++)
VERIFY0(thread_join(run_threads[t]));
+
+ /*
+ * Close all datasets. This must be done after all the threads
+ * are joined so we can be sure none of the datasets are in-use
+ * by any of the threads.
+ */
+ for (t = 0; t < ztest_opts.zo_threads; t++) {
if (t < ztest_opts.zo_datasets)
ztest_dataset_close(t);
}
umem_free(run_threads, ztest_opts.zo_threads * sizeof (kthread_t *));
- /* Kill the resume thread */
+ /* Kill the resume and deadman threads */
ztest_exiting = B_TRUE;
VERIFY0(thread_join(resume_thread));
+ VERIFY0(thread_join(deadman_thread));
ztest_resume(spa);
/*
static void
ztest_import(ztest_shared_t *zs)
{
- libzfs_handle_t *hdl;
importargs_t args = { 0 };
spa_t *spa;
nvlist_t *cfg = NULL;
VERIFY0(pthread_rwlock_init(&ztest_name_lock, NULL));
kernel_init(FREAD | FWRITE);
- hdl = libzfs_init();
searchdirs[0] = ztest_opts.zo_dir;
args.paths = nsearch;
args.path = searchdirs;
args.can_be_active = B_FALSE;
- error = zpool_tryimport(hdl, name, &cfg, &args);
+ error = zpool_find_config(NULL, name, &cfg, &args,
+ &libzpool_config_ops);
if (error)
(void) fatal(0, "No pools found\n");
1ULL << spa->spa_root_vdev->vdev_child[0]->vdev_ms_shift;
spa_close(spa, FTAG);
- libzfs_fini(hdl);
kernel_fini();
if (!ztest_opts.zo_mmp_test) {
dprintf_setup(&argc, argv);
zfs_deadman_synctime_ms = 300000;
+ zfs_deadman_checktime_ms = 30000;
/*
* As two-word space map entries may not come up often (especially
* if pool and vdev sizes are small) we want to force at least some
* Verify that even extensively damaged split blocks with many
* segments can be reconstructed in a reasonable amount of time
* when reconstruction is known to be possible.
+ *
+ * Note: the lower this value is, the more damage we inflict, and
+ * the more time ztest spends in recovering that damage. We chose
+ * to induce damage 1/100th of the time so recovery is tested but
+ * not so frequently that ztest doesn't get to test other code paths.
*/
- zfs_reconstruct_indirect_damage_fraction = 4;
+ zfs_reconstruct_indirect_damage_fraction = 100;
action.sa_handler = sig_handler;
sigemptyset(&action.sa_mask);
projects / mirror_zfs.git / blobdiff