*/
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2011, 2017 by Delphix. All rights reserved.
+ * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
* Copyright (c) 2011 Nexenta Systems, Inc. All rights reserved.
+ * Copyright (c) 2017, Intel Corporation.
*/
#include <sys/sysmacros.h>
#include <sys/trace_zio.h>
#include <sys/abd.h>
#include <sys/dsl_crypt.h>
+#include <sys/cityhash.h>
/*
* ==========================================================================
int zio_delay_max = ZIO_DELAY_MAX;
-#define ZIO_PIPELINE_CONTINUE 0x100
-#define ZIO_PIPELINE_STOP 0x101
-
#define BP_SPANB(indblkshift, level) \
(((uint64_t)1) << ((level) * ((indblkshift) - SPA_BLKPTRSHIFT)))
#define COMPARE_META_LEVEL 0x80000000ul
zio->io_abd, tmp, zio->io_size, size);
abd_return_buf_copy(data, tmp, size);
+ if (zio_injection_enabled && ret == 0)
+ ret = zio_handle_fault_injection(zio, EINVAL);
+
if (ret != 0)
zio->io_error = SET_ERROR(EIO);
}
int ret;
void *tmp;
blkptr_t *bp = zio->io_bp;
+ spa_t *spa = zio->io_spa;
+ uint64_t dsobj = zio->io_bookmark.zb_objset;
uint64_t lsize = BP_GET_LSIZE(bp);
dmu_object_type_t ot = BP_GET_TYPE(bp);
uint8_t salt[ZIO_DATA_SALT_LEN];
}
abd_copy(data, zio->io_abd, size);
+ if (zio_injection_enabled && ot != DMU_OT_DNODE && ret == 0) {
+ ret = zio_handle_decrypt_injection(spa,
+ &zio->io_bookmark, ot, ECKSUM);
+ }
if (ret != 0)
goto error;
*/
if (BP_IS_AUTHENTICATED(bp)) {
if (ot == DMU_OT_OBJSET) {
- ret = spa_do_crypt_objset_mac_abd(B_FALSE, zio->io_spa,
- zio->io_bookmark.zb_objset, zio->io_abd, size,
- BP_SHOULD_BYTESWAP(bp));
+ ret = spa_do_crypt_objset_mac_abd(B_FALSE, spa,
+ dsobj, zio->io_abd, size, BP_SHOULD_BYTESWAP(bp));
} else {
zio_crypt_decode_mac_bp(bp, mac);
- ret = spa_do_crypt_mac_abd(B_FALSE, zio->io_spa,
- zio->io_bookmark.zb_objset, zio->io_abd, size, mac);
+ ret = spa_do_crypt_mac_abd(B_FALSE, spa, dsobj,
+ zio->io_abd, size, mac);
+ if (zio_injection_enabled && ret == 0) {
+ ret = zio_handle_decrypt_injection(spa,
+ &zio->io_bookmark, ot, ECKSUM);
+ }
}
abd_copy(data, zio->io_abd, size);
zio_crypt_decode_mac_bp(bp, mac);
}
- ret = spa_do_crypt_abd(B_FALSE, zio->io_spa, zio->io_bookmark.zb_objset,
- bp, bp->blk_birth, size, data, zio->io_abd, iv, mac, salt,
- &no_crypt);
+ ret = spa_do_crypt_abd(B_FALSE, spa, &zio->io_bookmark, BP_GET_TYPE(bp),
+ BP_GET_DEDUP(bp), BP_SHOULD_BYTESWAP(bp), salt, iv, mac, size, data,
+ zio->io_abd, &no_crypt);
if (no_crypt)
abd_copy(data, zio->io_abd, size);
error:
/* assert that the key was found unless this was speculative */
- ASSERT(ret != ENOENT || (zio->io_flags & ZIO_FLAG_SPECULATIVE));
+ ASSERT(ret != EACCES || (zio->io_flags & ZIO_FLAG_SPECULATIVE));
/*
* If there was a decryption / authentication error return EIO as
* the io_error. If this was not a speculative zio, create an ereport.
*/
if (ret == ECKSUM) {
- ret = SET_ERROR(EIO);
+ zio->io_error = SET_ERROR(EIO);
if ((zio->io_flags & ZIO_FLAG_SPECULATIVE) == 0) {
+ spa_log_error(spa, &zio->io_bookmark);
zfs_ereport_post(FM_EREPORT_ZFS_AUTHENTICATION,
- zio->io_spa, NULL, &zio->io_bookmark, zio, 0, 0);
+ spa, NULL, &zio->io_bookmark, zio, 0, 0);
}
} else {
zio->io_error = ret;
}
static boolean_t
-zio_wait_for_children(zio_t *zio, enum zio_child child, enum zio_wait_type wait)
+zio_wait_for_children(zio_t *zio, uint8_t childbits, enum zio_wait_type wait)
{
- uint64_t *countp = &zio->io_children[child][wait];
boolean_t waiting = B_FALSE;
mutex_enter(&zio->io_lock);
ASSERT(zio->io_stall == NULL);
- if (*countp != 0) {
- zio->io_stage >>= 1;
- ASSERT3U(zio->io_stage, !=, ZIO_STAGE_OPEN);
- zio->io_stall = countp;
- waiting = B_TRUE;
+ for (int c = 0; c < ZIO_CHILD_TYPES; c++) {
+ if (!(ZIO_CHILD_BIT_IS_SET(childbits, c)))
+ continue;
+
+ uint64_t *countp = &zio->io_children[c][wait];
+ if (*countp != 0) {
+ zio->io_stage >>= 1;
+ ASSERT3U(zio->io_stage, !=, ZIO_STAGE_OPEN);
+ zio->io_stall = countp;
+ waiting = B_TRUE;
+ break;
+ }
}
mutex_exit(&zio->io_lock);
-
return (waiting);
}
__attribute__((always_inline))
static inline void
-zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait)
+zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait,
+ zio_t **next_to_executep)
{
uint64_t *countp = &pio->io_children[zio->io_child_type][wait];
int *errorp = &pio->io_child_error[zio->io_child_type];
ZIO_TASKQ_INTERRUPT;
pio->io_stall = NULL;
mutex_exit(&pio->io_lock);
+
/*
- * Dispatch the parent zio in its own taskq so that
- * the child can continue to make progress. This also
- * prevents overflowing the stack when we have deeply nested
- * parent-child relationships.
+ * If we can tell the caller to execute this parent next, do
+ * so. Otherwise dispatch the parent zio as its own task.
+ *
+ * Having the caller execute the parent when possible reduces
+ * locking on the zio taskq's, reduces context switch
+ * overhead, and has no recursion penalty. Note that one
+ * read from disk typically causes at least 3 zio's: a
+ * zio_null(), the logical zio_read(), and then a physical
+ * zio. When the physical ZIO completes, we are able to call
+ * zio_done() on all 3 of these zio's from one invocation of
+ * zio_execute() by returning the parent back to
+ * zio_execute(). Since the parent isn't executed until this
+ * thread returns back to zio_execute(), the caller should do
+ * so promptly.
+ *
+ * In other cases, dispatching the parent prevents
+ * overflowing the stack when we have deeply nested
+ * parent-child relationships, as we do with the "mega zio"
+ * of writes for spa_sync(), and the chain of ZIL blocks.
*/
- zio_taskq_dispatch(pio, type, B_FALSE);
+ if (next_to_executep != NULL && *next_to_executep == NULL) {
+ *next_to_executep = pio;
+ } else {
+ zio_taskq_dispatch(pio, type, B_FALSE);
+ }
} else {
mutex_exit(&pio->io_lock);
}
zio->io_bookmark = *zb;
if (pio != NULL) {
+ if (zio->io_metaslab_class == NULL)
+ zio->io_metaslab_class = pio->io_metaslab_class;
if (zio->io_logical == NULL)
zio->io_logical = pio->io_logical;
if (zio->io_child_type == ZIO_CHILD_GANG)
}
}
+ /*
+ * Do not verify individual DVAs if the config is not trusted. This
+ * will be done once the zio is executed in vdev_mirror_map_alloc.
+ */
+ if (!spa->spa_trust_config)
+ return;
+
/*
* Pool-specific checks.
*
}
}
+boolean_t
+zfs_dva_valid(spa_t *spa, const dva_t *dva, const blkptr_t *bp)
+{
+ uint64_t vdevid = DVA_GET_VDEV(dva);
+
+ if (vdevid >= spa->spa_root_vdev->vdev_children)
+ return (B_FALSE);
+
+ vdev_t *vd = spa->spa_root_vdev->vdev_child[vdevid];
+ if (vd == NULL)
+ return (B_FALSE);
+
+ if (vd->vdev_ops == &vdev_hole_ops)
+ return (B_FALSE);
+
+ if (vd->vdev_ops == &vdev_missing_ops) {
+ return (B_FALSE);
+ }
+
+ uint64_t offset = DVA_GET_OFFSET(dva);
+ uint64_t asize = DVA_GET_ASIZE(dva);
+
+ if (BP_IS_GANG(bp))
+ asize = vdev_psize_to_asize(vd, SPA_GANGBLOCKSIZE);
+ if (offset + asize > vd->vdev_asize)
+ return (B_FALSE);
+
+ return (B_TRUE);
+}
+
zio_t *
zio_read(zio_t *pio, spa_t *spa, const blkptr_t *bp,
abd_t *data, uint64_t size, zio_done_func_t *done, void *private,
zio_free(spa_t *spa, uint64_t txg, const blkptr_t *bp)
{
+ zfs_blkptr_verify(spa, bp);
+
/*
* The check for EMBEDDED is a performance optimization. We
* process the free here (by ignoring it) rather than
{
zio_t *zio;
- dprintf_bp(bp, "claiming in txg %llu", txg);
+ zfs_blkptr_verify(spa, bp);
if (BP_IS_EMBEDDED(bp))
return (zio_null(pio, spa, NULL, NULL, NULL, 0));
* starts allocating blocks -- so that nothing is allocated twice.
* If txg == 0 we just verify that the block is claimable.
*/
- ASSERT3U(spa->spa_uberblock.ub_rootbp.blk_birth, <, spa_first_txg(spa));
- ASSERT(txg == spa_first_txg(spa) || txg == 0);
+ ASSERT3U(spa->spa_uberblock.ub_rootbp.blk_birth, <,
+ spa_min_claim_txg(spa));
+ ASSERT(txg == spa_min_claim_txg(spa) || txg == 0);
ASSERT(!BP_GET_DEDUP(bp) || !spa_writeable(spa)); /* zdb(1M) */
zio = zio_create(pio, spa, txg, bp, NULL, BP_GET_PSIZE(bp),
enum zio_stage pipeline = ZIO_VDEV_CHILD_PIPELINE;
zio_t *zio;
- ASSERT(vd->vdev_parent ==
- (pio->io_vd ? pio->io_vd : pio->io_spa->spa_root_vdev));
+ /*
+ * vdev child I/Os do not propagate their error to the parent.
+ * Therefore, for correct operation the caller *must* check for
+ * and handle the error in the child i/o's done callback.
+ * The only exceptions are i/os that we don't care about
+ * (OPTIONAL or REPAIR).
+ */
+ ASSERT((flags & ZIO_FLAG_OPTIONAL) || (flags & ZIO_FLAG_IO_REPAIR) ||
+ done != NULL);
if (type == ZIO_TYPE_READ && bp != NULL) {
/*
pio->io_pipeline &= ~ZIO_STAGE_CHECKSUM_VERIFY;
}
- if (vd->vdev_children == 0)
+ if (vd->vdev_ops->vdev_op_leaf) {
+ ASSERT0(vd->vdev_children);
offset += VDEV_LABEL_START_SIZE;
+ }
- flags |= ZIO_VDEV_CHILD_FLAGS(pio) | ZIO_FLAG_DONT_PROPAGATE;
+ flags |= ZIO_VDEV_CHILD_FLAGS(pio);
/*
* If we've decided to do a repair, the write is not speculative --
*/
if (flags & ZIO_FLAG_IO_ALLOCATING &&
(vd != vd->vdev_top || (flags & ZIO_FLAG_IO_RETRY))) {
- ASSERTV(metaslab_class_t *mc = spa_normal_class(pio->io_spa));
-
- ASSERT(mc->mc_alloc_throttle_enabled);
+ ASSERT(pio->io_metaslab_class != NULL);
+ ASSERT(pio->io_metaslab_class->mc_alloc_throttle_enabled);
ASSERT(type == ZIO_TYPE_WRITE);
ASSERT(priority == ZIO_PRIORITY_ASYNC_WRITE);
ASSERT(!(flags & ZIO_FLAG_IO_REPAIR));
zio_t *
zio_vdev_delegated_io(vdev_t *vd, uint64_t offset, abd_t *data, uint64_t size,
- int type, zio_priority_t priority, enum zio_flag flags,
+ zio_type_t type, zio_priority_t priority, enum zio_flag flags,
zio_done_func_t *done, void *private)
{
zio_t *zio;
* ==========================================================================
*/
-static int
+static zio_t *
zio_read_bp_init(zio_t *zio)
{
blkptr_t *bp = zio->io_bp;
uint64_t psize =
BP_IS_EMBEDDED(bp) ? BPE_GET_PSIZE(bp) : BP_GET_PSIZE(bp);
+ ASSERT3P(zio->io_bp, ==, &zio->io_bp_copy);
+
if (BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF &&
zio->io_child_type == ZIO_CHILD_LOGICAL &&
!(zio->io_flags & ZIO_FLAG_RAW_COMPRESS)) {
abd_return_buf_copy(zio->io_abd, data, psize);
} else {
ASSERT(!BP_IS_EMBEDDED(bp));
+ ASSERT3P(zio->io_bp, ==, &zio->io_bp_copy);
}
if (!DMU_OT_IS_METADATA(BP_GET_TYPE(bp)) && BP_GET_LEVEL(bp) == 0)
if (BP_GET_DEDUP(bp) && zio->io_child_type == ZIO_CHILD_LOGICAL)
zio->io_pipeline = ZIO_DDT_READ_PIPELINE;
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
-static int
+static zio_t *
zio_write_bp_init(zio_t *zio)
{
if (!IO_IS_ALLOCATING(zio))
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
ASSERT(zio->io_child_type != ZIO_CHILD_DDT);
zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
if (BP_IS_EMBEDDED(bp))
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
/*
* If we've been overridden and nopwrite is set then
ASSERT(!zp->zp_dedup);
ASSERT3U(BP_GET_CHECKSUM(bp), ==, zp->zp_checksum);
zio->io_flags |= ZIO_FLAG_NOPWRITE;
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
ASSERT(!zp->zp_nopwrite);
if (BP_IS_HOLE(bp) || !zp->zp_dedup)
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
ASSERT((zio_checksum_table[zp->zp_checksum].ci_flags &
ZCHECKSUM_FLAG_DEDUP) || zp->zp_dedup_verify);
!zp->zp_encrypt) {
BP_SET_DEDUP(bp, 1);
zio->io_pipeline |= ZIO_STAGE_DDT_WRITE;
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
/*
zio->io_pipeline = zio->io_orig_pipeline;
}
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
-static int
+static zio_t *
zio_write_compress(zio_t *zio)
{
spa_t *spa = zio->io_spa;
* If our children haven't all reached the ready stage,
* wait for them and then repeat this pipeline stage.
*/
- if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_READY) ||
- zio_wait_for_children(zio, ZIO_CHILD_LOGICAL, ZIO_WAIT_READY))
- return (ZIO_PIPELINE_STOP);
+ if (zio_wait_for_children(zio, ZIO_CHILD_LOGICAL_BIT |
+ ZIO_CHILD_GANG_BIT, ZIO_WAIT_READY)) {
+ return (NULL);
+ }
if (!IO_IS_ALLOCATING(zio))
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
if (zio->io_children_ready != NULL) {
/*
zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
ASSERT(spa_feature_is_active(spa,
SPA_FEATURE_EMBEDDED_DATA));
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
} else {
/*
* Round up compressed size up to the ashift
*bp = zio->io_bp_orig;
zio->io_pipeline = zio->io_orig_pipeline;
+ } else if ((zio->io_flags & ZIO_FLAG_RAW_ENCRYPT) != 0 &&
+ zp->zp_type == DMU_OT_DNODE) {
+ /*
+ * The DMU actually relies on the zio layer's compression
+ * to free metadnode blocks that have had all contained
+ * dnodes freed. As a result, even when doing a raw
+ * receive, we must check whether the block can be compressed
+ * to a hole.
+ */
+ psize = zio_compress_data(ZIO_COMPRESS_EMPTY,
+ zio->io_abd, NULL, lsize);
+ if (psize == 0)
+ compress = ZIO_COMPRESS_OFF;
} else {
ASSERT3U(psize, !=, 0);
-
}
/*
if (!BP_IS_HOLE(bp) && bp->blk_birth == zio->io_txg &&
BP_GET_PSIZE(bp) == psize &&
pass >= zfs_sync_pass_rewrite) {
- ASSERT(psize != 0);
+ VERIFY3U(psize, !=, 0);
enum zio_stage gang_stages = zio->io_pipeline & ZIO_GANG_STAGES;
+
zio->io_pipeline = ZIO_REWRITE_PIPELINE | gang_stages;
zio->io_flags |= ZIO_FLAG_IO_REWRITE;
} else {
zio->io_pipeline |= ZIO_STAGE_NOP_WRITE;
}
}
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
-static int
+static zio_t *
zio_free_bp_init(zio_t *zio)
{
blkptr_t *bp = zio->io_bp;
zio->io_pipeline = ZIO_DDT_FREE_PIPELINE;
}
- return (ZIO_PIPELINE_CONTINUE);
+ ASSERT3P(zio->io_bp, ==, &zio->io_bp_copy);
+
+ return (zio);
}
/*
return (B_FALSE);
}
-static int
+static zio_t *
zio_issue_async(zio_t *zio)
{
zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE);
- return (ZIO_PIPELINE_STOP);
+ return (NULL);
}
void
zio_interrupt(zio);
}
+static void
+zio_deadman_impl(zio_t *pio)
+{
+ zio_t *cio, *cio_next;
+ zio_link_t *zl = NULL;
+ vdev_t *vd = pio->io_vd;
+
+ if (vd != NULL && vd->vdev_ops->vdev_op_leaf) {
+ vdev_queue_t *vq = &vd->vdev_queue;
+ zbookmark_phys_t *zb = &pio->io_bookmark;
+ uint64_t delta = gethrtime() - pio->io_timestamp;
+ uint64_t failmode = spa_get_deadman_failmode(pio->io_spa);
+
+ zfs_dbgmsg("slow zio: zio=%p timestamp=%llu "
+ "delta=%llu queued=%llu io=%llu "
+ "path=%s last=%llu "
+ "type=%d priority=%d flags=0x%x "
+ "stage=0x%x pipeline=0x%x pipeline-trace=0x%x "
+ "objset=%llu object=%llu level=%llu blkid=%llu "
+ "offset=%llu size=%llu error=%d",
+ pio, pio->io_timestamp,
+ delta, pio->io_delta, pio->io_delay,
+ vd->vdev_path, vq->vq_io_complete_ts,
+ pio->io_type, pio->io_priority, pio->io_flags,
+ pio->io_state, pio->io_pipeline, pio->io_pipeline_trace,
+ zb->zb_objset, zb->zb_object, zb->zb_level, zb->zb_blkid,
+ pio->io_offset, pio->io_size, pio->io_error);
+ zfs_ereport_post(FM_EREPORT_ZFS_DEADMAN,
+ pio->io_spa, vd, zb, pio, 0, 0);
+
+ if (failmode == ZIO_FAILURE_MODE_CONTINUE &&
+ taskq_empty_ent(&pio->io_tqent)) {
+ zio_interrupt(pio);
+ }
+ }
+
+ mutex_enter(&pio->io_lock);
+ for (cio = zio_walk_children(pio, &zl); cio != NULL; cio = cio_next) {
+ cio_next = zio_walk_children(pio, &zl);
+ zio_deadman_impl(cio);
+ }
+ mutex_exit(&pio->io_lock);
+}
+
+/*
+ * Log the critical information describing this zio and all of its children
+ * using the zfs_dbgmsg() interface then post deadman event for the ZED.
+ */
+void
+zio_deadman(zio_t *pio, char *tag)
+{
+ spa_t *spa = pio->io_spa;
+ char *name = spa_name(spa);
+
+ if (!zfs_deadman_enabled || spa_suspended(spa))
+ return;
+
+ zio_deadman_impl(pio);
+
+ switch (spa_get_deadman_failmode(spa)) {
+ case ZIO_FAILURE_MODE_WAIT:
+ zfs_dbgmsg("%s waiting for hung I/O to pool '%s'", tag, name);
+ break;
+
+ case ZIO_FAILURE_MODE_CONTINUE:
+ zfs_dbgmsg("%s restarting hung I/O for pool '%s'", tag, name);
+ break;
+
+ case ZIO_FAILURE_MODE_PANIC:
+ fm_panic("%s determined I/O to pool '%s' is hung.", tag, name);
+ break;
+ }
+}
+
/*
* Execute the I/O pipeline until one of the following occurs:
* (1) the I/O completes; (2) the pipeline stalls waiting for
static inline void
__zio_execute(zio_t *zio)
{
- zio->io_executor = curthread;
-
ASSERT3U(zio->io_queued_timestamp, >, 0);
while (zio->io_stage < ZIO_STAGE_DONE) {
enum zio_stage pipeline = zio->io_pipeline;
enum zio_stage stage = zio->io_stage;
- int rv;
+
+ zio->io_executor = curthread;
ASSERT(!MUTEX_HELD(&zio->io_lock));
ASSERT(ISP2(stage));
zio->io_stage = stage;
zio->io_pipeline_trace |= zio->io_stage;
- rv = zio_pipeline[highbit64(stage) - 1](zio);
- if (rv == ZIO_PIPELINE_STOP)
- return;
+ /*
+ * The zio pipeline stage returns the next zio to execute
+ * (typically the same as this one), or NULL if we should
+ * stop.
+ */
+ zio = zio_pipeline[highbit64(stage) - 1](zio);
- ASSERT(rv == ZIO_PIPELINE_CONTINUE);
+ if (zio == NULL)
+ return;
}
}
int
zio_wait(zio_t *zio)
{
+ long timeout = MSEC_TO_TICK(zfs_deadman_ziotime_ms);
int error;
ASSERT3S(zio->io_stage, ==, ZIO_STAGE_OPEN);
__zio_execute(zio);
mutex_enter(&zio->io_lock);
- while (zio->io_executor != NULL)
- cv_wait_io(&zio->io_cv, &zio->io_lock);
+ while (zio->io_executor != NULL) {
+ error = cv_timedwait_io(&zio->io_cv, &zio->io_lock,
+ ddi_get_lbolt() + timeout);
+
+ if (zfs_deadman_enabled && error == -1 &&
+ gethrtime() - zio->io_queued_timestamp >
+ spa_deadman_ziotime(zio->io_spa)) {
+ mutex_exit(&zio->io_lock);
+ timeout = MSEC_TO_TICK(zfs_deadman_checktime_ms);
+ zio_deadman(zio, FTAG);
+ mutex_enter(&zio->io_lock);
+ }
+ }
mutex_exit(&zio->io_lock);
error = zio->io_error;
}
void
-zio_cancel(zio_t *zio)
-{
- /*
- * Disallow cancellation of a zio that's already been issued.
- */
- VERIFY3P(zio->io_executor, ==, NULL);
-
- zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
- zio->io_done = NULL;
-
- zio_nowait(zio);
-}
-
-void
-zio_suspend(spa_t *spa, zio_t *zio)
+zio_suspend(spa_t *spa, zio_t *zio, zio_suspend_reason_t reason)
{
if (spa_get_failmode(spa) == ZIO_FAILURE_MODE_PANIC)
fm_panic("Pool '%s' has encountered an uncorrectable I/O "
ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE |
ZIO_FLAG_GODFATHER);
- spa->spa_suspended = B_TRUE;
+ spa->spa_suspended = reason;
if (zio != NULL) {
ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER));
* Reexecute all previously suspended i/o.
*/
mutex_enter(&spa->spa_suspend_lock);
- spa->spa_suspended = B_FALSE;
+ spa->spa_suspended = ZIO_SUSPEND_NONE;
cv_broadcast(&spa->spa_suspend_cv);
pio = spa->spa_suspend_zio_root;
spa->spa_suspend_zio_root = NULL;
zio_nowait(zio);
}
-static int
+static zio_t *
zio_gang_assemble(zio_t *zio)
{
blkptr_t *bp = zio->io_bp;
zio_gang_tree_assemble(zio, bp, &zio->io_gang_tree);
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
-static int
+static zio_t *
zio_gang_issue(zio_t *zio)
{
blkptr_t *bp = zio->io_bp;
- if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_DONE))
- return (ZIO_PIPELINE_STOP);
+ if (zio_wait_for_children(zio, ZIO_CHILD_GANG_BIT, ZIO_WAIT_DONE)) {
+ return (NULL);
+ }
ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == zio);
ASSERT(zio->io_child_type > ZIO_CHILD_GANG);
zio->io_pipeline = ZIO_INTERLOCK_PIPELINE;
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
static void
abd_put(zio->io_abd);
}
-static int
+static zio_t *
zio_write_gang_block(zio_t *pio)
{
spa_t *spa = pio->io_spa;
ASSERT(!(pio->io_flags & ZIO_FLAG_NODATA));
flags |= METASLAB_ASYNC_ALLOC;
- VERIFY(refcount_held(&mc->mc_alloc_slots, pio));
+ VERIFY(zfs_refcount_held(&mc->mc_alloc_slots[pio->io_allocator],
+ pio));
/*
* The logical zio has already placed a reservation for
* additional reservations for gang blocks.
*/
VERIFY(metaslab_class_throttle_reserve(mc, gbh_copies - copies,
- pio, flags));
+ pio->io_allocator, pio, flags));
}
error = metaslab_alloc(spa, mc, SPA_GANGBLOCKSIZE,
bp, gbh_copies, txg, pio == gio ? NULL : gio->io_bp, flags,
- &pio->io_alloc_list, pio);
+ &pio->io_alloc_list, pio, pio->io_allocator);
if (error) {
if (pio->io_flags & ZIO_FLAG_IO_ALLOCATING) {
ASSERT(pio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
* stage.
*/
metaslab_class_throttle_unreserve(mc,
- gbh_copies - copies, pio);
+ gbh_copies - copies, pio->io_allocator, pio);
}
pio->io_error = error;
- return (ZIO_PIPELINE_CONTINUE);
+ return (pio);
}
if (pio == gio) {
* slot for them here.
*/
VERIFY(metaslab_class_throttle_reserve(mc,
- zp.zp_copies, cio, flags));
+ zp.zp_copies, cio->io_allocator, cio, flags));
}
zio_nowait(cio);
}
zio_nowait(zio);
- return (ZIO_PIPELINE_CONTINUE);
+ return (pio);
}
/*
* used for nopwrite, assuming that the salt and the checksums
* themselves remain secret.
*/
-static int
+static zio_t *
zio_nop_write(zio_t *zio)
{
blkptr_t *bp = zio->io_bp;
BP_GET_COMPRESS(bp) != BP_GET_COMPRESS(bp_orig) ||
BP_GET_DEDUP(bp) != BP_GET_DEDUP(bp_orig) ||
zp->zp_copies != BP_GET_NDVAS(bp_orig))
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
/*
* If the checksums match then reset the pipeline so that we
zio->io_flags |= ZIO_FLAG_NOPWRITE;
}
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
/*
mutex_exit(&pio->io_lock);
}
-static int
+static zio_t *
zio_ddt_read_start(zio_t *zio)
{
blkptr_t *bp = zio->io_bp;
zio->io_vsd = dde;
if (ddp_self == NULL)
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) {
if (ddp->ddp_phys_birth == 0 || ddp == ddp_self)
zio->io_priority, ZIO_DDT_CHILD_FLAGS(zio) |
ZIO_FLAG_DONT_PROPAGATE, &zio->io_bookmark));
}
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
zio_nowait(zio_read(zio, zio->io_spa, bp,
zio->io_abd, zio->io_size, NULL, NULL, zio->io_priority,
ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark));
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
-static int
+static zio_t *
zio_ddt_read_done(zio_t *zio)
{
blkptr_t *bp = zio->io_bp;
- if (zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_DONE))
- return (ZIO_PIPELINE_STOP);
+ if (zio_wait_for_children(zio, ZIO_CHILD_DDT_BIT, ZIO_WAIT_DONE)) {
+ return (NULL);
+ }
ASSERT(BP_GET_DEDUP(bp));
ASSERT(BP_GET_PSIZE(bp) == zio->io_size);
ddt_entry_t *dde = zio->io_vsd;
if (ddt == NULL) {
ASSERT(spa_load_state(zio->io_spa) != SPA_LOAD_NONE);
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
if (dde == NULL) {
zio->io_stage = ZIO_STAGE_DDT_READ_START >> 1;
zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE);
- return (ZIO_PIPELINE_STOP);
+ return (NULL);
}
if (dde->dde_repair_abd != NULL) {
abd_copy(zio->io_abd, dde->dde_repair_abd,
ASSERT(zio->io_vsd == NULL);
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
static boolean_t
ddt_exit(ddt);
}
-static int
+static zio_t *
zio_ddt_write(zio_t *zio)
{
spa_t *spa = zio->io_spa;
}
zio->io_pipeline = ZIO_WRITE_PIPELINE;
ddt_exit(ddt);
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
ditto_copies = ddt_ditto_copies_needed(ddt, dde, ddp);
zio->io_bp_override = NULL;
BP_ZERO(bp);
ddt_exit(ddt);
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
dio = zio_write(zio, spa, txg, bp, zio->io_orig_abd,
if (dio)
zio_nowait(dio);
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
ddt_entry_t *freedde; /* for debugging */
-static int
+static zio_t *
zio_ddt_free(zio_t *zio)
{
spa_t *spa = zio->io_spa;
}
ddt_exit(ddt);
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
/*
*/
static zio_t *
-zio_io_to_allocate(spa_t *spa)
+zio_io_to_allocate(spa_t *spa, int allocator)
{
zio_t *zio;
- ASSERT(MUTEX_HELD(&spa->spa_alloc_lock));
+ ASSERT(MUTEX_HELD(&spa->spa_alloc_locks[allocator]));
- zio = avl_first(&spa->spa_alloc_tree);
+ zio = avl_first(&spa->spa_alloc_trees[allocator]);
if (zio == NULL)
return (NULL);
* Try to place a reservation for this zio. If we're unable to
* reserve then we throttle.
*/
- if (!metaslab_class_throttle_reserve(spa_normal_class(spa),
- zio->io_prop.zp_copies, zio, 0)) {
+ ASSERT3U(zio->io_allocator, ==, allocator);
+ if (!metaslab_class_throttle_reserve(zio->io_metaslab_class,
+ zio->io_prop.zp_copies, zio->io_allocator, zio, 0)) {
return (NULL);
}
- avl_remove(&spa->spa_alloc_tree, zio);
+ avl_remove(&spa->spa_alloc_trees[allocator], zio);
ASSERT3U(zio->io_stage, <, ZIO_STAGE_DVA_ALLOCATE);
return (zio);
}
-static int
+static zio_t *
zio_dva_throttle(zio_t *zio)
{
spa_t *spa = zio->io_spa;
zio_t *nio;
+ metaslab_class_t *mc;
+
+ /* locate an appropriate allocation class */
+ mc = spa_preferred_class(spa, zio->io_size, zio->io_prop.zp_type,
+ zio->io_prop.zp_level, zio->io_prop.zp_zpl_smallblk);
if (zio->io_priority == ZIO_PRIORITY_SYNC_WRITE ||
- !spa_normal_class(zio->io_spa)->mc_alloc_throttle_enabled ||
+ !mc->mc_alloc_throttle_enabled ||
zio->io_child_type == ZIO_CHILD_GANG ||
zio->io_flags & ZIO_FLAG_NODATA) {
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
ASSERT(zio->io_child_type > ZIO_CHILD_GANG);
ASSERT3U(zio->io_queued_timestamp, >, 0);
ASSERT(zio->io_stage == ZIO_STAGE_DVA_THROTTLE);
- mutex_enter(&spa->spa_alloc_lock);
-
+ zbookmark_phys_t *bm = &zio->io_bookmark;
+ /*
+ * We want to try to use as many allocators as possible to help improve
+ * performance, but we also want logically adjacent IOs to be physically
+ * adjacent to improve sequential read performance. We chunk each object
+ * into 2^20 block regions, and then hash based on the objset, object,
+ * level, and region to accomplish both of these goals.
+ */
+ zio->io_allocator = cityhash4(bm->zb_objset, bm->zb_object,
+ bm->zb_level, bm->zb_blkid >> 20) % spa->spa_alloc_count;
+ mutex_enter(&spa->spa_alloc_locks[zio->io_allocator]);
ASSERT(zio->io_type == ZIO_TYPE_WRITE);
- avl_add(&spa->spa_alloc_tree, zio);
-
- nio = zio_io_to_allocate(zio->io_spa);
- mutex_exit(&spa->spa_alloc_lock);
-
- if (nio == zio)
- return (ZIO_PIPELINE_CONTINUE);
-
- if (nio != NULL) {
- ASSERT(nio->io_stage == ZIO_STAGE_DVA_THROTTLE);
- /*
- * We are passing control to a new zio so make sure that
- * it is processed by a different thread. We do this to
- * avoid stack overflows that can occur when parents are
- * throttled and children are making progress. We allow
- * it to go to the head of the taskq since it's already
- * been waiting.
- */
- zio_taskq_dispatch(nio, ZIO_TASKQ_ISSUE, B_TRUE);
- }
- return (ZIO_PIPELINE_STOP);
+ zio->io_metaslab_class = mc;
+ avl_add(&spa->spa_alloc_trees[zio->io_allocator], zio);
+ nio = zio_io_to_allocate(spa, zio->io_allocator);
+ mutex_exit(&spa->spa_alloc_locks[zio->io_allocator]);
+ return (nio);
}
-void
-zio_allocate_dispatch(spa_t *spa)
+static void
+zio_allocate_dispatch(spa_t *spa, int allocator)
{
zio_t *zio;
- mutex_enter(&spa->spa_alloc_lock);
- zio = zio_io_to_allocate(spa);
- mutex_exit(&spa->spa_alloc_lock);
+ mutex_enter(&spa->spa_alloc_locks[allocator]);
+ zio = zio_io_to_allocate(spa, allocator);
+ mutex_exit(&spa->spa_alloc_locks[allocator]);
if (zio == NULL)
return;
zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_TRUE);
}
-static int
+static zio_t *
zio_dva_allocate(zio_t *zio)
{
spa_t *spa = zio->io_spa;
- metaslab_class_t *mc = spa_normal_class(spa);
+ metaslab_class_t *mc;
blkptr_t *bp = zio->io_bp;
int error;
int flags = 0;
if (zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE)
flags |= METASLAB_ASYNC_ALLOC;
+ /*
+ * if not already chosen, locate an appropriate allocation class
+ */
+ mc = zio->io_metaslab_class;
+ if (mc == NULL) {
+ mc = spa_preferred_class(spa, zio->io_size,
+ zio->io_prop.zp_type, zio->io_prop.zp_level,
+ zio->io_prop.zp_zpl_smallblk);
+ zio->io_metaslab_class = mc;
+ }
+
error = metaslab_alloc(spa, mc, zio->io_size, bp,
zio->io_prop.zp_copies, zio->io_txg, NULL, flags,
- &zio->io_alloc_list, zio);
+ &zio->io_alloc_list, zio, zio->io_allocator);
+
+ /*
+ * Fallback to normal class when an alloc class is full
+ */
+ if (error == ENOSPC && mc != spa_normal_class(spa)) {
+ /*
+ * If throttling, transfer reservation over to normal class.
+ * The io_allocator slot can remain the same even though we
+ * are switching classes.
+ */
+ if (mc->mc_alloc_throttle_enabled &&
+ (zio->io_flags & ZIO_FLAG_IO_ALLOCATING)) {
+ metaslab_class_throttle_unreserve(mc,
+ zio->io_prop.zp_copies, zio->io_allocator, zio);
+ zio->io_flags &= ~ZIO_FLAG_IO_ALLOCATING;
+
+ mc = spa_normal_class(spa);
+ VERIFY(metaslab_class_throttle_reserve(mc,
+ zio->io_prop.zp_copies, zio->io_allocator, zio,
+ flags | METASLAB_MUST_RESERVE));
+ } else {
+ mc = spa_normal_class(spa);
+ }
+ zio->io_metaslab_class = mc;
+
+ error = metaslab_alloc(spa, mc, zio->io_size, bp,
+ zio->io_prop.zp_copies, zio->io_txg, NULL, flags,
+ &zio->io_alloc_list, zio, zio->io_allocator);
+ }
if (error != 0) {
- spa_dbgmsg(spa, "%s: metaslab allocation failure: zio %p, "
+ zfs_dbgmsg("%s: metaslab allocation failure: zio %p, "
"size %llu, error %d", spa_name(spa), zio, zio->io_size,
error);
if (error == ENOSPC && zio->io_size > SPA_MINBLOCKSIZE)
zio->io_error = error;
}
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
-static int
+static zio_t *
zio_dva_free(zio_t *zio)
{
metaslab_free(zio->io_spa, zio->io_bp, zio->io_txg, B_FALSE);
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
-static int
+static zio_t *
zio_dva_claim(zio_t *zio)
{
int error;
if (error)
zio->io_error = error;
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
/*
ASSERT(txg > spa_syncing_txg(spa));
metaslab_trace_init(&io_alloc_list);
+
+ /*
+ * Block pointer fields are useful to metaslabs for stats and debugging.
+ * Fill in the obvious ones before calling into metaslab_alloc().
+ */
+ BP_SET_TYPE(new_bp, DMU_OT_INTENT_LOG);
+ BP_SET_PSIZE(new_bp, size);
+ BP_SET_LEVEL(new_bp, 0);
+
+ /*
+ * When allocating a zil block, we don't have information about
+ * the final destination of the block except the objset it's part
+ * of, so we just hash the objset ID to pick the allocator to get
+ * some parallelism.
+ */
error = metaslab_alloc(spa, spa_log_class(spa), size, new_bp, 1,
- txg, NULL, METASLAB_FASTWRITE, &io_alloc_list, NULL);
+ txg, NULL, METASLAB_FASTWRITE, &io_alloc_list, NULL,
+ cityhash4(0, 0, 0, os->os_dsl_dataset->ds_object) %
+ spa->spa_alloc_count);
if (error == 0) {
*slog = TRUE;
} else {
error = metaslab_alloc(spa, spa_normal_class(spa), size,
new_bp, 1, txg, NULL, METASLAB_FASTWRITE,
- &io_alloc_list, NULL);
+ &io_alloc_list, NULL, cityhash4(0, 0, 0,
+ os->os_dsl_dataset->ds_object) % spa->spa_alloc_count);
if (error == 0)
*slog = FALSE;
}
return (error);
}
-/*
- * Free an intent log block.
- */
-void
-zio_free_zil(spa_t *spa, uint64_t txg, blkptr_t *bp)
-{
- ASSERT(BP_GET_TYPE(bp) == DMU_OT_INTENT_LOG);
- ASSERT(!BP_IS_GANG(bp));
-
- zio_free(spa, txg, bp);
-}
-
/*
* ==========================================================================
* Read and write to physical devices
* force the underlying vdev layers to call either zio_execute() or
* zio_interrupt() to ensure that the pipeline continues with the correct I/O.
*/
-static int
+static zio_t *
zio_vdev_io_start(zio_t *zio)
{
vdev_t *vd = zio->io_vd;
* The mirror_ops handle multiple DVAs in a single BP.
*/
vdev_mirror_ops.vdev_op_io_start(zio);
- return (ZIO_PIPELINE_STOP);
+ return (NULL);
}
ASSERT3P(zio->io_logical, !=, zio);
+ if (zio->io_type == ZIO_TYPE_WRITE) {
+ ASSERT(spa->spa_trust_config);
+
+ /*
+ * Note: the code can handle other kinds of writes,
+ * but we don't expect them.
+ */
+ if (zio->io_vd->vdev_removing) {
+ ASSERT(zio->io_flags &
+ (ZIO_FLAG_PHYSICAL | ZIO_FLAG_SELF_HEAL |
+ ZIO_FLAG_RESILVER | ZIO_FLAG_INDUCE_DAMAGE));
+ }
+ }
align = 1ULL << vd->vdev_top->vdev_ashift;
* If this is a repair I/O, and there's no self-healing involved --
* that is, we're just resilvering what we expect to resilver --
* then don't do the I/O unless zio's txg is actually in vd's DTL.
- * This prevents spurious resilvering with nested replication.
- * For example, given a mirror of mirrors, (A+B)+(C+D), if only
- * A is out of date, we'll read from C+D, then use the data to
- * resilver A+B -- but we don't actually want to resilver B, just A.
- * The top-level mirror has no way to know this, so instead we just
- * discard unnecessary repairs as we work our way down the vdev tree.
- * The same logic applies to any form of nested replication:
- * ditto + mirror, RAID-Z + replacing, etc. This covers them all.
+ * This prevents spurious resilvering.
+ *
+ * There are a few ways that we can end up creating these spurious
+ * resilver i/os:
+ *
+ * 1. A resilver i/o will be issued if any DVA in the BP has a
+ * dirty DTL. The mirror code will issue resilver writes to
+ * each DVA, including the one(s) that are not on vdevs with dirty
+ * DTLs.
+ *
+ * 2. With nested replication, which happens when we have a
+ * "replacing" or "spare" vdev that's a child of a mirror or raidz.
+ * For example, given mirror(replacing(A+B), C), it's likely that
+ * only A is out of date (it's the new device). In this case, we'll
+ * read from C, then use the data to resilver A+B -- but we don't
+ * actually want to resilver B, just A. The top-level mirror has no
+ * way to know this, so instead we just discard unnecessary repairs
+ * as we work our way down the vdev tree.
+ *
+ * 3. ZTEST also creates mirrors of mirrors, mirrors of raidz, etc.
+ * The same logic applies to any form of nested replication: ditto
+ * + mirror, RAID-Z + replacing, etc.
+ *
+ * However, indirect vdevs point off to other vdevs which may have
+ * DTL's, so we never bypass them. The child i/os on concrete vdevs
+ * will be properly bypassed instead.
*/
if ((zio->io_flags & ZIO_FLAG_IO_REPAIR) &&
!(zio->io_flags & ZIO_FLAG_SELF_HEAL) &&
zio->io_txg != 0 && /* not a delegated i/o */
+ vd->vdev_ops != &vdev_indirect_ops &&
!vdev_dtl_contains(vd, DTL_PARTIAL, zio->io_txg, 1)) {
ASSERT(zio->io_type == ZIO_TYPE_WRITE);
zio_vdev_io_bypass(zio);
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
if (vd->vdev_ops->vdev_op_leaf &&
(zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE)) {
if (zio->io_type == ZIO_TYPE_READ && vdev_cache_read(zio))
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
if ((zio = vdev_queue_io(zio)) == NULL)
- return (ZIO_PIPELINE_STOP);
+ return (NULL);
if (!vdev_accessible(vd, zio)) {
zio->io_error = SET_ERROR(ENXIO);
zio_interrupt(zio);
- return (ZIO_PIPELINE_STOP);
+ return (NULL);
}
zio->io_delay = gethrtime();
}
vd->vdev_ops->vdev_op_io_start(zio);
- return (ZIO_PIPELINE_STOP);
+ return (NULL);
}
-static int
+static zio_t *
zio_vdev_io_done(zio_t *zio)
{
vdev_t *vd = zio->io_vd;
vdev_ops_t *ops = vd ? vd->vdev_ops : &vdev_mirror_ops;
boolean_t unexpected_error = B_FALSE;
- if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE))
- return (ZIO_PIPELINE_STOP);
+ if (zio_wait_for_children(zio, ZIO_CHILD_VDEV_BIT, ZIO_WAIT_DONE)) {
+ return (NULL);
+ }
ASSERT(zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE);
if (unexpected_error)
VERIFY(vdev_probe(vd, zio) == NULL);
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
/*
zcr->zcr_free = zio_abd_free;
}
-static int
+static zio_t *
zio_vdev_io_assess(zio_t *zio)
{
vdev_t *vd = zio->io_vd;
- if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE))
- return (ZIO_PIPELINE_STOP);
+ if (zio_wait_for_children(zio, ZIO_CHILD_VDEV_BIT, ZIO_WAIT_DONE)) {
+ return (NULL);
+ }
if (vd == NULL && !(zio->io_flags & ZIO_FLAG_CONFIG_WRITER))
spa_config_exit(zio->io_spa, SCL_ZIO, zio);
zio->io_stage = ZIO_STAGE_VDEV_IO_START >> 1;
zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE,
zio_requeue_io_start_cut_in_line);
- return (ZIO_PIPELINE_STOP);
+ return (NULL);
}
/*
zio->io_physdone(zio->io_logical);
}
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
void
* managing the storage of encryption parameters and passing them to the
* lower-level encryption functions.
*/
-static int
+static zio_t *
zio_encrypt(zio_t *zio)
{
zio_prop_t *zp = &zio->io_prop;
spa_t *spa = zio->io_spa;
blkptr_t *bp = zio->io_bp;
uint64_t psize = BP_GET_PSIZE(bp);
+ uint64_t dsobj = zio->io_bookmark.zb_objset;
dmu_object_type_t ot = BP_GET_TYPE(bp);
void *enc_buf = NULL;
abd_t *eabd = NULL;
/* the root zio already encrypted the data */
if (zio->io_child_type == ZIO_CHILD_GANG)
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
/* only ZIL blocks are re-encrypted on rewrite */
if (!IO_IS_ALLOCATING(zio) && ot != DMU_OT_INTENT_LOG)
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
if (!(zp->zp_encrypt || BP_IS_ENCRYPTED(bp))) {
BP_SET_CRYPT(bp, B_FALSE);
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
/* if we are doing raw encryption set the provided encryption params */
if (zio->io_flags & ZIO_FLAG_RAW_ENCRYPT) {
+ ASSERT0(BP_GET_LEVEL(bp));
BP_SET_CRYPT(bp, B_TRUE);
BP_SET_BYTEORDER(bp, zp->zp_byteorder);
if (ot != DMU_OT_OBJSET)
zio_crypt_encode_mac_bp(bp, zp->zp_mac);
+
+ /* dnode blocks must be written out in the provided byteorder */
+ if (zp->zp_byteorder != ZFS_HOST_BYTEORDER &&
+ ot == DMU_OT_DNODE) {
+ void *bswap_buf = zio_buf_alloc(psize);
+ abd_t *babd = abd_get_from_buf(bswap_buf, psize);
+
+ ASSERT3U(BP_GET_COMPRESS(bp), ==, ZIO_COMPRESS_OFF);
+ abd_copy_to_buf(bswap_buf, zio->io_abd, psize);
+ dmu_ot_byteswap[DMU_OT_BYTESWAP(ot)].ob_func(bswap_buf,
+ psize);
+
+ abd_take_ownership_of_buf(babd, B_TRUE);
+ zio_push_transform(zio, babd, psize, psize, NULL);
+ }
+
if (DMU_OT_IS_ENCRYPTED(ot))
zio_crypt_encode_params_bp(bp, zp->zp_salt, zp->zp_iv);
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
/* indirect blocks only maintain a cksum of the lower level MACs */
zio->io_orig_abd, BP_GET_LSIZE(bp), BP_SHOULD_BYTESWAP(bp),
mac));
zio_crypt_encode_mac_bp(bp, mac);
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
/*
ASSERT0(DMU_OT_IS_ENCRYPTED(ot));
ASSERT3U(BP_GET_COMPRESS(bp), ==, ZIO_COMPRESS_OFF);
BP_SET_CRYPT(bp, B_TRUE);
- VERIFY0(spa_do_crypt_objset_mac_abd(B_TRUE, spa,
- zio->io_bookmark.zb_objset, zio->io_abd, psize,
- BP_SHOULD_BYTESWAP(bp)));
- return (ZIO_PIPELINE_CONTINUE);
+ VERIFY0(spa_do_crypt_objset_mac_abd(B_TRUE, spa, dsobj,
+ zio->io_abd, psize, BP_SHOULD_BYTESWAP(bp)));
+ return (zio);
}
/* unencrypted object types are only authenticated with a MAC */
if (!DMU_OT_IS_ENCRYPTED(ot)) {
BP_SET_CRYPT(bp, B_TRUE);
- VERIFY0(spa_do_crypt_mac_abd(B_TRUE, spa,
- zio->io_bookmark.zb_objset, zio->io_abd, psize, mac));
+ VERIFY0(spa_do_crypt_mac_abd(B_TRUE, spa, dsobj,
+ zio->io_abd, psize, mac));
zio_crypt_encode_mac_bp(bp, mac);
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
/*
}
/* Perform the encryption. This should not fail */
- VERIFY0(spa_do_crypt_abd(B_TRUE, spa, zio->io_bookmark.zb_objset, bp,
- zio->io_txg, psize, zio->io_abd, eabd, iv, mac, salt, &no_crypt));
+ VERIFY0(spa_do_crypt_abd(B_TRUE, spa, &zio->io_bookmark,
+ BP_GET_TYPE(bp), BP_GET_DEDUP(bp), BP_SHOULD_BYTESWAP(bp),
+ salt, iv, mac, psize, zio->io_abd, eabd, &no_crypt));
/* encode encryption metadata into the bp */
if (ot == DMU_OT_INTENT_LOG) {
}
}
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
/*
* Generate and verify checksums
* ==========================================================================
*/
-static int
+static zio_t *
zio_checksum_generate(zio_t *zio)
{
blkptr_t *bp = zio->io_bp;
checksum = zio->io_prop.zp_checksum;
if (checksum == ZIO_CHECKSUM_OFF)
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
ASSERT(checksum == ZIO_CHECKSUM_LABEL);
} else {
zio_checksum_compute(zio, checksum, zio->io_abd, zio->io_size);
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
-static int
+static zio_t *
zio_checksum_verify(zio_t *zio)
{
zio_bad_cksum_t info;
* We're either verifying a label checksum, or nothing at all.
*/
if (zio->io_prop.zp_checksum == ZIO_CHECKSUM_OFF)
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
ASSERT(zio->io_prop.zp_checksum == ZIO_CHECKSUM_LABEL);
}
}
}
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
/*
* I/O completion
* ==========================================================================
*/
-static int
+static zio_t *
zio_ready(zio_t *zio)
{
blkptr_t *bp = zio->io_bp;
zio_t *pio, *pio_next;
zio_link_t *zl = NULL;
- if (zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_READY) ||
- zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_READY))
- return (ZIO_PIPELINE_STOP);
+ if (zio_wait_for_children(zio, ZIO_CHILD_GANG_BIT | ZIO_CHILD_DDT_BIT,
+ ZIO_WAIT_READY)) {
+ return (NULL);
+ }
if (zio->io_ready) {
ASSERT(IO_IS_ALLOCATING(zio));
if (zio->io_flags & ZIO_FLAG_IO_ALLOCATING) {
ASSERT(IO_IS_ALLOCATING(zio));
ASSERT(zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
+ ASSERT(zio->io_metaslab_class != NULL);
+
/*
* We were unable to allocate anything, unreserve and
* issue the next I/O to allocate.
*/
metaslab_class_throttle_unreserve(
- spa_normal_class(zio->io_spa),
- zio->io_prop.zp_copies, zio);
- zio_allocate_dispatch(zio->io_spa);
+ zio->io_metaslab_class, zio->io_prop.zp_copies,
+ zio->io_allocator, zio);
+ zio_allocate_dispatch(zio->io_spa, zio->io_allocator);
}
}
*/
for (; pio != NULL; pio = pio_next) {
pio_next = zio_walk_parents(zio, &zl);
- zio_notify_parent(pio, zio, ZIO_WAIT_READY);
+ zio_notify_parent(pio, zio, ZIO_WAIT_READY, NULL);
}
if (zio->io_flags & ZIO_FLAG_NODATA) {
zio->io_spa->spa_syncing_txg == zio->io_txg)
zio_handle_ignored_writes(zio);
- return (ZIO_PIPELINE_CONTINUE);
+ return (zio);
}
/*
ASSERT(zio->io_logical != NULL);
ASSERT(!(zio->io_flags & ZIO_FLAG_IO_REPAIR));
ASSERT0(zio->io_flags & ZIO_FLAG_NOPWRITE);
+ ASSERT(zio->io_metaslab_class != NULL);
mutex_enter(&pio->io_lock);
- metaslab_group_alloc_decrement(zio->io_spa, vd->vdev_id, pio, flags);
+ metaslab_group_alloc_decrement(zio->io_spa, vd->vdev_id, pio, flags,
+ pio->io_allocator, B_TRUE);
mutex_exit(&pio->io_lock);
- metaslab_class_throttle_unreserve(spa_normal_class(zio->io_spa),
- 1, pio);
+ metaslab_class_throttle_unreserve(zio->io_metaslab_class, 1,
+ pio->io_allocator, pio);
/*
* Call into the pipeline to see if there is more work that
* needs to be done. If there is work to be done it will be
* dispatched to another taskq thread.
*/
- zio_allocate_dispatch(zio->io_spa);
+ zio_allocate_dispatch(zio->io_spa, pio->io_allocator);
}
-static int
+static zio_t *
zio_done(zio_t *zio)
{
/*
* If our children haven't all completed,
* wait for them and then repeat this pipeline stage.
*/
- if (zio_wait_for_children(zio, ZIO_CHILD_VDEV, ZIO_WAIT_DONE) ||
- zio_wait_for_children(zio, ZIO_CHILD_GANG, ZIO_WAIT_DONE) ||
- zio_wait_for_children(zio, ZIO_CHILD_DDT, ZIO_WAIT_DONE) ||
- zio_wait_for_children(zio, ZIO_CHILD_LOGICAL, ZIO_WAIT_DONE))
- return (ZIO_PIPELINE_STOP);
+ if (zio_wait_for_children(zio, ZIO_CHILD_ALL_BITS, ZIO_WAIT_DONE)) {
+ return (NULL);
+ }
/*
* If the allocation throttle is enabled, then update the accounting.
*/
if (zio->io_flags & ZIO_FLAG_IO_ALLOCATING &&
zio->io_child_type == ZIO_CHILD_VDEV) {
- ASSERT(spa_normal_class(
- zio->io_spa)->mc_alloc_throttle_enabled);
+ ASSERT(zio->io_metaslab_class != NULL);
+ ASSERT(zio->io_metaslab_class->mc_alloc_throttle_enabled);
zio_dva_throttle_done(zio);
}
ASSERT(zio->io_type == ZIO_TYPE_WRITE);
ASSERT(zio->io_priority == ZIO_PRIORITY_ASYNC_WRITE);
ASSERT(zio->io_bp != NULL);
- metaslab_group_alloc_verify(zio->io_spa, zio->io_bp, zio);
- VERIFY(refcount_not_held(
- &(spa_normal_class(zio->io_spa)->mc_alloc_slots), zio));
+
+ metaslab_group_alloc_verify(zio->io_spa, zio->io_bp, zio,
+ zio->io_allocator);
+ VERIFY(zfs_refcount_not_held(
+ &zio->io_metaslab_class->mc_alloc_slots[zio->io_allocator],
+ zio));
}
if (zio->io_type == ZIO_TYPE_WRITE && !BP_IS_HOLE(zio->io_bp) &&
zio->io_bp_override == NULL &&
!(zio->io_flags & ZIO_FLAG_IO_REPAIR)) {
- ASSERT(!BP_SHOULD_BYTESWAP(zio->io_bp));
ASSERT3U(zio->io_prop.zp_copies, <=,
BP_GET_NDVAS(zio->io_bp));
ASSERT(BP_COUNT_GANG(zio->io_bp) == 0 ||
if ((pio->io_flags & ZIO_FLAG_GODFATHER) &&
(zio->io_reexecute & ZIO_REEXECUTE_SUSPEND)) {
zio_remove_child(pio, zio, remove_zl);
- zio_notify_parent(pio, zio, ZIO_WAIT_DONE);
+ /*
+ * This is a rare code path, so we don't
+ * bother with "next_to_execute".
+ */
+ zio_notify_parent(pio, zio, ZIO_WAIT_DONE,
+ NULL);
}
}
*/
ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER));
zio->io_flags |= ZIO_FLAG_DONT_PROPAGATE;
- zio_notify_parent(pio, zio, ZIO_WAIT_DONE);
+ /*
+ * This is a rare code path, so we don't bother with
+ * "next_to_execute".
+ */
+ zio_notify_parent(pio, zio, ZIO_WAIT_DONE, NULL);
} else if (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND) {
/*
* We'd fail again if we reexecuted now, so suspend
* until conditions improve (e.g. device comes online).
*/
- zio_suspend(zio->io_spa, zio);
+ zio_suspend(zio->io_spa, zio, ZIO_SUSPEND_IOERR);
} else {
/*
* Reexecution is potentially a huge amount of work.
(task_func_t *)zio_reexecute, zio, 0,
&zio->io_tqent);
}
- return (ZIO_PIPELINE_STOP);
+ return (NULL);
}
ASSERT(zio->io_child_count == 0);
zio->io_state[ZIO_WAIT_DONE] = 1;
mutex_exit(&zio->io_lock);
+ /*
+ * We are done executing this zio. We may want to execute a parent
+ * next. See the comment in zio_notify_parent().
+ */
+ zio_t *next_to_execute = NULL;
zl = NULL;
for (pio = zio_walk_parents(zio, &zl); pio != NULL; pio = pio_next) {
zio_link_t *remove_zl = zl;
pio_next = zio_walk_parents(zio, &zl);
zio_remove_child(pio, zio, remove_zl);
- zio_notify_parent(pio, zio, ZIO_WAIT_DONE);
+ zio_notify_parent(pio, zio, ZIO_WAIT_DONE, &next_to_execute);
}
if (zio->io_waiter != NULL) {
zio_destroy(zio);
}
- return (ZIO_PIPELINE_STOP);
+ return (next_to_execute);
}
/*
last_block) <= 0);
}
-#if defined(_KERNEL) && defined(HAVE_SPL)
+#if defined(_KERNEL)
EXPORT_SYMBOL(zio_type_name);
EXPORT_SYMBOL(zio_buf_alloc);
EXPORT_SYMBOL(zio_data_buf_alloc);
projects / mirror_zfs.git / blobdiff