[mirror_zfs-debian.git] / module / zfs / vdev_mirror.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright (c) 2013 by Delphix. All rights reserved.
 */

#include <sys/zfs_context.h>
#include <sys/spa.h>
#include <sys/vdev_impl.h>
#include <sys/zio.h>
#include <sys/fs/zfs.h>

/*
 * Virtual device vector for mirroring.
 */

typedef struct mirror_child {
	vdev_t		*mc_vd;
	uint64_t	mc_offset;
	int		mc_error;
	int		mc_pending;
	uint8_t		mc_tried;
	uint8_t		mc_skipped;
	uint8_t		mc_speculative;
} mirror_child_t;

typedef struct mirror_map {
	int		mm_children;
	int		mm_replacing;
	int		mm_preferred;
	int		mm_root;
	mirror_child_t	mm_child[1];
} mirror_map_t;

/*
 * When the children are equally busy queue incoming requests to a single
 * child for N microseconds.  This is done to maximize the likelihood that
 * the Linux elevator will be able to merge requests while it is plugged.
 * Otherwise, requests are queued to the least busy device.
 *
 * For rotational disks the Linux elevator will plug for 10ms which is
 * why zfs_vdev_mirror_switch_us is set to 10ms by default.  For non-
 * rotational disks the elevator will not plug, but 10ms is still a small
 * enough value that the requests will get spread over all the children.
 *
 * For fast SSDs it may make sense to decrease zfs_vdev_mirror_switch_us
 * significantly to bound the worst case latencies.  It would probably be
 * ideal to calculate a decaying average of the last observed latencies and
 * use that to dynamically adjust the zfs_vdev_mirror_switch_us time.
 */
int zfs_vdev_mirror_switch_us = 10000;

static void
vdev_mirror_map_free(zio_t *zio)
{
	mirror_map_t *mm = zio->io_vsd;

	kmem_free(mm, offsetof(mirror_map_t, mm_child[mm->mm_children]));
}

static const zio_vsd_ops_t vdev_mirror_vsd_ops = {
	vdev_mirror_map_free,
	zio_vsd_default_cksum_report
};

static int
vdev_mirror_pending(vdev_t *vd)
{
	return (avl_numnodes(&vd->vdev_queue.vq_active_tree));
}

/*
 * Avoid inlining the function to keep vdev_mirror_io_start(), which
 * is this functions only caller, as small as possible on the stack.
 */
noinline static mirror_map_t *
vdev_mirror_map_alloc(zio_t *zio)
{
	mirror_map_t *mm = NULL;
	mirror_child_t *mc;
	vdev_t *vd = zio->io_vd;
	int c, d;

	if (vd == NULL) {
		dva_t *dva = zio->io_bp->blk_dva;
		spa_t *spa = zio->io_spa;

		c = BP_GET_NDVAS(zio->io_bp);

		mm = kmem_zalloc(offsetof(mirror_map_t, mm_child[c]),
		    KM_SLEEP);
		mm->mm_children = c;
		mm->mm_replacing = B_FALSE;
		mm->mm_preferred = spa_get_random(c);
		mm->mm_root = B_TRUE;

		/*
		 * Check the other, lower-index DVAs to see if they're on
		 * the same vdev as the child we picked.  If they are, use
		 * them since they are likely to have been allocated from
		 * the primary metaslab in use at the time, and hence are
		 * more likely to have locality with single-copy data.
		 */
		for (c = mm->mm_preferred, d = c - 1; d >= 0; d--) {
			if (DVA_GET_VDEV(&dva[d]) == DVA_GET_VDEV(&dva[c]))
				mm->mm_preferred = d;
		}

		for (c = 0; c < mm->mm_children; c++) {
			mc = &mm->mm_child[c];

			mc->mc_vd = vdev_lookup_top(spa, DVA_GET_VDEV(&dva[c]));
			mc->mc_offset = DVA_GET_OFFSET(&dva[c]);
		}
	} else {
		int lowest_pending = INT_MAX;
		int lowest_nr = 1;

		c = vd->vdev_children;

		mm = kmem_zalloc(offsetof(mirror_map_t, mm_child[c]),
		    KM_SLEEP);
		mm->mm_children = c;
		mm->mm_replacing = (vd->vdev_ops == &vdev_replacing_ops ||
		    vd->vdev_ops == &vdev_spare_ops);
		mm->mm_preferred = 0;
		mm->mm_root = B_FALSE;

		for (c = 0; c < mm->mm_children; c++) {
			mc = &mm->mm_child[c];
			mc->mc_vd = vd->vdev_child[c];
			mc->mc_offset = zio->io_offset;

			if (mm->mm_replacing)
				continue;

			if (!vdev_readable(mc->mc_vd)) {
				mc->mc_error = SET_ERROR(ENXIO);
				mc->mc_tried = 1;
				mc->mc_skipped = 1;
				mc->mc_pending = INT_MAX;
				continue;
			}

			mc->mc_pending = vdev_mirror_pending(mc->mc_vd);
			if (mc->mc_pending < lowest_pending) {
				lowest_pending = mc->mc_pending;
				lowest_nr = 1;
			} else if (mc->mc_pending == lowest_pending) {
				lowest_nr++;
			}
		}

		d = gethrtime() / (NSEC_PER_USEC * zfs_vdev_mirror_switch_us);
		d = (d % lowest_nr) + 1;

		for (c = 0; c < mm->mm_children; c++) {
			mc = &mm->mm_child[c];

			if (mm->mm_child[c].mc_pending == lowest_pending) {
				if (--d == 0) {
					mm->mm_preferred = c;
					break;
				}
			}
		}
	}

	zio->io_vsd = mm;
	zio->io_vsd_ops = &vdev_mirror_vsd_ops;
	return (mm);
}

static int
vdev_mirror_open(vdev_t *vd, uint64_t *asize, uint64_t *max_asize,
    uint64_t *ashift)
{
	int numerrors = 0;
	int lasterror = 0;
	int c;

	if (vd->vdev_children == 0) {
		vd->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL;
		return (SET_ERROR(EINVAL));
	}

	vdev_open_children(vd);

	for (c = 0; c < vd->vdev_children; c++) {
		vdev_t *cvd = vd->vdev_child[c];

		if (cvd->vdev_open_error) {
			lasterror = cvd->vdev_open_error;
			numerrors++;
			continue;
		}

		*asize = MIN(*asize - 1, cvd->vdev_asize - 1) + 1;
		*max_asize = MIN(*max_asize - 1, cvd->vdev_max_asize - 1) + 1;
		*ashift = MAX(*ashift, cvd->vdev_ashift);
	}

	if (numerrors == vd->vdev_children) {
		vd->vdev_stat.vs_aux = VDEV_AUX_NO_REPLICAS;
		return (lasterror);
	}

	return (0);
}

static void
vdev_mirror_close(vdev_t *vd)
{
	int c;

	for (c = 0; c < vd->vdev_children; c++)
		vdev_close(vd->vdev_child[c]);
}

static void
vdev_mirror_child_done(zio_t *zio)
{
	mirror_child_t *mc = zio->io_private;

	mc->mc_error = zio->io_error;
	mc->mc_tried = 1;
	mc->mc_skipped = 0;
}

static void
vdev_mirror_scrub_done(zio_t *zio)
{
	mirror_child_t *mc = zio->io_private;

	if (zio->io_error == 0) {
		zio_t *pio;

		mutex_enter(&zio->io_lock);
		while ((pio = zio_walk_parents(zio)) != NULL) {
			mutex_enter(&pio->io_lock);
			ASSERT3U(zio->io_size, >=, pio->io_size);
			bcopy(zio->io_data, pio->io_data, pio->io_size);
			mutex_exit(&pio->io_lock);
		}
		mutex_exit(&zio->io_lock);
	}

	zio_buf_free(zio->io_data, zio->io_size);

	mc->mc_error = zio->io_error;
	mc->mc_tried = 1;
	mc->mc_skipped = 0;
}

/*
 * Try to find a child whose DTL doesn't contain the block we want to read.
 * If we can't, try the read on any vdev we haven't already tried.
 */
static int
vdev_mirror_child_select(zio_t *zio)
{
	mirror_map_t *mm = zio->io_vsd;
	mirror_child_t *mc;
	uint64_t txg = zio->io_txg;
	int i, c;

	ASSERT(zio->io_bp == NULL || BP_PHYSICAL_BIRTH(zio->io_bp) == txg);

	/*
	 * Try to find a child whose DTL doesn't contain the block to read.
	 * If a child is known to be completely inaccessible (indicated by
	 * vdev_readable() returning B_FALSE), don't even try.
	 */
	for (i = 0, c = mm->mm_preferred; i < mm->mm_children; i++, c++) {
		if (c >= mm->mm_children)
			c = 0;
		mc = &mm->mm_child[c];
		if (mc->mc_tried || mc->mc_skipped)
			continue;
		if (mc->mc_vd == NULL || !vdev_readable(mc->mc_vd)) {
			mc->mc_error = SET_ERROR(ENXIO);
			mc->mc_tried = 1;	/* don't even try */
			mc->mc_skipped = 1;
			continue;
		}
		if (!vdev_dtl_contains(mc->mc_vd, DTL_MISSING, txg, 1))
			return (c);
		mc->mc_error = SET_ERROR(ESTALE);
		mc->mc_skipped = 1;
		mc->mc_speculative = 1;
	}

	/*
	 * Every device is either missing or has this txg in its DTL.
	 * Look for any child we haven't already tried before giving up.
	 */
	for (c = 0; c < mm->mm_children; c++)
		if (!mm->mm_child[c].mc_tried)
			return (c);

	/*
	 * Every child failed.  There's no place left to look.
	 */
	return (-1);
}

static int
vdev_mirror_io_start(zio_t *zio)
{
	mirror_map_t *mm;
	mirror_child_t *mc;
	int c, children;

	mm = vdev_mirror_map_alloc(zio);

	if (zio->io_type == ZIO_TYPE_READ) {
		if ((zio->io_flags & ZIO_FLAG_SCRUB) && !mm->mm_replacing) {
			/*
			 * For scrubbing reads we need to allocate a read
			 * buffer for each child and issue reads to all
			 * children.  If any child succeeds, it will copy its
			 * data into zio->io_data in vdev_mirror_scrub_done.
			 */
			for (c = 0; c < mm->mm_children; c++) {
				mc = &mm->mm_child[c];
				zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
				    mc->mc_vd, mc->mc_offset,
				    zio_buf_alloc(zio->io_size), zio->io_size,
				    zio->io_type, zio->io_priority, 0,
				    vdev_mirror_scrub_done, mc));
			}
			return (ZIO_PIPELINE_CONTINUE);
		}
		/*
		 * For normal reads just pick one child.
		 */
		c = vdev_mirror_child_select(zio);
		children = (c >= 0);
	} else {
		ASSERT(zio->io_type == ZIO_TYPE_WRITE);

		/*
		 * Writes go to all children.
		 */
		c = 0;
		children = mm->mm_children;
	}

	while (children--) {
		mc = &mm->mm_child[c];
		zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
		    mc->mc_vd, mc->mc_offset, zio->io_data, zio->io_size,
		    zio->io_type, zio->io_priority, 0,
		    vdev_mirror_child_done, mc));
		c++;
	}

	return (ZIO_PIPELINE_CONTINUE);
}

static int
vdev_mirror_worst_error(mirror_map_t *mm)
{
	int c, error[2] = { 0, 0 };

	for (c = 0; c < mm->mm_children; c++) {
		mirror_child_t *mc = &mm->mm_child[c];
		int s = mc->mc_speculative;
		error[s] = zio_worst_error(error[s], mc->mc_error);
	}

	return (error[0] ? error[0] : error[1]);
}

static void
vdev_mirror_io_done(zio_t *zio)
{
	mirror_map_t *mm = zio->io_vsd;
	mirror_child_t *mc;
	int c;
	int good_copies = 0;
	int unexpected_errors = 0;

	for (c = 0; c < mm->mm_children; c++) {
		mc = &mm->mm_child[c];

		if (mc->mc_error) {
			if (!mc->mc_skipped)
				unexpected_errors++;
		} else if (mc->mc_tried) {
			good_copies++;
		}
	}

	if (zio->io_type == ZIO_TYPE_WRITE) {
		/*
		 * XXX -- for now, treat partial writes as success.
		 *
		 * Now that we support write reallocation, it would be better
		 * to treat partial failure as real failure unless there are
		 * no non-degraded top-level vdevs left, and not update DTLs
		 * if we intend to reallocate.
		 */
		/* XXPOLICY */
		if (good_copies != mm->mm_children) {
			/*
			 * Always require at least one good copy.
			 *
			 * For ditto blocks (io_vd == NULL), require
			 * all copies to be good.
			 *
			 * XXX -- for replacing vdevs, there's no great answer.
			 * If the old device is really dead, we may not even
			 * be able to access it -- so we only want to
			 * require good writes to the new device.  But if
			 * the new device turns out to be flaky, we want
			 * to be able to detach it -- which requires all
			 * writes to the old device to have succeeded.
			 */
			if (good_copies == 0 || zio->io_vd == NULL)
				zio->io_error = vdev_mirror_worst_error(mm);
		}
		return;
	}

	ASSERT(zio->io_type == ZIO_TYPE_READ);

	/*
	 * If we don't have a good copy yet, keep trying other children.
	 */
	/* XXPOLICY */
	if (good_copies == 0 && (c = vdev_mirror_child_select(zio)) != -1) {
		ASSERT(c >= 0 && c < mm->mm_children);
		mc = &mm->mm_child[c];
		zio_vdev_io_redone(zio);
		zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
		    mc->mc_vd, mc->mc_offset, zio->io_data, zio->io_size,
		    ZIO_TYPE_READ, zio->io_priority, 0,
		    vdev_mirror_child_done, mc));
		return;
	}

	/* XXPOLICY */
	if (good_copies == 0) {
		zio->io_error = vdev_mirror_worst_error(mm);
		ASSERT(zio->io_error != 0);
	}

	if (good_copies && spa_writeable(zio->io_spa) &&
	    (unexpected_errors ||
	    (zio->io_flags & ZIO_FLAG_RESILVER) ||
	    ((zio->io_flags & ZIO_FLAG_SCRUB) && mm->mm_replacing))) {
		/*
		 * Use the good data we have in hand to repair damaged children.
		 */
		for (c = 0; c < mm->mm_children; c++) {
			/*
			 * Don't rewrite known good children.
			 * Not only is it unnecessary, it could
			 * actually be harmful: if the system lost
			 * power while rewriting the only good copy,
			 * there would be no good copies left!
			 */
			mc = &mm->mm_child[c];

			if (mc->mc_error == 0) {
				if (mc->mc_tried)
					continue;
				if (!(zio->io_flags & ZIO_FLAG_SCRUB) &&
				    !vdev_dtl_contains(mc->mc_vd, DTL_PARTIAL,
				    zio->io_txg, 1))
					continue;
				mc->mc_error = SET_ERROR(ESTALE);
			}

			zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
			    mc->mc_vd, mc->mc_offset,
			    zio->io_data, zio->io_size,
			    ZIO_TYPE_WRITE, ZIO_PRIORITY_ASYNC_WRITE,
			    ZIO_FLAG_IO_REPAIR | (unexpected_errors ?
			    ZIO_FLAG_SELF_HEAL : 0), NULL, NULL));
		}
	}
}

static void
vdev_mirror_state_change(vdev_t *vd, int faulted, int degraded)
{
	if (faulted == vd->vdev_children)
		vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
		    VDEV_AUX_NO_REPLICAS);
	else if (degraded + faulted != 0)
		vdev_set_state(vd, B_FALSE, VDEV_STATE_DEGRADED, VDEV_AUX_NONE);
	else
		vdev_set_state(vd, B_FALSE, VDEV_STATE_HEALTHY, VDEV_AUX_NONE);
}

vdev_ops_t vdev_mirror_ops = {
	vdev_mirror_open,
	vdev_mirror_close,
	vdev_default_asize,
	vdev_mirror_io_start,
	vdev_mirror_io_done,
	vdev_mirror_state_change,
	NULL,
	NULL,
	VDEV_TYPE_MIRROR,	/* name of this vdev type */
	B_FALSE			/* not a leaf vdev */
};

vdev_ops_t vdev_replacing_ops = {
	vdev_mirror_open,
	vdev_mirror_close,
	vdev_default_asize,
	vdev_mirror_io_start,
	vdev_mirror_io_done,
	vdev_mirror_state_change,
	NULL,
	NULL,
	VDEV_TYPE_REPLACING,	/* name of this vdev type */
	B_FALSE			/* not a leaf vdev */
};

vdev_ops_t vdev_spare_ops = {
	vdev_mirror_open,
	vdev_mirror_close,
	vdev_default_asize,
	vdev_mirror_io_start,
	vdev_mirror_io_done,
	vdev_mirror_state_change,
	NULL,
	NULL,
	VDEV_TYPE_SPARE,	/* name of this vdev type */
	B_FALSE			/* not a leaf vdev */
};

#if defined(_KERNEL) && defined(HAVE_SPL)
module_param(zfs_vdev_mirror_switch_us, int, 0644);
MODULE_PARM_DESC(zfs_vdev_mirror_switch_us, "Switch mirrors every N usecs");
#endif
Commit	Line	Data
34dc7c2f BB	1	/*
	2	* CDDL HEADER START
	3	*
	4	* The contents of this file are subject to the terms of the
	5	* Common Development and Distribution License (the "License").
	6	* You may not use this file except in compliance with the License.
	7	*
	8	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	9	* or http://www.opensolaris.org/os/licensing.
	10	* See the License for the specific language governing permissions
	11	* and limitations under the License.
	12	*
	13	* When distributing Covered Code, include this CDDL HEADER in each
	14	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	15	* If applicable, add the following below this CDDL HEADER, with the
	16	* fields enclosed by brackets "[]" replaced with your own identifying
	17	* information: Portions Copyright [yyyy] [name of copyright owner]
	18	*
	19	* CDDL HEADER END
	20	*/
	21	/*
428870ff	22	* Copyright 2010 Sun Microsystems, Inc. All rights reserved.
34dc7c2f BB	23	* Use is subject to license terms.
	24	*/
	25
1bd201e7	26	/*
a08ee875	27	* Copyright (c) 2013 by Delphix. All rights reserved.
1bd201e7 CS	28	*/
1bd201e7 CS	29
34dc7c2f BB	30	#include <sys/zfs_context.h>
	31	#include <sys/spa.h>
	32	#include <sys/vdev_impl.h>
	33	#include <sys/zio.h>
	34	#include <sys/fs/zfs.h>
	35
	36	/*
	37	* Virtual device vector for mirroring.
	38	*/
	39
	40	typedef struct mirror_child {
	41	vdev_t *mc_vd;
	42	uint64_t mc_offset;
	43	int mc_error;
c06d4368	44	int mc_pending;
b128c09f BB	45	uint8_t mc_tried;
	46	uint8_t mc_skipped;
	47	uint8_t mc_speculative;
34dc7c2f BB	48	} mirror_child_t;
	49
	50	typedef struct mirror_map {
	51	int mm_children;
	52	int mm_replacing;
	53	int mm_preferred;
	54	int mm_root;
	55	mirror_child_t mm_child[1];
	56	} mirror_map_t;
	57
c06d4368 AX	58	/*
	59	* When the children are equally busy queue incoming requests to a single
	60	* child for N microseconds. This is done to maximize the likelihood that
	61	* the Linux elevator will be able to merge requests while it is plugged.
	62	* Otherwise, requests are queued to the least busy device.
	63	*
	64	* For rotational disks the Linux elevator will plug for 10ms which is
	65	* why zfs_vdev_mirror_switch_us is set to 10ms by default. For non-
	66	* rotational disks the elevator will not plug, but 10ms is still a small
	67	* enough value that the requests will get spread over all the children.
	68	*
	69	* For fast SSDs it may make sense to decrease zfs_vdev_mirror_switch_us
	70	* significantly to bound the worst case latencies. It would probably be
	71	* ideal to calculate a decaying average of the last observed latencies and
	72	* use that to dynamically adjust the zfs_vdev_mirror_switch_us time.
	73	*/
	74	int zfs_vdev_mirror_switch_us = 10000;
34dc7c2f	75
b128c09f BB	76	static void
	77	vdev_mirror_map_free(zio_t *zio)
	78	{
	79	mirror_map_t *mm = zio->io_vsd;
	80
	81	kmem_free(mm, offsetof(mirror_map_t, mm_child[mm->mm_children]));
	82	}
	83
428870ff BB	84	static const zio_vsd_ops_t vdev_mirror_vsd_ops = {
	85	vdev_mirror_map_free,
	86	zio_vsd_default_cksum_report
	87	};
	88
c06d4368 AX	89	static int
	90	vdev_mirror_pending(vdev_t *vd)
	91	{
a08ee875	92	return (avl_numnodes(&vd->vdev_queue.vq_active_tree));
c06d4368 AX	93	}
c06d4368 AX	94
a08ee875 LG	95	/*
	96	* Avoid inlining the function to keep vdev_mirror_io_start(), which
	97	* is this functions only caller, as small as possible on the stack.
	98	*/
	99	noinline static mirror_map_t *
34dc7c2f BB	100	vdev_mirror_map_alloc(zio_t *zio)
	101	{
	102	mirror_map_t *mm = NULL;
	103	mirror_child_t *mc;
	104	vdev_t *vd = zio->io_vd;
	105	int c, d;
	106
	107	if (vd == NULL) {
	108	dva_t *dva = zio->io_bp->blk_dva;
	109	spa_t *spa = zio->io_spa;
	110
	111	c = BP_GET_NDVAS(zio->io_bp);
	112
a08ee875	113	mm = kmem_zalloc(offsetof(mirror_map_t, mm_child[c]),
ea04106b	114	KM_SLEEP);
34dc7c2f BB	115	mm->mm_children = c;
	116	mm->mm_replacing = B_FALSE;
	117	mm->mm_preferred = spa_get_random(c);
	118	mm->mm_root = B_TRUE;
	119
	120	/*
	121	* Check the other, lower-index DVAs to see if they're on
	122	* the same vdev as the child we picked. If they are, use
	123	* them since they are likely to have been allocated from
	124	* the primary metaslab in use at the time, and hence are
	125	* more likely to have locality with single-copy data.
	126	*/
	127	for (c = mm->mm_preferred, d = c - 1; d >= 0; d--) {
	128	if (DVA_GET_VDEV(&dva[d]) == DVA_GET_VDEV(&dva[c]))
	129	mm->mm_preferred = d;
	130	}
	131
	132	for (c = 0; c < mm->mm_children; c++) {
	133	mc = &mm->mm_child[c];
	134
	135	mc->mc_vd = vdev_lookup_top(spa, DVA_GET_VDEV(&dva[c]));
	136	mc->mc_offset = DVA_GET_OFFSET(&dva[c]);
	137	}
	138	} else {
c06d4368 AX	139	int lowest_pending = INT_MAX;
	140	int lowest_nr = 1;
	141
34dc7c2f BB	142	c = vd->vdev_children;
34dc7c2f BB	143
a08ee875	144	mm = kmem_zalloc(offsetof(mirror_map_t, mm_child[c]),
ea04106b	145	KM_SLEEP);
34dc7c2f BB	146	mm->mm_children = c;
	147	mm->mm_replacing = (vd->vdev_ops == &vdev_replacing_ops \|\|
	148	vd->vdev_ops == &vdev_spare_ops);
c06d4368	149	mm->mm_preferred = 0;
34dc7c2f BB	150	mm->mm_root = B_FALSE;
	151
	152	for (c = 0; c < mm->mm_children; c++) {
	153	mc = &mm->mm_child[c];
	154	mc->mc_vd = vd->vdev_child[c];
	155	mc->mc_offset = zio->io_offset;
c06d4368 AX	156
	157	if (mm->mm_replacing)
	158	continue;
	159
	160	if (!vdev_readable(mc->mc_vd)) {
a08ee875	161	mc->mc_error = SET_ERROR(ENXIO);
c06d4368 AX	162	mc->mc_tried = 1;
	163	mc->mc_skipped = 1;
	164	mc->mc_pending = INT_MAX;
	165	continue;
	166	}
	167
	168	mc->mc_pending = vdev_mirror_pending(mc->mc_vd);
	169	if (mc->mc_pending < lowest_pending) {
	170	lowest_pending = mc->mc_pending;
	171	lowest_nr = 1;
	172	} else if (mc->mc_pending == lowest_pending) {
	173	lowest_nr++;
	174	}
	175	}
	176
	177	d = gethrtime() / (NSEC_PER_USEC * zfs_vdev_mirror_switch_us);
	178	d = (d % lowest_nr) + 1;
	179
	180	for (c = 0; c < mm->mm_children; c++) {
	181	mc = &mm->mm_child[c];
	182
	183	if (mm->mm_child[c].mc_pending == lowest_pending) {
	184	if (--d == 0) {
	185	mm->mm_preferred = c;
	186	break;
	187	}
	188	}
34dc7c2f BB	189	}
	190	}
	191
	192	zio->io_vsd = mm;
428870ff	193	zio->io_vsd_ops = &vdev_mirror_vsd_ops;
34dc7c2f BB	194	return (mm);
	195	}
	196
34dc7c2f	197	static int
1bd201e7 CS	198	vdev_mirror_open(vdev_t vd, uint64_t asize, uint64_t *max_asize,
1bd201e7 CS	199	uint64_t *ashift)
34dc7c2f	200	{
34dc7c2f	201	int numerrors = 0;
45d1cae3	202	int lasterror = 0;
d6320ddb	203	int c;
34dc7c2f BB	204
	205	if (vd->vdev_children == 0) {
	206	vd->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL;
a08ee875	207	return (SET_ERROR(EINVAL));
34dc7c2f BB	208	}
34dc7c2f BB	209
45d1cae3	210	vdev_open_children(vd);
34dc7c2f	211
d6320ddb	212	for (c = 0; c < vd->vdev_children; c++) {
45d1cae3 BB	213	vdev_t *cvd = vd->vdev_child[c];
	214
	215	if (cvd->vdev_open_error) {
	216	lasterror = cvd->vdev_open_error;
34dc7c2f BB	217	numerrors++;
	218	continue;
	219	}
	220
	221	asize = MIN(asize - 1, cvd->vdev_asize - 1) + 1;
1bd201e7	222	max_asize = MIN(max_asize - 1, cvd->vdev_max_asize - 1) + 1;
34dc7c2f BB	223	ashift = MAX(ashift, cvd->vdev_ashift);
	224	}
	225
	226	if (numerrors == vd->vdev_children) {
	227	vd->vdev_stat.vs_aux = VDEV_AUX_NO_REPLICAS;
	228	return (lasterror);
	229	}
	230
	231	return (0);
	232	}
	233
	234	static void
	235	vdev_mirror_close(vdev_t *vd)
	236	{
d6320ddb BB	237	int c;
	238
	239	for (c = 0; c < vd->vdev_children; c++)
34dc7c2f BB	240	vdev_close(vd->vdev_child[c]);
	241	}
	242
	243	static void
	244	vdev_mirror_child_done(zio_t *zio)
	245	{
	246	mirror_child_t *mc = zio->io_private;
	247
	248	mc->mc_error = zio->io_error;
	249	mc->mc_tried = 1;
	250	mc->mc_skipped = 0;
	251	}
	252
	253	static void
	254	vdev_mirror_scrub_done(zio_t *zio)
	255	{
	256	mirror_child_t *mc = zio->io_private;
	257
	258	if (zio->io_error == 0) {
d164b209 BB	259	zio_t *pio;
	260
	261	mutex_enter(&zio->io_lock);
	262	while ((pio = zio_walk_parents(zio)) != NULL) {
	263	mutex_enter(&pio->io_lock);
	264	ASSERT3U(zio->io_size, >=, pio->io_size);
	265	bcopy(zio->io_data, pio->io_data, pio->io_size);
	266	mutex_exit(&pio->io_lock);
	267	}
	268	mutex_exit(&zio->io_lock);
34dc7c2f BB	269	}
	270
	271	zio_buf_free(zio->io_data, zio->io_size);
	272
	273	mc->mc_error = zio->io_error;
	274	mc->mc_tried = 1;
	275	mc->mc_skipped = 0;
	276	}
	277
34dc7c2f BB	278	/*
	279	* Try to find a child whose DTL doesn't contain the block we want to read.
	280	* If we can't, try the read on any vdev we haven't already tried.
	281	*/
	282	static int
	283	vdev_mirror_child_select(zio_t *zio)
	284	{
	285	mirror_map_t *mm = zio->io_vsd;
	286	mirror_child_t *mc;
	287	uint64_t txg = zio->io_txg;
	288	int i, c;
	289
428870ff	290	ASSERT(zio->io_bp == NULL \|\| BP_PHYSICAL_BIRTH(zio->io_bp) == txg);
34dc7c2f BB	291
	292	/*
	293	* Try to find a child whose DTL doesn't contain the block to read.
	294	* If a child is known to be completely inaccessible (indicated by
	295	* vdev_readable() returning B_FALSE), don't even try.
	296	*/
	297	for (i = 0, c = mm->mm_preferred; i < mm->mm_children; i++, c++) {
	298	if (c >= mm->mm_children)
	299	c = 0;
	300	mc = &mm->mm_child[c];
	301	if (mc->mc_tried \|\| mc->mc_skipped)
	302	continue;
ea04106b	303	if (mc->mc_vd == NULL \|\| !vdev_readable(mc->mc_vd)) {
a08ee875	304	mc->mc_error = SET_ERROR(ENXIO);
34dc7c2f BB	305	mc->mc_tried = 1; /* don't even try */
	306	mc->mc_skipped = 1;
	307	continue;
	308	}
fb5f0bc8	309	if (!vdev_dtl_contains(mc->mc_vd, DTL_MISSING, txg, 1))
34dc7c2f	310	return (c);
a08ee875	311	mc->mc_error = SET_ERROR(ESTALE);
34dc7c2f	312	mc->mc_skipped = 1;
b128c09f	313	mc->mc_speculative = 1;
34dc7c2f BB	314	}
	315
	316	/*
	317	* Every device is either missing or has this txg in its DTL.
	318	* Look for any child we haven't already tried before giving up.
	319	*/
	320	for (c = 0; c < mm->mm_children; c++)
	321	if (!mm->mm_child[c].mc_tried)
	322	return (c);
	323
	324	/*
	325	* Every child failed. There's no place left to look.
	326	*/
	327	return (-1);
	328	}
	329
	330	static int
	331	vdev_mirror_io_start(zio_t *zio)
	332	{
	333	mirror_map_t *mm;
	334	mirror_child_t *mc;
	335	int c, children;
	336
	337	mm = vdev_mirror_map_alloc(zio);
	338
	339	if (zio->io_type == ZIO_TYPE_READ) {
	340	if ((zio->io_flags & ZIO_FLAG_SCRUB) && !mm->mm_replacing) {
	341	/*
	342	* For scrubbing reads we need to allocate a read
	343	* buffer for each child and issue reads to all
	344	* children. If any child succeeds, it will copy its
	345	* data into zio->io_data in vdev_mirror_scrub_done.
	346	*/
	347	for (c = 0; c < mm->mm_children; c++) {
	348	mc = &mm->mm_child[c];
	349	zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
	350	mc->mc_vd, mc->mc_offset,
	351	zio_buf_alloc(zio->io_size), zio->io_size,
b128c09f	352	zio->io_type, zio->io_priority, 0,
34dc7c2f BB	353	vdev_mirror_scrub_done, mc));
34dc7c2f BB	354	}
b128c09f	355	return (ZIO_PIPELINE_CONTINUE);
34dc7c2f BB	356	}
	357	/*
	358	* For normal reads just pick one child.
	359	*/
	360	c = vdev_mirror_child_select(zio);
	361	children = (c >= 0);
	362	} else {
	363	ASSERT(zio->io_type == ZIO_TYPE_WRITE);
	364
	365	/*
fb5f0bc8	366	* Writes go to all children.
34dc7c2f	367	*/
fb5f0bc8 BB	368	c = 0;
fb5f0bc8 BB	369	children = mm->mm_children;
34dc7c2f BB	370	}
	371
	372	while (children--) {
	373	mc = &mm->mm_child[c];
	374	zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
b128c09f BB	375	mc->mc_vd, mc->mc_offset, zio->io_data, zio->io_size,
	376	zio->io_type, zio->io_priority, 0,
	377	vdev_mirror_child_done, mc));
34dc7c2f BB	378	c++;
	379	}
	380
b128c09f	381	return (ZIO_PIPELINE_CONTINUE);
34dc7c2f BB	382	}
	383
	384	static int
b128c09f BB	385	vdev_mirror_worst_error(mirror_map_t *mm)
b128c09f BB	386	{
d6320ddb	387	int c, error[2] = { 0, 0 };
b128c09f	388
d6320ddb	389	for (c = 0; c < mm->mm_children; c++) {
b128c09f BB	390	mirror_child_t *mc = &mm->mm_child[c];
	391	int s = mc->mc_speculative;
	392	error[s] = zio_worst_error(error[s], mc->mc_error);
	393	}
	394
	395	return (error[0] ? error[0] : error[1]);
	396	}
	397
	398	static void
34dc7c2f BB	399	vdev_mirror_io_done(zio_t *zio)
	400	{
	401	mirror_map_t *mm = zio->io_vsd;
	402	mirror_child_t *mc;
	403	int c;
	404	int good_copies = 0;
	405	int unexpected_errors = 0;
	406
34dc7c2f BB	407	for (c = 0; c < mm->mm_children; c++) {
	408	mc = &mm->mm_child[c];
	409
34dc7c2f	410	if (mc->mc_error) {
34dc7c2f BB	411	if (!mc->mc_skipped)
34dc7c2f BB	412	unexpected_errors++;
b128c09f BB	413	} else if (mc->mc_tried) {
b128c09f BB	414	good_copies++;
34dc7c2f BB	415	}
	416	}
	417
	418	if (zio->io_type == ZIO_TYPE_WRITE) {
	419	/*
	420	* XXX -- for now, treat partial writes as success.
b128c09f BB	421	*
	422	* Now that we support write reallocation, it would be better
	423	* to treat partial failure as real failure unless there are
	424	* no non-degraded top-level vdevs left, and not update DTLs
	425	* if we intend to reallocate.
34dc7c2f BB	426	*/
34dc7c2f BB	427	/* XXPOLICY */
b128c09f BB	428	if (good_copies != mm->mm_children) {
	429	/*
	430	* Always require at least one good copy.
	431	*
	432	* For ditto blocks (io_vd == NULL), require
	433	* all copies to be good.
	434	*
	435	* XXX -- for replacing vdevs, there's no great answer.
	436	* If the old device is really dead, we may not even
	437	* be able to access it -- so we only want to
	438	* require good writes to the new device. But if
	439	* the new device turns out to be flaky, we want
	440	* to be able to detach it -- which requires all
	441	* writes to the old device to have succeeded.
	442	*/
	443	if (good_copies == 0 \|\| zio->io_vd == NULL)
	444	zio->io_error = vdev_mirror_worst_error(mm);
	445	}
	446	return;
34dc7c2f BB	447	}
	448
	449	ASSERT(zio->io_type == ZIO_TYPE_READ);
	450
	451	/*
	452	* If we don't have a good copy yet, keep trying other children.
	453	*/
	454	/* XXPOLICY */
	455	if (good_copies == 0 && (c = vdev_mirror_child_select(zio)) != -1) {
	456	ASSERT(c >= 0 && c < mm->mm_children);
	457	mc = &mm->mm_child[c];
34dc7c2f BB	458	zio_vdev_io_redone(zio);
	459	zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
	460	mc->mc_vd, mc->mc_offset, zio->io_data, zio->io_size,
b128c09f	461	ZIO_TYPE_READ, zio->io_priority, 0,
34dc7c2f	462	vdev_mirror_child_done, mc));
b128c09f	463	return;
34dc7c2f BB	464	}
	465
	466	/* XXPOLICY */
b128c09f BB	467	if (good_copies == 0) {
b128c09f BB	468	zio->io_error = vdev_mirror_worst_error(mm);
34dc7c2f	469	ASSERT(zio->io_error != 0);
b128c09f	470	}
34dc7c2f	471
fb5f0bc8	472	if (good_copies && spa_writeable(zio->io_spa) &&
34dc7c2f BB	473	(unexpected_errors \|\|
	474	(zio->io_flags & ZIO_FLAG_RESILVER) \|\|
	475	((zio->io_flags & ZIO_FLAG_SCRUB) && mm->mm_replacing))) {
34dc7c2f BB	476	/*
34dc7c2f BB	477	* Use the good data we have in hand to repair damaged children.
34dc7c2f	478	*/
34dc7c2f BB	479	for (c = 0; c < mm->mm_children; c++) {
	480	/*
	481	* Don't rewrite known good children.
	482	* Not only is it unnecessary, it could
	483	* actually be harmful: if the system lost
	484	* power while rewriting the only good copy,
	485	* there would be no good copies left!
	486	*/
	487	mc = &mm->mm_child[c];
	488
	489	if (mc->mc_error == 0) {
	490	if (mc->mc_tried)
	491	continue;
	492	if (!(zio->io_flags & ZIO_FLAG_SCRUB) &&
fb5f0bc8	493	!vdev_dtl_contains(mc->mc_vd, DTL_PARTIAL,
34dc7c2f BB	494	zio->io_txg, 1))
34dc7c2f BB	495	continue;
a08ee875	496	mc->mc_error = SET_ERROR(ESTALE);
34dc7c2f BB	497	}
34dc7c2f BB	498
b128c09f BB	499	zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
	500	mc->mc_vd, mc->mc_offset,
	501	zio->io_data, zio->io_size,
a08ee875	502	ZIO_TYPE_WRITE, ZIO_PRIORITY_ASYNC_WRITE,
fb5f0bc8 BB	503	ZIO_FLAG_IO_REPAIR \| (unexpected_errors ?
fb5f0bc8 BB	504	ZIO_FLAG_SELF_HEAL : 0), NULL, NULL));
34dc7c2f	505	}
34dc7c2f	506	}
34dc7c2f BB	507	}
	508
	509	static void
	510	vdev_mirror_state_change(vdev_t *vd, int faulted, int degraded)
	511	{
	512	if (faulted == vd->vdev_children)
	513	vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
	514	VDEV_AUX_NO_REPLICAS);
	515	else if (degraded + faulted != 0)
	516	vdev_set_state(vd, B_FALSE, VDEV_STATE_DEGRADED, VDEV_AUX_NONE);
	517	else
	518	vdev_set_state(vd, B_FALSE, VDEV_STATE_HEALTHY, VDEV_AUX_NONE);
	519	}
	520
	521	vdev_ops_t vdev_mirror_ops = {
	522	vdev_mirror_open,
	523	vdev_mirror_close,
34dc7c2f BB	524	vdev_default_asize,
	525	vdev_mirror_io_start,
	526	vdev_mirror_io_done,
	527	vdev_mirror_state_change,
428870ff BB	528	NULL,
428870ff BB	529	NULL,
34dc7c2f BB	530	VDEV_TYPE_MIRROR, /* name of this vdev type */
	531	B_FALSE /* not a leaf vdev */
	532	};
	533
	534	vdev_ops_t vdev_replacing_ops = {
	535	vdev_mirror_open,
	536	vdev_mirror_close,
34dc7c2f BB	537	vdev_default_asize,
	538	vdev_mirror_io_start,
	539	vdev_mirror_io_done,
	540	vdev_mirror_state_change,
428870ff BB	541	NULL,
428870ff BB	542	NULL,
34dc7c2f BB	543	VDEV_TYPE_REPLACING, /* name of this vdev type */
	544	B_FALSE /* not a leaf vdev */
	545	};
	546
	547	vdev_ops_t vdev_spare_ops = {
	548	vdev_mirror_open,
	549	vdev_mirror_close,
34dc7c2f BB	550	vdev_default_asize,
	551	vdev_mirror_io_start,
	552	vdev_mirror_io_done,
	553	vdev_mirror_state_change,
428870ff BB	554	NULL,
428870ff BB	555	NULL,
34dc7c2f BB	556	VDEV_TYPE_SPARE, /* name of this vdev type */
	557	B_FALSE /* not a leaf vdev */
	558	};
c06d4368 AX	559
	560	#if defined(_KERNEL) && defined(HAVE_SPL)
	561	module_param(zfs_vdev_mirror_switch_us, int, 0644);
	562	MODULE_PARM_DESC(zfs_vdev_mirror_switch_us, "Switch mirrors every N usecs");
	563	#endif