[mirror_zfs.git] / module / zfs / vdev_mirror.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright (c) 2012 by Delphix. All rights reserved.
 */

#include <sys/zfs_context.h>
#include <sys/spa.h>
#include <sys/vdev_impl.h>
#include <sys/zio.h>
#include <sys/fs/zfs.h>

/*
 * Virtual device vector for mirroring.
 */

typedef struct mirror_child {
	vdev_t		*mc_vd;
	uint64_t	mc_offset;
	int		mc_error;
	int		mc_pending;
	uint8_t		mc_tried;
	uint8_t		mc_skipped;
	uint8_t		mc_speculative;
} mirror_child_t;

typedef struct mirror_map {
	int		mm_children;
	int		mm_replacing;
	int		mm_preferred;
	int		mm_root;
	mirror_child_t	mm_child[1];
} mirror_map_t;

/*
 * When the children are equally busy queue incoming requests to a single
 * child for N microseconds.  This is done to maximize the likelihood that
 * the Linux elevator will be able to merge requests while it is plugged.
 * Otherwise, requests are queued to the least busy device.
 *
 * For rotational disks the Linux elevator will plug for 10ms which is
 * why zfs_vdev_mirror_switch_us is set to 10ms by default.  For non-
 * rotational disks the elevator will not plug, but 10ms is still a small
 * enough value that the requests will get spread over all the children.
 *
 * For fast SSDs it may make sense to decrease zfs_vdev_mirror_switch_us
 * significantly to bound the worst case latencies.  It would probably be
 * ideal to calculate a decaying average of the last observed latencies and
 * use that to dynamically adjust the zfs_vdev_mirror_switch_us time.
 */
int zfs_vdev_mirror_switch_us = 10000;

static void
vdev_mirror_map_free(zio_t *zio)
{
	mirror_map_t *mm = zio->io_vsd;

	kmem_free(mm, offsetof(mirror_map_t, mm_child[mm->mm_children]));
}

static const zio_vsd_ops_t vdev_mirror_vsd_ops = {
	vdev_mirror_map_free,
	zio_vsd_default_cksum_report
};

static int
vdev_mirror_pending(vdev_t *vd)
{
	return avl_numnodes(&vd->vdev_queue.vq_pending_tree);
}

static mirror_map_t *
vdev_mirror_map_alloc(zio_t *zio)
{
	mirror_map_t *mm = NULL;
	mirror_child_t *mc;
	vdev_t *vd = zio->io_vd;
	int c, d;

	if (vd == NULL) {
		dva_t *dva = zio->io_bp->blk_dva;
		spa_t *spa = zio->io_spa;

		c = BP_GET_NDVAS(zio->io_bp);

		mm = kmem_zalloc(offsetof(mirror_map_t, mm_child[c]), KM_PUSHPAGE);
		mm->mm_children = c;
		mm->mm_replacing = B_FALSE;
		mm->mm_preferred = spa_get_random(c);
		mm->mm_root = B_TRUE;

		/*
		 * Check the other, lower-index DVAs to see if they're on
		 * the same vdev as the child we picked.  If they are, use
		 * them since they are likely to have been allocated from
		 * the primary metaslab in use at the time, and hence are
		 * more likely to have locality with single-copy data.
		 */
		for (c = mm->mm_preferred, d = c - 1; d >= 0; d--) {
			if (DVA_GET_VDEV(&dva[d]) == DVA_GET_VDEV(&dva[c]))
				mm->mm_preferred = d;
		}

		for (c = 0; c < mm->mm_children; c++) {
			mc = &mm->mm_child[c];

			mc->mc_vd = vdev_lookup_top(spa, DVA_GET_VDEV(&dva[c]));
			mc->mc_offset = DVA_GET_OFFSET(&dva[c]);
		}
	} else {
		int lowest_pending = INT_MAX;
		int lowest_nr = 1;

		c = vd->vdev_children;

		mm = kmem_zalloc(offsetof(mirror_map_t, mm_child[c]), KM_PUSHPAGE);
		mm->mm_children = c;
		mm->mm_replacing = (vd->vdev_ops == &vdev_replacing_ops ||
		    vd->vdev_ops == &vdev_spare_ops);
		mm->mm_preferred = 0;
		mm->mm_root = B_FALSE;

		for (c = 0; c < mm->mm_children; c++) {
			mc = &mm->mm_child[c];
			mc->mc_vd = vd->vdev_child[c];
			mc->mc_offset = zio->io_offset;

			if (mm->mm_replacing)
				continue;

			if (!vdev_readable(mc->mc_vd)) {
				mc->mc_error = ENXIO;
				mc->mc_tried = 1;
				mc->mc_skipped = 1;
				mc->mc_pending = INT_MAX;
				continue;
			}

			mc->mc_pending = vdev_mirror_pending(mc->mc_vd);
			if (mc->mc_pending < lowest_pending) {
				lowest_pending = mc->mc_pending;
				lowest_nr = 1;
			} else if (mc->mc_pending == lowest_pending) {
				lowest_nr++;
			}
		}

		d = gethrtime() / (NSEC_PER_USEC * zfs_vdev_mirror_switch_us);
		d = (d % lowest_nr) + 1;

		for (c = 0; c < mm->mm_children; c++) {
			mc = &mm->mm_child[c];

			if (mm->mm_child[c].mc_pending == lowest_pending) {
				if (--d == 0) {
					mm->mm_preferred = c;
					break;
				}
			}
		}
	}

	zio->io_vsd = mm;
	zio->io_vsd_ops = &vdev_mirror_vsd_ops;
	return (mm);
}

static int
vdev_mirror_open(vdev_t *vd, uint64_t *asize, uint64_t *max_asize,
    uint64_t *ashift)
{
	int numerrors = 0;
	int lasterror = 0;
	int c;

	if (vd->vdev_children == 0) {
		vd->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL;
		return (EINVAL);
	}

	vdev_open_children(vd);

	for (c = 0; c < vd->vdev_children; c++) {
		vdev_t *cvd = vd->vdev_child[c];

		if (cvd->vdev_open_error) {
			lasterror = cvd->vdev_open_error;
			numerrors++;
			continue;
		}

		*asize = MIN(*asize - 1, cvd->vdev_asize - 1) + 1;
		*max_asize = MIN(*max_asize - 1, cvd->vdev_max_asize - 1) + 1;
		*ashift = MAX(*ashift, cvd->vdev_ashift);
	}

	if (numerrors == vd->vdev_children) {
		vd->vdev_stat.vs_aux = VDEV_AUX_NO_REPLICAS;
		return (lasterror);
	}

	return (0);
}

static void
vdev_mirror_close(vdev_t *vd)
{
	int c;

	for (c = 0; c < vd->vdev_children; c++)
		vdev_close(vd->vdev_child[c]);
}

static void
vdev_mirror_child_done(zio_t *zio)
{
	mirror_child_t *mc = zio->io_private;

	mc->mc_error = zio->io_error;
	mc->mc_tried = 1;
	mc->mc_skipped = 0;
}

static void
vdev_mirror_scrub_done(zio_t *zio)
{
	mirror_child_t *mc = zio->io_private;

	if (zio->io_error == 0) {
		zio_t *pio;

		mutex_enter(&zio->io_lock);
		while ((pio = zio_walk_parents(zio)) != NULL) {
			mutex_enter(&pio->io_lock);
			ASSERT3U(zio->io_size, >=, pio->io_size);
			bcopy(zio->io_data, pio->io_data, pio->io_size);
			mutex_exit(&pio->io_lock);
		}
		mutex_exit(&zio->io_lock);
	}

	zio_buf_free(zio->io_data, zio->io_size);

	mc->mc_error = zio->io_error;
	mc->mc_tried = 1;
	mc->mc_skipped = 0;
}

/*
 * Try to find a child whose DTL doesn't contain the block we want to read.
 * If we can't, try the read on any vdev we haven't already tried.
 */
static int
vdev_mirror_child_select(zio_t *zio)
{
	mirror_map_t *mm = zio->io_vsd;
	mirror_child_t *mc;
	uint64_t txg = zio->io_txg;
	int i, c;

	ASSERT(zio->io_bp == NULL || BP_PHYSICAL_BIRTH(zio->io_bp) == txg);

	/*
	 * Try to find a child whose DTL doesn't contain the block to read.
	 * If a child is known to be completely inaccessible (indicated by
	 * vdev_readable() returning B_FALSE), don't even try.
	 */
	for (i = 0, c = mm->mm_preferred; i < mm->mm_children; i++, c++) {
		if (c >= mm->mm_children)
			c = 0;
		mc = &mm->mm_child[c];
		if (mc->mc_tried || mc->mc_skipped)
			continue;
		if (!vdev_readable(mc->mc_vd)) {
			mc->mc_error = ENXIO;
			mc->mc_tried = 1;	/* don't even try */
			mc->mc_skipped = 1;
			continue;
		}
		if (!vdev_dtl_contains(mc->mc_vd, DTL_MISSING, txg, 1))
			return (c);
		mc->mc_error = ESTALE;
		mc->mc_skipped = 1;
		mc->mc_speculative = 1;
	}

	/*
	 * Every device is either missing or has this txg in its DTL.
	 * Look for any child we haven't already tried before giving up.
	 */
	for (c = 0; c < mm->mm_children; c++)
		if (!mm->mm_child[c].mc_tried)
			return (c);

	/*
	 * Every child failed.  There's no place left to look.
	 */
	return (-1);
}

static int
vdev_mirror_io_start(zio_t *zio)
{
	mirror_map_t *mm;
	mirror_child_t *mc;
	int c, children;

	mm = vdev_mirror_map_alloc(zio);

	if (zio->io_type == ZIO_TYPE_READ) {
		if ((zio->io_flags & ZIO_FLAG_SCRUB) && !mm->mm_replacing) {
			/*
			 * For scrubbing reads we need to allocate a read
			 * buffer for each child and issue reads to all
			 * children.  If any child succeeds, it will copy its
			 * data into zio->io_data in vdev_mirror_scrub_done.
			 */
			for (c = 0; c < mm->mm_children; c++) {
				mc = &mm->mm_child[c];
				zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
				    mc->mc_vd, mc->mc_offset,
				    zio_buf_alloc(zio->io_size), zio->io_size,
				    zio->io_type, zio->io_priority, 0,
				    vdev_mirror_scrub_done, mc));
			}
			return (ZIO_PIPELINE_CONTINUE);
		}
		/*
		 * For normal reads just pick one child.
		 */
		c = vdev_mirror_child_select(zio);
		children = (c >= 0);
	} else {
		ASSERT(zio->io_type == ZIO_TYPE_WRITE);

		/*
		 * Writes go to all children.
		 */
		c = 0;
		children = mm->mm_children;
	}

	while (children--) {
		mc = &mm->mm_child[c];
		zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
		    mc->mc_vd, mc->mc_offset, zio->io_data, zio->io_size,
		    zio->io_type, zio->io_priority, 0,
		    vdev_mirror_child_done, mc));
		c++;
	}

	return (ZIO_PIPELINE_CONTINUE);
}

static int
vdev_mirror_worst_error(mirror_map_t *mm)
{
	int c, error[2] = { 0, 0 };

	for (c = 0; c < mm->mm_children; c++) {
		mirror_child_t *mc = &mm->mm_child[c];
		int s = mc->mc_speculative;
		error[s] = zio_worst_error(error[s], mc->mc_error);
	}

	return (error[0] ? error[0] : error[1]);
}

static void
vdev_mirror_io_done(zio_t *zio)
{
	mirror_map_t *mm = zio->io_vsd;
	mirror_child_t *mc;
	int c;
	int good_copies = 0;
	int unexpected_errors = 0;

	for (c = 0; c < mm->mm_children; c++) {
		mc = &mm->mm_child[c];

		if (mc->mc_error) {
			if (!mc->mc_skipped)
				unexpected_errors++;
		} else if (mc->mc_tried) {
			good_copies++;
		}
	}

	if (zio->io_type == ZIO_TYPE_WRITE) {
		/*
		 * XXX -- for now, treat partial writes as success.
		 *
		 * Now that we support write reallocation, it would be better
		 * to treat partial failure as real failure unless there are
		 * no non-degraded top-level vdevs left, and not update DTLs
		 * if we intend to reallocate.
		 */
		/* XXPOLICY */
		if (good_copies != mm->mm_children) {
			/*
			 * Always require at least one good copy.
			 *
			 * For ditto blocks (io_vd == NULL), require
			 * all copies to be good.
			 *
			 * XXX -- for replacing vdevs, there's no great answer.
			 * If the old device is really dead, we may not even
			 * be able to access it -- so we only want to
			 * require good writes to the new device.  But if
			 * the new device turns out to be flaky, we want
			 * to be able to detach it -- which requires all
			 * writes to the old device to have succeeded.
			 */
			if (good_copies == 0 || zio->io_vd == NULL)
				zio->io_error = vdev_mirror_worst_error(mm);
		}
		return;
	}

	ASSERT(zio->io_type == ZIO_TYPE_READ);

	/*
	 * If we don't have a good copy yet, keep trying other children.
	 */
	/* XXPOLICY */
	if (good_copies == 0 && (c = vdev_mirror_child_select(zio)) != -1) {
		ASSERT(c >= 0 && c < mm->mm_children);
		mc = &mm->mm_child[c];
		zio_vdev_io_redone(zio);
		zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
		    mc->mc_vd, mc->mc_offset, zio->io_data, zio->io_size,
		    ZIO_TYPE_READ, zio->io_priority, 0,
		    vdev_mirror_child_done, mc));
		return;
	}

	/* XXPOLICY */
	if (good_copies == 0) {
		zio->io_error = vdev_mirror_worst_error(mm);
		ASSERT(zio->io_error != 0);
	}

	if (good_copies && spa_writeable(zio->io_spa) &&
	    (unexpected_errors ||
	    (zio->io_flags & ZIO_FLAG_RESILVER) ||
	    ((zio->io_flags & ZIO_FLAG_SCRUB) && mm->mm_replacing))) {
		/*
		 * Use the good data we have in hand to repair damaged children.
		 */
		for (c = 0; c < mm->mm_children; c++) {
			/*
			 * Don't rewrite known good children.
			 * Not only is it unnecessary, it could
			 * actually be harmful: if the system lost
			 * power while rewriting the only good copy,
			 * there would be no good copies left!
			 */
			mc = &mm->mm_child[c];

			if (mc->mc_error == 0) {
				if (mc->mc_tried)
					continue;
				if (!(zio->io_flags & ZIO_FLAG_SCRUB) &&
				    !vdev_dtl_contains(mc->mc_vd, DTL_PARTIAL,
				    zio->io_txg, 1))
					continue;
				mc->mc_error = ESTALE;
			}

			zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
			    mc->mc_vd, mc->mc_offset,
			    zio->io_data, zio->io_size,
			    ZIO_TYPE_WRITE, zio->io_priority,
			    ZIO_FLAG_IO_REPAIR | (unexpected_errors ?
			    ZIO_FLAG_SELF_HEAL : 0), NULL, NULL));
		}
	}
}

static void
vdev_mirror_state_change(vdev_t *vd, int faulted, int degraded)
{
	if (faulted == vd->vdev_children)
		vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
		    VDEV_AUX_NO_REPLICAS);
	else if (degraded + faulted != 0)
		vdev_set_state(vd, B_FALSE, VDEV_STATE_DEGRADED, VDEV_AUX_NONE);
	else
		vdev_set_state(vd, B_FALSE, VDEV_STATE_HEALTHY, VDEV_AUX_NONE);
}

vdev_ops_t vdev_mirror_ops = {
	vdev_mirror_open,
	vdev_mirror_close,
	vdev_default_asize,
	vdev_mirror_io_start,
	vdev_mirror_io_done,
	vdev_mirror_state_change,
	NULL,
	NULL,
	VDEV_TYPE_MIRROR,	/* name of this vdev type */
	B_FALSE			/* not a leaf vdev */
};

vdev_ops_t vdev_replacing_ops = {
	vdev_mirror_open,
	vdev_mirror_close,
	vdev_default_asize,
	vdev_mirror_io_start,
	vdev_mirror_io_done,
	vdev_mirror_state_change,
	NULL,
	NULL,
	VDEV_TYPE_REPLACING,	/* name of this vdev type */
	B_FALSE			/* not a leaf vdev */
};

vdev_ops_t vdev_spare_ops = {
	vdev_mirror_open,
	vdev_mirror_close,
	vdev_default_asize,
	vdev_mirror_io_start,
	vdev_mirror_io_done,
	vdev_mirror_state_change,
	NULL,
	NULL,
	VDEV_TYPE_SPARE,	/* name of this vdev type */
	B_FALSE			/* not a leaf vdev */
};

#if defined(_KERNEL) && defined(HAVE_SPL)
module_param(zfs_vdev_mirror_switch_us, int, 0644);
MODULE_PARM_DESC(zfs_vdev_mirror_switch_us, "Switch mirrors every N usecs");
#endif
Commit	Line	Data
34dc7c2f BB	1	/*
	2	* CDDL HEADER START
	3	*
	4	* The contents of this file are subject to the terms of the
	5	* Common Development and Distribution License (the "License").
	6	* You may not use this file except in compliance with the License.
	7	*
	8	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	9	* or http://www.opensolaris.org/os/licensing.
	10	* See the License for the specific language governing permissions
	11	* and limitations under the License.
	12	*
	13	* When distributing Covered Code, include this CDDL HEADER in each
	14	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	15	* If applicable, add the following below this CDDL HEADER, with the
	16	* fields enclosed by brackets "[]" replaced with your own identifying
	17	* information: Portions Copyright [yyyy] [name of copyright owner]
	18	*
	19	* CDDL HEADER END
	20	*/
	21	/*
428870ff	22	* Copyright 2010 Sun Microsystems, Inc. All rights reserved.
34dc7c2f BB	23	* Use is subject to license terms.
	24	*/
	25
1bd201e7 CS	26	/*
	27	* Copyright (c) 2012 by Delphix. All rights reserved.
	28	*/
	29
34dc7c2f BB	30	#include <sys/zfs_context.h>
	31	#include <sys/spa.h>
	32	#include <sys/vdev_impl.h>
	33	#include <sys/zio.h>
	34	#include <sys/fs/zfs.h>
	35
	36	/*
	37	* Virtual device vector for mirroring.
	38	*/
	39
	40	typedef struct mirror_child {
	41	vdev_t *mc_vd;
	42	uint64_t mc_offset;
	43	int mc_error;
556011db	44	int mc_pending;
b128c09f BB	45	uint8_t mc_tried;
	46	uint8_t mc_skipped;
	47	uint8_t mc_speculative;
34dc7c2f BB	48	} mirror_child_t;
	49
	50	typedef struct mirror_map {
	51	int mm_children;
	52	int mm_replacing;
	53	int mm_preferred;
	54	int mm_root;
	55	mirror_child_t mm_child[1];
	56	} mirror_map_t;
	57
556011db BB	58	/*
	59	* When the children are equally busy queue incoming requests to a single
	60	* child for N microseconds. This is done to maximize the likelihood that
	61	* the Linux elevator will be able to merge requests while it is plugged.
	62	* Otherwise, requests are queued to the least busy device.
	63	*
	64	* For rotational disks the Linux elevator will plug for 10ms which is
	65	* why zfs_vdev_mirror_switch_us is set to 10ms by default. For non-
	66	* rotational disks the elevator will not plug, but 10ms is still a small
	67	* enough value that the requests will get spread over all the children.
	68	*
	69	* For fast SSDs it may make sense to decrease zfs_vdev_mirror_switch_us
	70	* significantly to bound the worst case latencies. It would probably be
	71	* ideal to calculate a decaying average of the last observed latencies and
	72	* use that to dynamically adjust the zfs_vdev_mirror_switch_us time.
	73	*/
	74	int zfs_vdev_mirror_switch_us = 10000;
34dc7c2f	75
b128c09f BB	76	static void
	77	vdev_mirror_map_free(zio_t *zio)
	78	{
	79	mirror_map_t *mm = zio->io_vsd;
	80
	81	kmem_free(mm, offsetof(mirror_map_t, mm_child[mm->mm_children]));
	82	}
	83
428870ff BB	84	static const zio_vsd_ops_t vdev_mirror_vsd_ops = {
	85	vdev_mirror_map_free,
	86	zio_vsd_default_cksum_report
	87	};
	88
556011db BB	89	static int
	90	vdev_mirror_pending(vdev_t *vd)
	91	{
f9f3f1ef	92	return avl_numnodes(&vd->vdev_queue.vq_pending_tree);
556011db BB	93	}
556011db BB	94
34dc7c2f BB	95	static mirror_map_t *
	96	vdev_mirror_map_alloc(zio_t *zio)
	97	{
	98	mirror_map_t *mm = NULL;
	99	mirror_child_t *mc;
	100	vdev_t *vd = zio->io_vd;
	101	int c, d;
	102
	103	if (vd == NULL) {
	104	dva_t *dva = zio->io_bp->blk_dva;
	105	spa_t *spa = zio->io_spa;
	106
	107	c = BP_GET_NDVAS(zio->io_bp);
	108
b8d06fca	109	mm = kmem_zalloc(offsetof(mirror_map_t, mm_child[c]), KM_PUSHPAGE);
34dc7c2f BB	110	mm->mm_children = c;
	111	mm->mm_replacing = B_FALSE;
	112	mm->mm_preferred = spa_get_random(c);
	113	mm->mm_root = B_TRUE;
	114
	115	/*
	116	* Check the other, lower-index DVAs to see if they're on
	117	* the same vdev as the child we picked. If they are, use
	118	* them since they are likely to have been allocated from
	119	* the primary metaslab in use at the time, and hence are
	120	* more likely to have locality with single-copy data.
	121	*/
	122	for (c = mm->mm_preferred, d = c - 1; d >= 0; d--) {
	123	if (DVA_GET_VDEV(&dva[d]) == DVA_GET_VDEV(&dva[c]))
	124	mm->mm_preferred = d;
	125	}
	126
	127	for (c = 0; c < mm->mm_children; c++) {
	128	mc = &mm->mm_child[c];
	129
	130	mc->mc_vd = vdev_lookup_top(spa, DVA_GET_VDEV(&dva[c]));
	131	mc->mc_offset = DVA_GET_OFFSET(&dva[c]);
	132	}
	133	} else {
556011db BB	134	int lowest_pending = INT_MAX;
	135	int lowest_nr = 1;
	136
34dc7c2f BB	137	c = vd->vdev_children;
34dc7c2f BB	138
b8d06fca	139	mm = kmem_zalloc(offsetof(mirror_map_t, mm_child[c]), KM_PUSHPAGE);
34dc7c2f BB	140	mm->mm_children = c;
	141	mm->mm_replacing = (vd->vdev_ops == &vdev_replacing_ops \|\|
	142	vd->vdev_ops == &vdev_spare_ops);
556011db	143	mm->mm_preferred = 0;
34dc7c2f BB	144	mm->mm_root = B_FALSE;
	145
	146	for (c = 0; c < mm->mm_children; c++) {
	147	mc = &mm->mm_child[c];
	148	mc->mc_vd = vd->vdev_child[c];
	149	mc->mc_offset = zio->io_offset;
556011db BB	150
	151	if (mm->mm_replacing)
	152	continue;
	153
	154	if (!vdev_readable(mc->mc_vd)) {
	155	mc->mc_error = ENXIO;
	156	mc->mc_tried = 1;
	157	mc->mc_skipped = 1;
	158	mc->mc_pending = INT_MAX;
	159	continue;
	160	}
	161
	162	mc->mc_pending = vdev_mirror_pending(mc->mc_vd);
	163	if (mc->mc_pending < lowest_pending) {
	164	lowest_pending = mc->mc_pending;
	165	lowest_nr = 1;
	166	} else if (mc->mc_pending == lowest_pending) {
	167	lowest_nr++;
	168	}
	169	}
	170
	171	d = gethrtime() / (NSEC_PER_USEC * zfs_vdev_mirror_switch_us);
	172	d = (d % lowest_nr) + 1;
	173
	174	for (c = 0; c < mm->mm_children; c++) {
	175	mc = &mm->mm_child[c];
	176
	177	if (mm->mm_child[c].mc_pending == lowest_pending) {
	178	if (--d == 0) {
	179	mm->mm_preferred = c;
	180	break;
	181	}
	182	}
34dc7c2f BB	183	}
	184	}
	185
	186	zio->io_vsd = mm;
428870ff	187	zio->io_vsd_ops = &vdev_mirror_vsd_ops;
34dc7c2f BB	188	return (mm);
	189	}
	190
34dc7c2f	191	static int
1bd201e7 CS	192	vdev_mirror_open(vdev_t vd, uint64_t asize, uint64_t *max_asize,
1bd201e7 CS	193	uint64_t *ashift)
34dc7c2f	194	{
34dc7c2f	195	int numerrors = 0;
45d1cae3	196	int lasterror = 0;
d6320ddb	197	int c;
34dc7c2f BB	198
	199	if (vd->vdev_children == 0) {
	200	vd->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL;
	201	return (EINVAL);
	202	}
	203
45d1cae3	204	vdev_open_children(vd);
34dc7c2f	205
d6320ddb	206	for (c = 0; c < vd->vdev_children; c++) {
45d1cae3 BB	207	vdev_t *cvd = vd->vdev_child[c];
	208
	209	if (cvd->vdev_open_error) {
	210	lasterror = cvd->vdev_open_error;
34dc7c2f BB	211	numerrors++;
	212	continue;
	213	}
	214
	215	asize = MIN(asize - 1, cvd->vdev_asize - 1) + 1;
1bd201e7	216	max_asize = MIN(max_asize - 1, cvd->vdev_max_asize - 1) + 1;
34dc7c2f BB	217	ashift = MAX(ashift, cvd->vdev_ashift);
	218	}
	219
	220	if (numerrors == vd->vdev_children) {
	221	vd->vdev_stat.vs_aux = VDEV_AUX_NO_REPLICAS;
	222	return (lasterror);
	223	}
	224
	225	return (0);
	226	}
	227
	228	static void
	229	vdev_mirror_close(vdev_t *vd)
	230	{
d6320ddb BB	231	int c;
	232
	233	for (c = 0; c < vd->vdev_children; c++)
34dc7c2f BB	234	vdev_close(vd->vdev_child[c]);
	235	}
	236
	237	static void
	238	vdev_mirror_child_done(zio_t *zio)
	239	{
	240	mirror_child_t *mc = zio->io_private;
	241
	242	mc->mc_error = zio->io_error;
	243	mc->mc_tried = 1;
	244	mc->mc_skipped = 0;
	245	}
	246
	247	static void
	248	vdev_mirror_scrub_done(zio_t *zio)
	249	{
	250	mirror_child_t *mc = zio->io_private;
	251
	252	if (zio->io_error == 0) {
d164b209 BB	253	zio_t *pio;
	254
	255	mutex_enter(&zio->io_lock);
	256	while ((pio = zio_walk_parents(zio)) != NULL) {
	257	mutex_enter(&pio->io_lock);
	258	ASSERT3U(zio->io_size, >=, pio->io_size);
	259	bcopy(zio->io_data, pio->io_data, pio->io_size);
	260	mutex_exit(&pio->io_lock);
	261	}
	262	mutex_exit(&zio->io_lock);
34dc7c2f BB	263	}
	264
	265	zio_buf_free(zio->io_data, zio->io_size);
	266
	267	mc->mc_error = zio->io_error;
	268	mc->mc_tried = 1;
	269	mc->mc_skipped = 0;
	270	}
	271
34dc7c2f BB	272	/*
	273	* Try to find a child whose DTL doesn't contain the block we want to read.
	274	* If we can't, try the read on any vdev we haven't already tried.
	275	*/
	276	static int
	277	vdev_mirror_child_select(zio_t *zio)
	278	{
	279	mirror_map_t *mm = zio->io_vsd;
	280	mirror_child_t *mc;
	281	uint64_t txg = zio->io_txg;
	282	int i, c;
	283
428870ff	284	ASSERT(zio->io_bp == NULL \|\| BP_PHYSICAL_BIRTH(zio->io_bp) == txg);
34dc7c2f BB	285
	286	/*
	287	* Try to find a child whose DTL doesn't contain the block to read.
	288	* If a child is known to be completely inaccessible (indicated by
	289	* vdev_readable() returning B_FALSE), don't even try.
	290	*/
	291	for (i = 0, c = mm->mm_preferred; i < mm->mm_children; i++, c++) {
	292	if (c >= mm->mm_children)
	293	c = 0;
	294	mc = &mm->mm_child[c];
	295	if (mc->mc_tried \|\| mc->mc_skipped)
	296	continue;
b128c09f	297	if (!vdev_readable(mc->mc_vd)) {
34dc7c2f BB	298	mc->mc_error = ENXIO;
	299	mc->mc_tried = 1; /* don't even try */
	300	mc->mc_skipped = 1;
	301	continue;
	302	}
fb5f0bc8	303	if (!vdev_dtl_contains(mc->mc_vd, DTL_MISSING, txg, 1))
34dc7c2f BB	304	return (c);
	305	mc->mc_error = ESTALE;
	306	mc->mc_skipped = 1;
b128c09f	307	mc->mc_speculative = 1;
34dc7c2f BB	308	}
	309
	310	/*
	311	* Every device is either missing or has this txg in its DTL.
	312	* Look for any child we haven't already tried before giving up.
	313	*/
	314	for (c = 0; c < mm->mm_children; c++)
	315	if (!mm->mm_child[c].mc_tried)
	316	return (c);
	317
	318	/*
	319	* Every child failed. There's no place left to look.
	320	*/
	321	return (-1);
	322	}
	323
	324	static int
	325	vdev_mirror_io_start(zio_t *zio)
	326	{
	327	mirror_map_t *mm;
	328	mirror_child_t *mc;
	329	int c, children;
	330
	331	mm = vdev_mirror_map_alloc(zio);
	332
	333	if (zio->io_type == ZIO_TYPE_READ) {
	334	if ((zio->io_flags & ZIO_FLAG_SCRUB) && !mm->mm_replacing) {
	335	/*
	336	* For scrubbing reads we need to allocate a read
	337	* buffer for each child and issue reads to all
	338	* children. If any child succeeds, it will copy its
	339	* data into zio->io_data in vdev_mirror_scrub_done.
	340	*/
	341	for (c = 0; c < mm->mm_children; c++) {
	342	mc = &mm->mm_child[c];
	343	zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
	344	mc->mc_vd, mc->mc_offset,
	345	zio_buf_alloc(zio->io_size), zio->io_size,
b128c09f	346	zio->io_type, zio->io_priority, 0,
34dc7c2f BB	347	vdev_mirror_scrub_done, mc));
34dc7c2f BB	348	}
b128c09f	349	return (ZIO_PIPELINE_CONTINUE);
34dc7c2f BB	350	}
	351	/*
	352	* For normal reads just pick one child.
	353	*/
	354	c = vdev_mirror_child_select(zio);
	355	children = (c >= 0);
	356	} else {
	357	ASSERT(zio->io_type == ZIO_TYPE_WRITE);
	358
	359	/*
fb5f0bc8	360	* Writes go to all children.
34dc7c2f	361	*/
fb5f0bc8 BB	362	c = 0;
fb5f0bc8 BB	363	children = mm->mm_children;
34dc7c2f BB	364	}
	365
	366	while (children--) {
	367	mc = &mm->mm_child[c];
	368	zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
b128c09f BB	369	mc->mc_vd, mc->mc_offset, zio->io_data, zio->io_size,
	370	zio->io_type, zio->io_priority, 0,
	371	vdev_mirror_child_done, mc));
34dc7c2f BB	372	c++;
	373	}
	374
b128c09f	375	return (ZIO_PIPELINE_CONTINUE);
34dc7c2f BB	376	}
	377
	378	static int
b128c09f BB	379	vdev_mirror_worst_error(mirror_map_t *mm)
b128c09f BB	380	{
d6320ddb	381	int c, error[2] = { 0, 0 };
b128c09f	382
d6320ddb	383	for (c = 0; c < mm->mm_children; c++) {
b128c09f BB	384	mirror_child_t *mc = &mm->mm_child[c];
	385	int s = mc->mc_speculative;
	386	error[s] = zio_worst_error(error[s], mc->mc_error);
	387	}
	388
	389	return (error[0] ? error[0] : error[1]);
	390	}
	391
	392	static void
34dc7c2f BB	393	vdev_mirror_io_done(zio_t *zio)
	394	{
	395	mirror_map_t *mm = zio->io_vsd;
	396	mirror_child_t *mc;
	397	int c;
	398	int good_copies = 0;
	399	int unexpected_errors = 0;
	400
34dc7c2f BB	401	for (c = 0; c < mm->mm_children; c++) {
	402	mc = &mm->mm_child[c];
	403
34dc7c2f	404	if (mc->mc_error) {
34dc7c2f BB	405	if (!mc->mc_skipped)
34dc7c2f BB	406	unexpected_errors++;
b128c09f BB	407	} else if (mc->mc_tried) {
b128c09f BB	408	good_copies++;
34dc7c2f BB	409	}
	410	}
	411
	412	if (zio->io_type == ZIO_TYPE_WRITE) {
	413	/*
	414	* XXX -- for now, treat partial writes as success.
b128c09f BB	415	*
	416	* Now that we support write reallocation, it would be better
	417	* to treat partial failure as real failure unless there are
	418	* no non-degraded top-level vdevs left, and not update DTLs
	419	* if we intend to reallocate.
34dc7c2f BB	420	*/
34dc7c2f BB	421	/* XXPOLICY */
b128c09f BB	422	if (good_copies != mm->mm_children) {
	423	/*
	424	* Always require at least one good copy.
	425	*
	426	* For ditto blocks (io_vd == NULL), require
	427	* all copies to be good.
	428	*
	429	* XXX -- for replacing vdevs, there's no great answer.
	430	* If the old device is really dead, we may not even
	431	* be able to access it -- so we only want to
	432	* require good writes to the new device. But if
	433	* the new device turns out to be flaky, we want
	434	* to be able to detach it -- which requires all
	435	* writes to the old device to have succeeded.
	436	*/
	437	if (good_copies == 0 \|\| zio->io_vd == NULL)
	438	zio->io_error = vdev_mirror_worst_error(mm);
	439	}
	440	return;
34dc7c2f BB	441	}
	442
	443	ASSERT(zio->io_type == ZIO_TYPE_READ);
	444
	445	/*
	446	* If we don't have a good copy yet, keep trying other children.
	447	*/
	448	/* XXPOLICY */
	449	if (good_copies == 0 && (c = vdev_mirror_child_select(zio)) != -1) {
	450	ASSERT(c >= 0 && c < mm->mm_children);
	451	mc = &mm->mm_child[c];
34dc7c2f BB	452	zio_vdev_io_redone(zio);
	453	zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
	454	mc->mc_vd, mc->mc_offset, zio->io_data, zio->io_size,
b128c09f	455	ZIO_TYPE_READ, zio->io_priority, 0,
34dc7c2f	456	vdev_mirror_child_done, mc));
b128c09f	457	return;
34dc7c2f BB	458	}
	459
	460	/* XXPOLICY */
b128c09f BB	461	if (good_copies == 0) {
b128c09f BB	462	zio->io_error = vdev_mirror_worst_error(mm);
34dc7c2f	463	ASSERT(zio->io_error != 0);
b128c09f	464	}
34dc7c2f	465
fb5f0bc8	466	if (good_copies && spa_writeable(zio->io_spa) &&
34dc7c2f BB	467	(unexpected_errors \|\|
	468	(zio->io_flags & ZIO_FLAG_RESILVER) \|\|
	469	((zio->io_flags & ZIO_FLAG_SCRUB) && mm->mm_replacing))) {
34dc7c2f BB	470	/*
34dc7c2f BB	471	* Use the good data we have in hand to repair damaged children.
34dc7c2f	472	*/
34dc7c2f BB	473	for (c = 0; c < mm->mm_children; c++) {
	474	/*
	475	* Don't rewrite known good children.
	476	* Not only is it unnecessary, it could
	477	* actually be harmful: if the system lost
	478	* power while rewriting the only good copy,
	479	* there would be no good copies left!
	480	*/
	481	mc = &mm->mm_child[c];
	482
	483	if (mc->mc_error == 0) {
	484	if (mc->mc_tried)
	485	continue;
	486	if (!(zio->io_flags & ZIO_FLAG_SCRUB) &&
fb5f0bc8	487	!vdev_dtl_contains(mc->mc_vd, DTL_PARTIAL,
34dc7c2f BB	488	zio->io_txg, 1))
	489	continue;
	490	mc->mc_error = ESTALE;
	491	}
	492
b128c09f BB	493	zio_nowait(zio_vdev_child_io(zio, zio->io_bp,
	494	mc->mc_vd, mc->mc_offset,
	495	zio->io_data, zio->io_size,
34dc7c2f	496	ZIO_TYPE_WRITE, zio->io_priority,
fb5f0bc8 BB	497	ZIO_FLAG_IO_REPAIR \| (unexpected_errors ?
fb5f0bc8 BB	498	ZIO_FLAG_SELF_HEAL : 0), NULL, NULL));
34dc7c2f	499	}
34dc7c2f	500	}
34dc7c2f BB	501	}
	502
	503	static void
	504	vdev_mirror_state_change(vdev_t *vd, int faulted, int degraded)
	505	{
	506	if (faulted == vd->vdev_children)
	507	vdev_set_state(vd, B_FALSE, VDEV_STATE_CANT_OPEN,
	508	VDEV_AUX_NO_REPLICAS);
	509	else if (degraded + faulted != 0)
	510	vdev_set_state(vd, B_FALSE, VDEV_STATE_DEGRADED, VDEV_AUX_NONE);
	511	else
	512	vdev_set_state(vd, B_FALSE, VDEV_STATE_HEALTHY, VDEV_AUX_NONE);
	513	}
	514
	515	vdev_ops_t vdev_mirror_ops = {
	516	vdev_mirror_open,
	517	vdev_mirror_close,
34dc7c2f BB	518	vdev_default_asize,
	519	vdev_mirror_io_start,
	520	vdev_mirror_io_done,
	521	vdev_mirror_state_change,
428870ff BB	522	NULL,
428870ff BB	523	NULL,
34dc7c2f BB	524	VDEV_TYPE_MIRROR, /* name of this vdev type */
	525	B_FALSE /* not a leaf vdev */
	526	};
	527
	528	vdev_ops_t vdev_replacing_ops = {
	529	vdev_mirror_open,
	530	vdev_mirror_close,
34dc7c2f BB	531	vdev_default_asize,
	532	vdev_mirror_io_start,
	533	vdev_mirror_io_done,
	534	vdev_mirror_state_change,
428870ff BB	535	NULL,
428870ff BB	536	NULL,
34dc7c2f BB	537	VDEV_TYPE_REPLACING, /* name of this vdev type */
	538	B_FALSE /* not a leaf vdev */
	539	};
	540
	541	vdev_ops_t vdev_spare_ops = {
	542	vdev_mirror_open,
	543	vdev_mirror_close,
34dc7c2f BB	544	vdev_default_asize,
	545	vdev_mirror_io_start,
	546	vdev_mirror_io_done,
	547	vdev_mirror_state_change,
428870ff BB	548	NULL,
428870ff BB	549	NULL,
34dc7c2f BB	550	VDEV_TYPE_SPARE, /* name of this vdev type */
	551	B_FALSE /* not a leaf vdev */
	552	};
556011db BB	553
	554	#if defined(_KERNEL) && defined(HAVE_SPL)
	555	module_param(zfs_vdev_mirror_switch_us, int, 0644);
	556	MODULE_PARM_DESC(zfs_vdev_mirror_switch_us, "Switch mirrors every N usecs");
	557	#endif