[mirror_zfs-debian.git] / module / zfs / dmu_zfetch.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright (c) 2013, 2015 by Delphix. All rights reserved.
 */

#include <sys/zfs_context.h>
#include <sys/dnode.h>
#include <sys/dmu_objset.h>
#include <sys/dmu_zfetch.h>
#include <sys/dmu.h>
#include <sys/dbuf.h>
#include <sys/kstat.h>

/*
 * This tunable disables predictive prefetch.  Note that it leaves "prescient"
 * prefetch (e.g. prefetch for zfs send) intact.  Unlike predictive prefetch,
 * prescient prefetch never issues i/os that end up not being needed,
 * so it can't hurt performance.
 */

int zfs_prefetch_disable = B_FALSE;

/* max # of streams per zfetch */
unsigned int	zfetch_max_streams = 8;
/* min time before stream reclaim */
unsigned int	zfetch_min_sec_reap = 2;
/* max bytes to prefetch per stream (default 8MB) */
unsigned int	zfetch_max_distance = 8 * 1024 * 1024;
/* max bytes to prefetch indirects for per stream (default 64MB) */
unsigned int	zfetch_max_idistance = 64 * 1024 * 1024;
/* max number of bytes in an array_read in which we allow prefetching (1MB) */
unsigned long	zfetch_array_rd_sz = 1024 * 1024;

typedef struct zfetch_stats {
	kstat_named_t zfetchstat_hits;
	kstat_named_t zfetchstat_misses;
	kstat_named_t zfetchstat_max_streams;
} zfetch_stats_t;

static zfetch_stats_t zfetch_stats = {
	{ "hits",			KSTAT_DATA_UINT64 },
	{ "misses",			KSTAT_DATA_UINT64 },
	{ "max_streams",		KSTAT_DATA_UINT64 },
};

#define	ZFETCHSTAT_BUMP(stat) \
	atomic_inc_64(&zfetch_stats.stat.value.ui64);

kstat_t		*zfetch_ksp;

void
zfetch_init(void)
{
	zfetch_ksp = kstat_create("zfs", 0, "zfetchstats", "misc",
	    KSTAT_TYPE_NAMED, sizeof (zfetch_stats) / sizeof (kstat_named_t),
	    KSTAT_FLAG_VIRTUAL);

	if (zfetch_ksp != NULL) {
		zfetch_ksp->ks_data = &zfetch_stats;
		kstat_install(zfetch_ksp);
	}
}

void
zfetch_fini(void)
{
	if (zfetch_ksp != NULL) {
		kstat_delete(zfetch_ksp);
		zfetch_ksp = NULL;
	}
}

/*
 * This takes a pointer to a zfetch structure and a dnode.  It performs the
 * necessary setup for the zfetch structure, grokking data from the
 * associated dnode.
 */
void
dmu_zfetch_init(zfetch_t *zf, dnode_t *dno)
{
	if (zf == NULL)
		return;

	zf->zf_dnode = dno;

	list_create(&zf->zf_stream, sizeof (zstream_t),
	    offsetof(zstream_t, zs_node));

	rw_init(&zf->zf_rwlock, NULL, RW_DEFAULT, NULL);
}

static void
dmu_zfetch_stream_remove(zfetch_t *zf, zstream_t *zs)
{
	ASSERT(RW_WRITE_HELD(&zf->zf_rwlock));
	list_remove(&zf->zf_stream, zs);
	mutex_destroy(&zs->zs_lock);
	kmem_free(zs, sizeof (*zs));
}

/*
 * Clean-up state associated with a zfetch structure (e.g. destroy the
 * streams).  This doesn't free the zfetch_t itself, that's left to the caller.
 */
void
dmu_zfetch_fini(zfetch_t *zf)
{
	zstream_t *zs;

	ASSERT(!RW_LOCK_HELD(&zf->zf_rwlock));

	rw_enter(&zf->zf_rwlock, RW_WRITER);
	while ((zs = list_head(&zf->zf_stream)) != NULL)
		dmu_zfetch_stream_remove(zf, zs);
	rw_exit(&zf->zf_rwlock);
	list_destroy(&zf->zf_stream);
	rw_destroy(&zf->zf_rwlock);

	zf->zf_dnode = NULL;
}

/*
 * If there aren't too many streams already, create a new stream.
 * The "blkid" argument is the next block that we expect this stream to access.
 * While we're here, clean up old streams (which haven't been
 * accessed for at least zfetch_min_sec_reap seconds).
 */
static void
dmu_zfetch_stream_create(zfetch_t *zf, uint64_t blkid)
{
	zstream_t *zs;
	zstream_t *zs_next;
	int numstreams = 0;
	uint32_t max_streams;

	ASSERT(RW_WRITE_HELD(&zf->zf_rwlock));

	/*
	 * Clean up old streams.
	 */
	for (zs = list_head(&zf->zf_stream);
	    zs != NULL; zs = zs_next) {
		zs_next = list_next(&zf->zf_stream, zs);
		if (((gethrtime() - zs->zs_atime) / NANOSEC) >
		    zfetch_min_sec_reap)
			dmu_zfetch_stream_remove(zf, zs);
		else
			numstreams++;
	}

	/*
	 * The maximum number of streams is normally zfetch_max_streams,
	 * but for small files we lower it such that it's at least possible
	 * for all the streams to be non-overlapping.
	 *
	 * If we are already at the maximum number of streams for this file,
	 * even after removing old streams, then don't create this stream.
	 */
	max_streams = MAX(1, MIN(zfetch_max_streams,
	    zf->zf_dnode->dn_maxblkid * zf->zf_dnode->dn_datablksz /
	    zfetch_max_distance));
	if (numstreams >= max_streams) {
		ZFETCHSTAT_BUMP(zfetchstat_max_streams);
		return;
	}

	zs = kmem_zalloc(sizeof (*zs), KM_SLEEP);
	zs->zs_blkid = blkid;
	zs->zs_pf_blkid = blkid;
	zs->zs_ipf_blkid = blkid;
	zs->zs_atime = gethrtime();
	mutex_init(&zs->zs_lock, NULL, MUTEX_DEFAULT, NULL);

	list_insert_head(&zf->zf_stream, zs);
}

/*
 * This is the predictive prefetch entry point.  It associates dnode access
 * specified with blkid and nblks arguments with prefetch stream, predicts
 * further accesses based on that stats and initiates speculative prefetch.
 * fetch_data argument specifies whether actual data blocks should be fetched:
 *   FALSE -- prefetch only indirect blocks for predicted data blocks;
 *   TRUE -- prefetch predicted data blocks plus following indirect blocks.
 */
void
dmu_zfetch(zfetch_t *zf, uint64_t blkid, uint64_t nblks, boolean_t fetch_data)
{
	zstream_t *zs;
	int64_t pf_start, ipf_start, ipf_istart, ipf_iend;
	int64_t pf_ahead_blks, max_blks, iblk;
	int epbs, max_dist_blks, pf_nblks, ipf_nblks, i;
	uint64_t end_of_access_blkid;
	end_of_access_blkid = blkid + nblks;

	if (zfs_prefetch_disable)
		return;

	/*
	 * As a fast path for small (single-block) files, ignore access
	 * to the first block.
	 */
	if (blkid == 0)
		return;

	rw_enter(&zf->zf_rwlock, RW_READER);

	for (zs = list_head(&zf->zf_stream); zs != NULL;
	    zs = list_next(&zf->zf_stream, zs)) {
		if (blkid == zs->zs_blkid) {
			mutex_enter(&zs->zs_lock);
			/*
			 * zs_blkid could have changed before we
			 * acquired zs_lock; re-check them here.
			 */
			if (blkid != zs->zs_blkid) {
				mutex_exit(&zs->zs_lock);
				continue;
			}
			break;
		}
	}

	if (zs == NULL) {
		/*
		 * This access is not part of any existing stream.  Create
		 * a new stream for it.
		 */
		ZFETCHSTAT_BUMP(zfetchstat_misses);
		if (rw_tryupgrade(&zf->zf_rwlock))
			dmu_zfetch_stream_create(zf, end_of_access_blkid);
		rw_exit(&zf->zf_rwlock);
		return;
	}

	/*
	 * This access was to a block that we issued a prefetch for on
	 * behalf of this stream. Issue further prefetches for this stream.
	 *
	 * Normally, we start prefetching where we stopped
	 * prefetching last (zs_pf_blkid).  But when we get our first
	 * hit on this stream, zs_pf_blkid == zs_blkid, we don't
	 * want to prefetch the block we just accessed.  In this case,
	 * start just after the block we just accessed.
	 */
	pf_start = MAX(zs->zs_pf_blkid, end_of_access_blkid);

	/*
	 * Double our amount of prefetched data, but don't let the
	 * prefetch get further ahead than zfetch_max_distance.
	 */
	if (fetch_data) {
		max_dist_blks =
		    zfetch_max_distance >> zf->zf_dnode->dn_datablkshift;
		/*
		 * Previously, we were (zs_pf_blkid - blkid) ahead.  We
		 * want to now be double that, so read that amount again,
		 * plus the amount we are catching up by (i.e. the amount
		 * read just now).
		 */
		pf_ahead_blks = zs->zs_pf_blkid - blkid + nblks;
		max_blks = max_dist_blks - (pf_start - end_of_access_blkid);
		pf_nblks = MIN(pf_ahead_blks, max_blks);
	} else {
		pf_nblks = 0;
	}

	zs->zs_pf_blkid = pf_start + pf_nblks;

	/*
	 * Do the same for indirects, starting from where we stopped last,
	 * or where we will stop reading data blocks (and the indirects
	 * that point to them).
	 */
	ipf_start = MAX(zs->zs_ipf_blkid, zs->zs_pf_blkid);
	max_dist_blks = zfetch_max_idistance >> zf->zf_dnode->dn_datablkshift;
	/*
	 * We want to double our distance ahead of the data prefetch
	 * (or reader, if we are not prefetching data).  Previously, we
	 * were (zs_ipf_blkid - blkid) ahead.  To double that, we read
	 * that amount again, plus the amount we are catching up by
	 * (i.e. the amount read now + the amount of data prefetched now).
	 */
	pf_ahead_blks = zs->zs_ipf_blkid - blkid + nblks + pf_nblks;
	max_blks = max_dist_blks - (ipf_start - end_of_access_blkid);
	ipf_nblks = MIN(pf_ahead_blks, max_blks);
	zs->zs_ipf_blkid = ipf_start + ipf_nblks;

	epbs = zf->zf_dnode->dn_indblkshift - SPA_BLKPTRSHIFT;
	ipf_istart = P2ROUNDUP(ipf_start, 1 << epbs) >> epbs;
	ipf_iend = P2ROUNDUP(zs->zs_ipf_blkid, 1 << epbs) >> epbs;

	zs->zs_atime = gethrtime();
	zs->zs_blkid = end_of_access_blkid;
	mutex_exit(&zs->zs_lock);
	rw_exit(&zf->zf_rwlock);

	/*
	 * dbuf_prefetch() is asynchronous (even when it needs to read
	 * indirect blocks), but we still prefer to drop our locks before
	 * calling it to reduce the time we hold them.
	 */

	for (i = 0; i < pf_nblks; i++) {
		dbuf_prefetch(zf->zf_dnode, 0, pf_start + i,
		    ZIO_PRIORITY_ASYNC_READ, ARC_FLAG_PREDICTIVE_PREFETCH);
	}
	for (iblk = ipf_istart; iblk < ipf_iend; iblk++) {
		dbuf_prefetch(zf->zf_dnode, 1, iblk,
		    ZIO_PRIORITY_ASYNC_READ, ARC_FLAG_PREDICTIVE_PREFETCH);
	}
	ZFETCHSTAT_BUMP(zfetchstat_hits);
}

#if defined(_KERNEL) && defined(HAVE_SPL)
/* BEGIN CSTYLED */
module_param(zfs_prefetch_disable, int, 0644);
MODULE_PARM_DESC(zfs_prefetch_disable, "Disable all ZFS prefetching");

module_param(zfetch_max_streams, uint, 0644);
MODULE_PARM_DESC(zfetch_max_streams, "Max number of streams per zfetch");

module_param(zfetch_min_sec_reap, uint, 0644);
MODULE_PARM_DESC(zfetch_min_sec_reap, "Min time before stream reclaim");

module_param(zfetch_max_distance, uint, 0644);
MODULE_PARM_DESC(zfetch_max_distance,
	"Max bytes to prefetch per stream (default 8MB)");

module_param(zfetch_array_rd_sz, ulong, 0644);
MODULE_PARM_DESC(zfetch_array_rd_sz, "Number of bytes in a array_read");
/* END CSTYLED */
#endif
Commit	Line	Data
34dc7c2f BB	1	/*
	2	* CDDL HEADER START
	3	*
	4	* The contents of this file are subject to the terms of the
	5	* Common Development and Distribution License (the "License").
	6	* You may not use this file except in compliance with the License.
	7	*
	8	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	9	* or http://www.opensolaris.org/os/licensing.
	10	* See the License for the specific language governing permissions
	11	* and limitations under the License.
	12	*
	13	* When distributing Covered Code, include this CDDL HEADER in each
	14	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	15	* If applicable, add the following below this CDDL HEADER, with the
	16	* fields enclosed by brackets "[]" replaced with your own identifying
	17	* information: Portions Copyright [yyyy] [name of copyright owner]
	18	*
	19	* CDDL HEADER END
	20	*/
	21	/*
428870ff	22	* Copyright 2009 Sun Microsystems, Inc. All rights reserved.
34dc7c2f BB	23	* Use is subject to license terms.
	24	*/
	25
a08ee875	26	/*
cae5b340	27	* Copyright (c) 2013, 2015 by Delphix. All rights reserved.
a08ee875 LG	28	*/
a08ee875 LG	29
34dc7c2f BB	30	#include <sys/zfs_context.h>
	31	#include <sys/dnode.h>
	32	#include <sys/dmu_objset.h>
	33	#include <sys/dmu_zfetch.h>
	34	#include <sys/dmu.h>
	35	#include <sys/dbuf.h>
428870ff	36	#include <sys/kstat.h>
34dc7c2f BB	37
34dc7c2f BB	38	/*
cae5b340 AX	39	* This tunable disables predictive prefetch. Note that it leaves "prescient"
	40	* prefetch (e.g. prefetch for zfs send) intact. Unlike predictive prefetch,
	41	* prescient prefetch never issues i/os that end up not being needed,
	42	* so it can't hurt performance.
34dc7c2f BB	43	*/
34dc7c2f BB	44
cae5b340	45	int zfs_prefetch_disable = B_FALSE;
34dc7c2f BB	46
34dc7c2f BB	47	/* max # of streams per zfetch */
c409e464	48	unsigned int zfetch_max_streams = 8;
34dc7c2f	49	/* min time before stream reclaim */
c409e464	50	unsigned int zfetch_min_sec_reap = 2;
cae5b340 AX	51	/* max bytes to prefetch per stream (default 8MB) */
	52	unsigned int zfetch_max_distance = 8 * 1024 * 1024;
	53	/* max bytes to prefetch indirects for per stream (default 64MB) */
	54	unsigned int zfetch_max_idistance = 64 * 1024 * 1024;
	55	/* max number of bytes in an array_read in which we allow prefetching (1MB) */
c409e464	56	unsigned long zfetch_array_rd_sz = 1024 * 1024;
34dc7c2f	57
428870ff BB	58	typedef struct zfetch_stats {
	59	kstat_named_t zfetchstat_hits;
	60	kstat_named_t zfetchstat_misses;
cae5b340	61	kstat_named_t zfetchstat_max_streams;
428870ff BB	62	} zfetch_stats_t;
	63
	64	static zfetch_stats_t zfetch_stats = {
	65	{ "hits", KSTAT_DATA_UINT64 },
	66	{ "misses", KSTAT_DATA_UINT64 },
cae5b340	67	{ "max_streams", KSTAT_DATA_UINT64 },
428870ff BB	68	};
428870ff BB	69
cae5b340 AX	70	#define ZFETCHSTAT_BUMP(stat) \
cae5b340 AX	71	atomic_inc_64(&zfetch_stats.stat.value.ui64);
428870ff BB	72
	73	kstat_t *zfetch_ksp;
	74
428870ff BB	75	void
	76	zfetch_init(void)
	77	{
428870ff BB	78	zfetch_ksp = kstat_create("zfs", 0, "zfetchstats", "misc",
	79	KSTAT_TYPE_NAMED, sizeof (zfetch_stats) / sizeof (kstat_named_t),
	80	KSTAT_FLAG_VIRTUAL);
	81
	82	if (zfetch_ksp != NULL) {
	83	zfetch_ksp->ks_data = &zfetch_stats;
	84	kstat_install(zfetch_ksp);
	85	}
	86	}
	87
	88	void
	89	zfetch_fini(void)
	90	{
	91	if (zfetch_ksp != NULL) {
	92	kstat_delete(zfetch_ksp);
	93	zfetch_ksp = NULL;
	94	}
34dc7c2f BB	95	}
	96
	97	/*
	98	* This takes a pointer to a zfetch structure and a dnode. It performs the
	99	* necessary setup for the zfetch structure, grokking data from the
	100	* associated dnode.
	101	*/
	102	void
	103	dmu_zfetch_init(zfetch_t zf, dnode_t dno)
	104	{
cae5b340	105	if (zf == NULL)
34dc7c2f	106	return;
34dc7c2f BB	107
34dc7c2f BB	108	zf->zf_dnode = dno;
34dc7c2f BB	109
34dc7c2f BB	110	list_create(&zf->zf_stream, sizeof (zstream_t),
cae5b340	111	offsetof(zstream_t, zs_node));
34dc7c2f BB	112
	113	rw_init(&zf->zf_rwlock, NULL, RW_DEFAULT, NULL);
	114	}
	115
cae5b340 AX	116	static void
cae5b340 AX	117	dmu_zfetch_stream_remove(zfetch_t zf, zstream_t zs)
34dc7c2f	118	{
cae5b340 AX	119	ASSERT(RW_WRITE_HELD(&zf->zf_rwlock));
	120	list_remove(&zf->zf_stream, zs);
	121	mutex_destroy(&zs->zs_lock);
	122	kmem_free(zs, sizeof (*zs));
34dc7c2f BB	123	}
	124
	125	/*
cae5b340 AX	126	* Clean-up state associated with a zfetch structure (e.g. destroy the
cae5b340 AX	127	* streams). This doesn't free the zfetch_t itself, that's left to the caller.
34dc7c2f BB	128	*/
34dc7c2f BB	129	void
cae5b340	130	dmu_zfetch_fini(zfetch_t *zf)
34dc7c2f	131	{
cae5b340	132	zstream_t *zs;
34dc7c2f BB	133
	134	ASSERT(!RW_LOCK_HELD(&zf->zf_rwlock));
	135
cae5b340 AX	136	rw_enter(&zf->zf_rwlock, RW_WRITER);
	137	while ((zs = list_head(&zf->zf_stream)) != NULL)
	138	dmu_zfetch_stream_remove(zf, zs);
	139	rw_exit(&zf->zf_rwlock);
34dc7c2f BB	140	list_destroy(&zf->zf_stream);
	141	rw_destroy(&zf->zf_rwlock);
	142
	143	zf->zf_dnode = NULL;
	144	}
	145
	146	/*
cae5b340 AX	147	* If there aren't too many streams already, create a new stream.
	148	* The "blkid" argument is the next block that we expect this stream to access.
	149	* While we're here, clean up old streams (which haven't been
	150	* accessed for at least zfetch_min_sec_reap seconds).
34dc7c2f	151	*/
cae5b340 AX	152	static void
cae5b340 AX	153	dmu_zfetch_stream_create(zfetch_t *zf, uint64_t blkid)
34dc7c2f	154	{
cae5b340 AX	155	zstream_t *zs;
	156	zstream_t *zs_next;
	157	int numstreams = 0;
	158	uint32_t max_streams;
34dc7c2f BB	159
	160	ASSERT(RW_WRITE_HELD(&zf->zf_rwlock));
	161
cae5b340 AX	162	/*
	163	* Clean up old streams.
	164	*/
	165	for (zs = list_head(&zf->zf_stream);
	166	zs != NULL; zs = zs_next) {
	167	zs_next = list_next(&zf->zf_stream, zs);
	168	if (((gethrtime() - zs->zs_atime) / NANOSEC) >
	169	zfetch_min_sec_reap)
	170	dmu_zfetch_stream_remove(zf, zs);
	171	else
	172	numstreams++;
34dc7c2f BB	173	}
34dc7c2f BB	174
cae5b340 AX	175	/*
	176	* The maximum number of streams is normally zfetch_max_streams,
	177	* but for small files we lower it such that it's at least possible
	178	* for all the streams to be non-overlapping.
	179	*
	180	* If we are already at the maximum number of streams for this file,
	181	* even after removing old streams, then don't create this stream.
	182	*/
	183	max_streams = MAX(1, MIN(zfetch_max_streams,
	184	zf->zf_dnode->dn_maxblkid * zf->zf_dnode->dn_datablksz /
	185	zfetch_max_distance));
	186	if (numstreams >= max_streams) {
	187	ZFETCHSTAT_BUMP(zfetchstat_max_streams);
	188	return;
34dc7c2f	189	}
34dc7c2f	190
cae5b340 AX	191	zs = kmem_zalloc(sizeof (*zs), KM_SLEEP);
	192	zs->zs_blkid = blkid;
	193	zs->zs_pf_blkid = blkid;
	194	zs->zs_ipf_blkid = blkid;
	195	zs->zs_atime = gethrtime();
	196	mutex_init(&zs->zs_lock, NULL, MUTEX_DEFAULT, NULL);
34dc7c2f	197
cae5b340	198	list_insert_head(&zf->zf_stream, zs);
34dc7c2f BB	199	}
	200
	201	/*
cae5b340 AX	202	* This is the predictive prefetch entry point. It associates dnode access
	203	* specified with blkid and nblks arguments with prefetch stream, predicts
	204	* further accesses based on that stats and initiates speculative prefetch.
	205	* fetch_data argument specifies whether actual data blocks should be fetched:
	206	* FALSE -- prefetch only indirect blocks for predicted data blocks;
	207	* TRUE -- prefetch predicted data blocks plus following indirect blocks.
34dc7c2f BB	208	*/
34dc7c2f BB	209	void
cae5b340	210	dmu_zfetch(zfetch_t *zf, uint64_t blkid, uint64_t nblks, boolean_t fetch_data)
34dc7c2f	211	{
cae5b340 AX	212	zstream_t *zs;
	213	int64_t pf_start, ipf_start, ipf_istart, ipf_iend;
	214	int64_t pf_ahead_blks, max_blks, iblk;
	215	int epbs, max_dist_blks, pf_nblks, ipf_nblks, i;
	216	uint64_t end_of_access_blkid;
	217	end_of_access_blkid = blkid + nblks;
34dc7c2f BB	218
	219	if (zfs_prefetch_disable)
	220	return;
	221
cae5b340 AX	222	/*
	223	* As a fast path for small (single-block) files, ignore access
	224	* to the first block.
	225	*/
	226	if (blkid == 0)
34dc7c2f BB	227	return;
34dc7c2f BB	228
cae5b340	229	rw_enter(&zf->zf_rwlock, RW_READER);
34dc7c2f	230
cae5b340 AX	231	for (zs = list_head(&zf->zf_stream); zs != NULL;
	232	zs = list_next(&zf->zf_stream, zs)) {
	233	if (blkid == zs->zs_blkid) {
	234	mutex_enter(&zs->zs_lock);
	235	/*
	236	* zs_blkid could have changed before we
	237	* acquired zs_lock; re-check them here.
	238	*/
	239	if (blkid != zs->zs_blkid) {
	240	mutex_exit(&zs->zs_lock);
	241	continue;
	242	}
	243	break;
	244	}
	245	}
34dc7c2f	246
cae5b340 AX	247	if (zs == NULL) {
	248	/*
	249	* This access is not part of any existing stream. Create
	250	* a new stream for it.
	251	*/
428870ff	252	ZFETCHSTAT_BUMP(zfetchstat_misses);
cae5b340 AX	253	if (rw_tryupgrade(&zf->zf_rwlock))
	254	dmu_zfetch_stream_create(zf, end_of_access_blkid);
	255	rw_exit(&zf->zf_rwlock);
	256	return;
34dc7c2f BB	257	}
34dc7c2f BB	258
cae5b340 AX	259	/*
	260	* This access was to a block that we issued a prefetch for on
	261	* behalf of this stream. Issue further prefetches for this stream.
	262	*
	263	* Normally, we start prefetching where we stopped
	264	* prefetching last (zs_pf_blkid). But when we get our first
	265	* hit on this stream, zs_pf_blkid == zs_blkid, we don't
	266	* want to prefetch the block we just accessed. In this case,
	267	* start just after the block we just accessed.
	268	*/
	269	pf_start = MAX(zs->zs_pf_blkid, end_of_access_blkid);
34dc7c2f	270
cae5b340 AX	271	/*
	272	* Double our amount of prefetched data, but don't let the
	273	* prefetch get further ahead than zfetch_max_distance.
	274	*/
	275	if (fetch_data) {
	276	max_dist_blks =
	277	zfetch_max_distance >> zf->zf_dnode->dn_datablkshift;
34dc7c2f	278	/*
cae5b340 AX	279	* Previously, we were (zs_pf_blkid - blkid) ahead. We
	280	* want to now be double that, so read that amount again,
	281	* plus the amount we are catching up by (i.e. the amount
	282	* read just now).
34dc7c2f	283	*/
cae5b340 AX	284	pf_ahead_blks = zs->zs_pf_blkid - blkid + nblks;
	285	max_blks = max_dist_blks - (pf_start - end_of_access_blkid);
	286	pf_nblks = MIN(pf_ahead_blks, max_blks);
	287	} else {
	288	pf_nblks = 0;
	289	}
34dc7c2f	290
cae5b340	291	zs->zs_pf_blkid = pf_start + pf_nblks;
34dc7c2f	292
cae5b340 AX	293	/*
	294	* Do the same for indirects, starting from where we stopped last,
	295	* or where we will stop reading data blocks (and the indirects
	296	* that point to them).
	297	*/
	298	ipf_start = MAX(zs->zs_ipf_blkid, zs->zs_pf_blkid);
	299	max_dist_blks = zfetch_max_idistance >> zf->zf_dnode->dn_datablkshift;
	300	/*
	301	* We want to double our distance ahead of the data prefetch
	302	* (or reader, if we are not prefetching data). Previously, we
	303	* were (zs_ipf_blkid - blkid) ahead. To double that, we read
	304	* that amount again, plus the amount we are catching up by
	305	* (i.e. the amount read now + the amount of data prefetched now).
	306	*/
	307	pf_ahead_blks = zs->zs_ipf_blkid - blkid + nblks + pf_nblks;
	308	max_blks = max_dist_blks - (ipf_start - end_of_access_blkid);
	309	ipf_nblks = MIN(pf_ahead_blks, max_blks);
	310	zs->zs_ipf_blkid = ipf_start + ipf_nblks;
	311
	312	epbs = zf->zf_dnode->dn_indblkshift - SPA_BLKPTRSHIFT;
	313	ipf_istart = P2ROUNDUP(ipf_start, 1 << epbs) >> epbs;
	314	ipf_iend = P2ROUNDUP(zs->zs_ipf_blkid, 1 << epbs) >> epbs;
	315
	316	zs->zs_atime = gethrtime();
	317	zs->zs_blkid = end_of_access_blkid;
	318	mutex_exit(&zs->zs_lock);
	319	rw_exit(&zf->zf_rwlock);
34dc7c2f	320
cae5b340 AX	321	/*
	322	* dbuf_prefetch() is asynchronous (even when it needs to read
	323	* indirect blocks), but we still prefer to drop our locks before
	324	* calling it to reduce the time we hold them.
	325	*/
34dc7c2f	326
cae5b340 AX	327	for (i = 0; i < pf_nblks; i++) {
	328	dbuf_prefetch(zf->zf_dnode, 0, pf_start + i,
	329	ZIO_PRIORITY_ASYNC_READ, ARC_FLAG_PREDICTIVE_PREFETCH);
	330	}
	331	for (iblk = ipf_istart; iblk < ipf_iend; iblk++) {
	332	dbuf_prefetch(zf->zf_dnode, 1, iblk,
	333	ZIO_PRIORITY_ASYNC_READ, ARC_FLAG_PREDICTIVE_PREFETCH);
34dc7c2f	334	}
cae5b340	335	ZFETCHSTAT_BUMP(zfetchstat_hits);
34dc7c2f	336	}
c28b2279 BB	337
c28b2279 BB	338	#if defined(_KERNEL) && defined(HAVE_SPL)
cae5b340	339	/* BEGIN CSTYLED */
c28b2279 BB	340	module_param(zfs_prefetch_disable, int, 0644);
c28b2279 BB	341	MODULE_PARM_DESC(zfs_prefetch_disable, "Disable all ZFS prefetching");
c409e464 BB	342
	343	module_param(zfetch_max_streams, uint, 0644);
	344	MODULE_PARM_DESC(zfetch_max_streams, "Max number of streams per zfetch");
	345
	346	module_param(zfetch_min_sec_reap, uint, 0644);
	347	MODULE_PARM_DESC(zfetch_min_sec_reap, "Min time before stream reclaim");
	348
cae5b340 AX	349	module_param(zfetch_max_distance, uint, 0644);
	350	MODULE_PARM_DESC(zfetch_max_distance,
	351	"Max bytes to prefetch per stream (default 8MB)");
c409e464 BB	352
	353	module_param(zfetch_array_rd_sz, ulong, 0644);
	354	MODULE_PARM_DESC(zfetch_array_rd_sz, "Number of bytes in a array_read");
cae5b340	355	/* END CSTYLED */
c28b2279	356	#endif