[mirror_zfs-debian.git] / include / linux / blkdev_compat.h

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (C) 2011 Lawrence Livermore National Security, LLC.
 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
 * Written by Brian Behlendorf <behlendorf1@llnl.gov>.
 * LLNL-CODE-403049.
 */

#ifndef _ZFS_BLKDEV_H
#define	_ZFS_BLKDEV_H

#include <linux/blkdev.h>
#include <linux/elevator.h>

#ifndef HAVE_FMODE_T
typedef unsigned __bitwise__ fmode_t;
#endif /* HAVE_FMODE_T */

#ifndef HAVE_BLK_FETCH_REQUEST
static inline struct request *
blk_fetch_request(struct request_queue *q)
{
	struct request *req;

	req = elv_next_request(q);
	if (req)
		blkdev_dequeue_request(req);

	return (req);
}
#endif /* HAVE_BLK_FETCH_REQUEST */

#ifndef HAVE_BLK_REQUEUE_REQUEST
static inline void
blk_requeue_request(request_queue_t *q, struct request *req)
{
	elv_requeue_request(q, req);
}
#endif /* HAVE_BLK_REQUEUE_REQUEST */

#ifndef HAVE_BLK_END_REQUEST
static inline bool
__blk_end_request(struct request *req, int error, unsigned int nr_bytes)
{
	LIST_HEAD(list);

	/*
	 * Request has already been dequeued but 2.6.18 version of
	 * end_request() unconditionally dequeues the request so we
	 * add it to a local list to prevent hitting the BUG_ON.
	 */
	list_add(&req->queuelist, &list);

	/*
	 * The old API required the driver to end each segment and not
	 * the entire request.  In our case we always need to end the
	 * entire request partial requests are not supported.
	 */
	req->hard_cur_sectors = nr_bytes >> 9;
	end_request(req, ((error == 0) ? 1 : error));

	return (0);
}

static inline bool
blk_end_request(struct request *req, int error, unsigned int nr_bytes)
{
	struct request_queue *q = req->q;
	bool rc;

	spin_lock_irq(q->queue_lock);
	rc = __blk_end_request(req, error, nr_bytes);
	spin_unlock_irq(q->queue_lock);

	return (rc);
}
#else
#ifdef HAVE_BLK_END_REQUEST_GPL_ONLY
/*
 * Define required to avoid conflicting 2.6.29 non-static prototype for a
 * GPL-only version of the helper.  As of 2.6.31 the helper is available
 * to non-GPL modules and is not explicitly exported GPL-only.
 */
#define	__blk_end_request __blk_end_request_x
#define	blk_end_request blk_end_request_x

static inline bool
__blk_end_request_x(struct request *req, int error, unsigned int nr_bytes)
{
	/*
	 * The old API required the driver to end each segment and not
	 * the entire request.  In our case we always need to end the
	 * entire request partial requests are not supported.
	 */
	req->hard_cur_sectors = nr_bytes >> 9;
	end_request(req, ((error == 0) ? 1 : error));

	return (0);
}
static inline bool
blk_end_request_x(struct request *req, int error, unsigned int nr_bytes)
{
	struct request_queue *q = req->q;
	bool rc;

	spin_lock_irq(q->queue_lock);
	rc = __blk_end_request_x(req, error, nr_bytes);
	spin_unlock_irq(q->queue_lock);

	return (rc);
}
#endif /* HAVE_BLK_END_REQUEST_GPL_ONLY */
#endif /* HAVE_BLK_END_REQUEST */

/*
 * 2.6.36 API change,
 * The blk_queue_flush() interface has replaced blk_queue_ordered()
 * interface.  However, while the old interface was available to all the
 * new one is GPL-only.   Thus if the GPL-only version is detected we
 * implement our own trivial helper compatibility funcion.   The hope is
 * that long term this function will be opened up.
 */
#if defined(HAVE_BLK_QUEUE_FLUSH) && defined(HAVE_BLK_QUEUE_FLUSH_GPL_ONLY)
#define	blk_queue_flush __blk_queue_flush
static inline void
__blk_queue_flush(struct request_queue *q, unsigned int flags)
{
	q->flush_flags = flags & (REQ_FLUSH | REQ_FUA);
}
#endif /* HAVE_BLK_QUEUE_FLUSH && HAVE_BLK_QUEUE_FLUSH_GPL_ONLY */

#ifndef HAVE_BLK_RQ_POS
static inline sector_t
blk_rq_pos(struct request *req)
{
	return (req->sector);
}
#endif /* HAVE_BLK_RQ_POS */

#ifndef HAVE_BLK_RQ_SECTORS
static inline unsigned int
blk_rq_sectors(struct request *req)
{
	return (req->nr_sectors);
}
#endif /* HAVE_BLK_RQ_SECTORS */

#if !defined(HAVE_BLK_RQ_BYTES) || defined(HAVE_BLK_RQ_BYTES_GPL_ONLY)
/*
 * Define required to avoid conflicting 2.6.29 non-static prototype for a
 * GPL-only version of the helper.  As of 2.6.31 the helper is available
 * to non-GPL modules in the form of a static inline in the header.
 */
#define	blk_rq_bytes __blk_rq_bytes
static inline unsigned int
__blk_rq_bytes(struct request *req)
{
	return (blk_rq_sectors(req) << 9);
}
#endif /* !HAVE_BLK_RQ_BYTES || HAVE_BLK_RQ_BYTES_GPL_ONLY */

/*
 * Most of the blk_* macros were removed in 2.6.36.  Ostensibly this was
 * done to improve readability and allow easier grepping.  However, from
 * a portability stand point the macros are helpful.  Therefore the needed
 * macros are redefined here if they are missing from the kernel.
 */
#ifndef blk_fs_request
#define	blk_fs_request(rq)	((rq)->cmd_type == REQ_TYPE_FS)
#endif

/*
 * 2.6.27 API change,
 * The blk_queue_stackable() queue flag was added in 2.6.27 to handle dm
 * stacking drivers.  Prior to this request stacking drivers were detected
 * by checking (q->request_fn == NULL), for earlier kernels we revert to
 * this legacy behavior.
 */
#ifndef blk_queue_stackable
#define	blk_queue_stackable(q)	((q)->request_fn == NULL)
#endif

/*
 * 2.6.34 API change,
 * The blk_queue_max_hw_sectors() function replaces blk_queue_max_sectors().
 */
#ifndef HAVE_BLK_QUEUE_MAX_HW_SECTORS
#define	blk_queue_max_hw_sectors __blk_queue_max_hw_sectors
static inline void
__blk_queue_max_hw_sectors(struct request_queue *q, unsigned int max_hw_sectors)
{
	blk_queue_max_sectors(q, max_hw_sectors);
}
#endif

/*
 * 2.6.34 API change,
 * The blk_queue_max_segments() function consolidates
 * blk_queue_max_hw_segments() and blk_queue_max_phys_segments().
 */
#ifndef HAVE_BLK_QUEUE_MAX_SEGMENTS
#define	blk_queue_max_segments __blk_queue_max_segments
static inline void
__blk_queue_max_segments(struct request_queue *q, unsigned short max_segments)
{
	blk_queue_max_phys_segments(q, max_segments);
	blk_queue_max_hw_segments(q, max_segments);
}
#endif

/*
 * 2.6.30 API change,
 * The blk_queue_physical_block_size() function was introduced to
 * indicate the smallest I/O the device can write without incurring
 * a read-modify-write penalty.  For older kernels this is a no-op.
 */
#ifndef HAVE_BLK_QUEUE_PHYSICAL_BLOCK_SIZE
#define	blk_queue_physical_block_size(q, x)	((void)(0))
#endif

/*
 * 2.6.30 API change,
 * The blk_queue_io_opt() function was added to indicate the optimal
 * I/O size for the device.  For older kernels this is a no-op.
 */
#ifndef HAVE_BLK_QUEUE_IO_OPT
#define	blk_queue_io_opt(q, x)			((void)(0))
#endif

#ifndef HAVE_GET_DISK_RO
static inline int
get_disk_ro(struct gendisk *disk)
{
	int policy = 0;

	if (disk->part[0])
		policy = disk->part[0]->policy;

	return (policy);
}
#endif /* HAVE_GET_DISK_RO */

#ifndef HAVE_RQ_IS_SYNC
static inline bool
rq_is_sync(struct request *req)
{
	return (req->flags & REQ_RW_SYNC);
}
#endif /* HAVE_RQ_IS_SYNC */

#ifndef HAVE_RQ_FOR_EACH_SEGMENT
struct req_iterator {
	int i;
	struct bio *bio;
};

#define	for_each_bio(_bio)              \
	for (; _bio; _bio = _bio->bi_next)

#define	__rq_for_each_bio(_bio, rq)     \
	if ((rq->bio))                  \
		for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next)

#define	rq_for_each_segment(bvl, _rq, _iter)                    \
	__rq_for_each_bio(_iter.bio, _rq)                       \
		bio_for_each_segment(bvl, _iter.bio, _iter.i)

#define	HAVE_RQ_FOR_EACH_SEGMENT_BVP 1
#endif /* HAVE_RQ_FOR_EACH_SEGMENT */

/*
 * 3.14 API change
 * rq_for_each_segment changed from taking bio_vec * to taking bio_vec.
 * We provide rq_for_each_segment4 which takes both.
 * You should not modify the fields in @bv and @bvp.
 *
 * Note: the if-else is just to inject the assignment before the loop body.
 */
#ifdef HAVE_RQ_FOR_EACH_SEGMENT_BVP
#define	rq_for_each_segment4(bv, bvp, rq, iter)	\
	rq_for_each_segment(bvp, rq, iter)	\
		if ((bv = *bvp), 0)		\
			;			\
		else
#else
#define	rq_for_each_segment4(bv, bvp, rq, iter)	\
	rq_for_each_segment(bv, rq, iter)	\
		if ((bvp = &bv), 0)		\
			;			\
		else
#endif

#ifdef HAVE_BIO_BVEC_ITER
#define	BIO_BI_SECTOR(bio)	(bio)->bi_iter.bi_sector
#define	BIO_BI_SIZE(bio)	(bio)->bi_iter.bi_size
#define	BIO_BI_IDX(bio)		(bio)->bi_iter.bi_idx
#else
#define	BIO_BI_SECTOR(bio)	(bio)->bi_sector
#define	BIO_BI_SIZE(bio)	(bio)->bi_size
#define	BIO_BI_IDX(bio)		(bio)->bi_idx
#endif

/*
 * Portable helper for correctly setting the FAILFAST flags.  The
 * correct usage has changed 3 times from 2.6.12 to 2.6.38.
 */
static inline void
bio_set_flags_failfast(struct block_device *bdev, int *flags)
{
#ifdef CONFIG_BUG
	/*
	 * Disable FAILFAST for loopback devices because of the
	 * following incorrect BUG_ON() in loop_make_request().
	 * This support is also disabled for md devices because the
	 * test suite layers md devices on top of loopback devices.
	 * This may be removed when the loopback driver is fixed.
	 *
	 *   BUG_ON(!lo || (rw != READ && rw != WRITE));
	 */
	if ((MAJOR(bdev->bd_dev) == LOOP_MAJOR) ||
	    (MAJOR(bdev->bd_dev) == MD_MAJOR))
		return;

#ifdef BLOCK_EXT_MAJOR
	if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR)
		return;
#endif /* BLOCK_EXT_MAJOR */
#endif /* CONFIG_BUG */

#ifdef HAVE_BIO_RW_FAILFAST_DTD
	/* BIO_RW_FAILFAST_* preferred interface from 2.6.28 - 2.6.35 */
	*flags |= (
	    (1 << BIO_RW_FAILFAST_DEV) |
	    (1 << BIO_RW_FAILFAST_TRANSPORT) |
	    (1 << BIO_RW_FAILFAST_DRIVER));
#else
#ifdef HAVE_BIO_RW_FAILFAST
	/* BIO_RW_FAILFAST preferred interface from 2.6.12 - 2.6.27 */
	*flags |= (1 << BIO_RW_FAILFAST);
#else
#ifdef HAVE_REQ_FAILFAST_MASK
	/*
	 * REQ_FAILFAST_* preferred interface from 2.6.36 - 2.6.xx,
	 * the BIO_* and REQ_* flags were unified under REQ_* flags.
	 */
	*flags |= REQ_FAILFAST_MASK;
#endif /* HAVE_REQ_FAILFAST_MASK */
#endif /* HAVE_BIO_RW_FAILFAST */
#endif /* HAVE_BIO_RW_FAILFAST_DTD */
}

/*
 * Maximum disk label length, it may be undefined for some kernels.
 */
#ifndef DISK_NAME_LEN
#define	DISK_NAME_LEN	32
#endif /* DISK_NAME_LEN */

/*
 * 2.6.24 API change,
 * The bio_end_io() prototype changed slightly.  These are helper
 * macro's to ensure the prototype and return value are handled.
 */
#ifdef HAVE_2ARGS_BIO_END_IO_T
#define	BIO_END_IO_PROTO(fn, x, y, z)	static void fn(struct bio *x, int z)
#define	BIO_END_IO_RETURN(rc)		return
#else
#define	BIO_END_IO_PROTO(fn, x, y, z)	static int fn( \
					    struct bio *x, \
					    unsigned int y, \
					    int z)
#define	BIO_END_IO_RETURN(rc)		return rc
#endif /* HAVE_2ARGS_BIO_END_IO_T */

/*
 * 2.6.38 - 2.6.x API,
 *   blkdev_get_by_path()
 *   blkdev_put()
 *
 * 2.6.28 - 2.6.37 API,
 *   open_bdev_exclusive()
 *   close_bdev_exclusive()
 *
 * 2.6.12 - 2.6.27 API,
 *   open_bdev_excl()
 *   close_bdev_excl()
 *
 * Used to exclusively open a block device from within the kernel.
 */
#if defined(HAVE_BLKDEV_GET_BY_PATH)
#define	vdev_bdev_open(path, md, hld)	blkdev_get_by_path(path, \
					    (md) | FMODE_EXCL, hld)
#define	vdev_bdev_close(bdev, md)	blkdev_put(bdev, (md) | FMODE_EXCL)
#elif defined(HAVE_OPEN_BDEV_EXCLUSIVE)
#define	vdev_bdev_open(path, md, hld)	open_bdev_exclusive(path, md, hld)
#define	vdev_bdev_close(bdev, md)	close_bdev_exclusive(bdev, md)
#else
#define	vdev_bdev_open(path, md, hld)	open_bdev_excl(path, md, hld)
#define	vdev_bdev_close(bdev, md)	close_bdev_excl(bdev)
#endif /* HAVE_BLKDEV_GET_BY_PATH | HAVE_OPEN_BDEV_EXCLUSIVE */

/*
 * 2.6.22 API change
 * The function invalidate_bdev() lost it's second argument because
 * it was unused.
 */
#ifdef HAVE_1ARG_INVALIDATE_BDEV
#define	vdev_bdev_invalidate(bdev)	invalidate_bdev(bdev)
#else
#define	vdev_bdev_invalidate(bdev)	invalidate_bdev(bdev, 1)
#endif /* HAVE_1ARG_INVALIDATE_BDEV */

/*
 * 2.6.27 API change
 * The function was exported for use, prior to this it existed by the
 * symbol was not exported.
 */
#ifndef HAVE_LOOKUP_BDEV
#define	lookup_bdev(path)		ERR_PTR(-ENOTSUP)
#endif

/*
 * 2.6.30 API change
 * To ensure good performance preferentially use the physical block size
 * for proper alignment.  The physical size is supposed to be the internal
 * sector size used by the device.  This is often 4096 byte for AF devices,
 * while a smaller 512 byte logical size is supported for compatibility.
 *
 * Unfortunately, many drives still misreport their physical sector size.
 * For devices which are known to lie you may need to manually set this
 * at pool creation time with 'zpool create -o ashift=12 ...'.
 *
 * When the physical block size interface isn't available, we fall back to
 * the logical block size interface and then the older hard sector size.
 */
#ifdef HAVE_BDEV_PHYSICAL_BLOCK_SIZE
#define	vdev_bdev_block_size(bdev)	bdev_physical_block_size(bdev)
#else
#ifdef HAVE_BDEV_LOGICAL_BLOCK_SIZE
#define	vdev_bdev_block_size(bdev)	bdev_logical_block_size(bdev)
#else
#define	vdev_bdev_block_size(bdev)	bdev_hardsect_size(bdev)
#endif /* HAVE_BDEV_LOGICAL_BLOCK_SIZE */
#endif /* HAVE_BDEV_PHYSICAL_BLOCK_SIZE */

/*
 * 2.6.37 API change
 * The WRITE_FLUSH, WRITE_FUA, and WRITE_FLUSH_FUA flags have been
 * introduced as a replacement for WRITE_BARRIER.  This was done to
 * allow richer semantics to be expressed to the block layer.  It is
 * the block layers responsibility to choose the correct way to
 * implement these semantics.
 *
 * The existence of these flags implies that REQ_FLUSH an REQ_FUA are
 * defined.  Thus we can safely define VDEV_REQ_FLUSH and VDEV_REQ_FUA
 * compatibility macros.
 */
#ifdef WRITE_FLUSH_FUA
#define	VDEV_WRITE_FLUSH_FUA		WRITE_FLUSH_FUA
#define	VDEV_REQ_FLUSH			REQ_FLUSH
#define	VDEV_REQ_FUA			REQ_FUA
#else
#define	VDEV_WRITE_FLUSH_FUA		WRITE_BARRIER
#define	VDEV_REQ_FLUSH			REQ_HARDBARRIER
#define	VDEV_REQ_FUA			REQ_HARDBARRIER
#endif

/*
 * 2.6.32 API change
 * Use the normal I/O patch for discards.
 */
#ifdef REQ_DISCARD
#define	VDEV_REQ_DISCARD		REQ_DISCARD
#endif

/*
 * 2.6.33 API change
 * Discard granularity and alignment restrictions may now be set.  For
 * older kernels which do not support this it is safe to skip it.
 */
#ifdef HAVE_DISCARD_GRANULARITY
static inline void
blk_queue_discard_granularity(struct request_queue *q, unsigned int dg)
{
	q->limits.discard_granularity = dg;
}
#else
#define	blk_queue_discard_granularity(x, dg)	((void)0)
#endif /* HAVE_DISCARD_GRANULARITY */

/*
 * Default Linux IO Scheduler,
 * Setting the scheduler to noop will allow the Linux IO scheduler to
 * still perform front and back merging, while leaving the request
 * ordering and prioritization to the ZFS IO scheduler.
 */
#define	VDEV_SCHEDULER			"noop"

/*
 * A common holder for vdev_bdev_open() is used to relax the exclusive open
 * semantics slightly.  Internal vdev disk callers may pass VDEV_HOLDER to
 * allow them to open the device multiple times.  Other kernel callers and
 * user space processes which don't pass this value will get EBUSY.  This is
 * currently required for the correct operation of hot spares.
 */
#define	VDEV_HOLDER			((void *)0x2401de7)

#endif /* _ZFS_BLKDEV_H */
Commit	Line	Data
60101509 BB	1	/*
	2	* CDDL HEADER START
	3	*
	4	* The contents of this file are subject to the terms of the
	5	* Common Development and Distribution License (the "License").
	6	* You may not use this file except in compliance with the License.
	7	*
	8	* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	9	* or http://www.opensolaris.org/os/licensing.
	10	* See the License for the specific language governing permissions
	11	* and limitations under the License.
	12	*
	13	* When distributing Covered Code, include this CDDL HEADER in each
	14	* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	15	* If applicable, add the following below this CDDL HEADER, with the
	16	* fields enclosed by brackets "[]" replaced with your own identifying
	17	* information: Portions Copyright [yyyy] [name of copyright owner]
	18	*
	19	* CDDL HEADER END
	20	*/
61e90960	21
60101509	22	/*
61e90960	23	* Copyright (C) 2011 Lawrence Livermore National Security, LLC.
60101509 BB	24	* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
	25	* Written by Brian Behlendorf <behlendorf1@llnl.gov>.
	26	* LLNL-CODE-403049.
	27	*/
	28
61e90960	29	#ifndef _ZFS_BLKDEV_H
a08ee875	30	#define _ZFS_BLKDEV_H
60101509 BB	31
	32	#include <linux/blkdev.h>
	33	#include <linux/elevator.h>
	34
	35	#ifndef HAVE_FMODE_T
	36	typedef unsigned __bitwise__ fmode_t;
	37	#endif /* HAVE_FMODE_T */
	38
	39	#ifndef HAVE_BLK_FETCH_REQUEST
	40	static inline struct request *
	41	blk_fetch_request(struct request_queue *q)
	42	{
	43	struct request *req;
	44
	45	req = elv_next_request(q);
	46	if (req)
	47	blkdev_dequeue_request(req);
	48
a08ee875	49	return (req);
60101509 BB	50	}
	51	#endif /* HAVE_BLK_FETCH_REQUEST */
	52
	53	#ifndef HAVE_BLK_REQUEUE_REQUEST
	54	static inline void
	55	blk_requeue_request(request_queue_t q, struct request req)
	56	{
	57	elv_requeue_request(q, req);
	58	}
	59	#endif /* HAVE_BLK_REQUEUE_REQUEST */
	60
	61	#ifndef HAVE_BLK_END_REQUEST
	62	static inline bool
	63	__blk_end_request(struct request *req, int error, unsigned int nr_bytes)
	64	{
	65	LIST_HEAD(list);
	66
	67	/*
	68	* Request has already been dequeued but 2.6.18 version of
	69	* end_request() unconditionally dequeues the request so we
	70	* add it to a local list to prevent hitting the BUG_ON.
	71	*/
	72	list_add(&req->queuelist, &list);
	73
	74	/*
	75	* The old API required the driver to end each segment and not
	76	* the entire request. In our case we always need to end the
	77	* entire request partial requests are not supported.
	78	*/
	79	req->hard_cur_sectors = nr_bytes >> 9;
	80	end_request(req, ((error == 0) ? 1 : error));
	81
a08ee875	82	return (0);
60101509 BB	83	}
	84
	85	static inline bool
	86	blk_end_request(struct request *req, int error, unsigned int nr_bytes)
	87	{
	88	struct request_queue *q = req->q;
	89	bool rc;
	90
	91	spin_lock_irq(q->queue_lock);
	92	rc = __blk_end_request(req, error, nr_bytes);
	93	spin_unlock_irq(q->queue_lock);
	94
a08ee875	95	return (rc);
60101509 BB	96	}
60101509 BB	97	#else
a08ee875	98	#ifdef HAVE_BLK_END_REQUEST_GPL_ONLY
60101509 BB	99	/*
	100	* Define required to avoid conflicting 2.6.29 non-static prototype for a
	101	* GPL-only version of the helper. As of 2.6.31 the helper is available
	102	* to non-GPL modules and is not explicitly exported GPL-only.
	103	*/
a08ee875 LG	104	#define __blk_end_request __blk_end_request_x
a08ee875 LG	105	#define blk_end_request blk_end_request_x
60101509 BB	106
	107	static inline bool
	108	__blk_end_request_x(struct request *req, int error, unsigned int nr_bytes)
	109	{
	110	/*
	111	* The old API required the driver to end each segment and not
	112	* the entire request. In our case we always need to end the
	113	* entire request partial requests are not supported.
	114	*/
	115	req->hard_cur_sectors = nr_bytes >> 9;
	116	end_request(req, ((error == 0) ? 1 : error));
	117
a08ee875	118	return (0);
60101509 BB	119	}
	120	static inline bool
	121	blk_end_request_x(struct request *req, int error, unsigned int nr_bytes)
	122	{
	123	struct request_queue *q = req->q;
	124	bool rc;
	125
	126	spin_lock_irq(q->queue_lock);
	127	rc = __blk_end_request_x(req, error, nr_bytes);
	128	spin_unlock_irq(q->queue_lock);
	129
a08ee875	130	return (rc);
60101509	131	}
a08ee875	132	#endif /* HAVE_BLK_END_REQUEST_GPL_ONLY */
60101509 BB	133	#endif /* HAVE_BLK_END_REQUEST */
60101509 BB	134
b18019d2 ED	135	/*
	136	* 2.6.36 API change,
	137	* The blk_queue_flush() interface has replaced blk_queue_ordered()
	138	* interface. However, while the old interface was available to all the
	139	* new one is GPL-only. Thus if the GPL-only version is detected we
	140	* implement our own trivial helper compatibility funcion. The hope is
	141	* that long term this function will be opened up.
	142	*/
	143	#if defined(HAVE_BLK_QUEUE_FLUSH) && defined(HAVE_BLK_QUEUE_FLUSH_GPL_ONLY)
a08ee875	144	#define blk_queue_flush __blk_queue_flush
b18019d2 ED	145	static inline void
	146	__blk_queue_flush(struct request_queue *q, unsigned int flags)
	147	{
	148	q->flush_flags = flags & (REQ_FLUSH \| REQ_FUA);
	149	}
	150	#endif /* HAVE_BLK_QUEUE_FLUSH && HAVE_BLK_QUEUE_FLUSH_GPL_ONLY */
	151
60101509 BB	152	#ifndef HAVE_BLK_RQ_POS
	153	static inline sector_t
	154	blk_rq_pos(struct request *req)
	155	{
a08ee875	156	return (req->sector);
60101509 BB	157	}
	158	#endif /* HAVE_BLK_RQ_POS */
	159
	160	#ifndef HAVE_BLK_RQ_SECTORS
	161	static inline unsigned int
	162	blk_rq_sectors(struct request *req)
	163	{
a08ee875	164	return (req->nr_sectors);
60101509 BB	165	}
	166	#endif /* HAVE_BLK_RQ_SECTORS */
	167
	168	#if !defined(HAVE_BLK_RQ_BYTES) \|\| defined(HAVE_BLK_RQ_BYTES_GPL_ONLY)
	169	/*
	170	* Define required to avoid conflicting 2.6.29 non-static prototype for a
	171	* GPL-only version of the helper. As of 2.6.31 the helper is available
	172	* to non-GPL modules in the form of a static inline in the header.
	173	*/
a08ee875	174	#define blk_rq_bytes __blk_rq_bytes
60101509 BB	175	static inline unsigned int
	176	__blk_rq_bytes(struct request *req)
	177	{
a08ee875	178	return (blk_rq_sectors(req) << 9);
60101509 BB	179	}
	180	#endif /* !HAVE_BLK_RQ_BYTES \|\| HAVE_BLK_RQ_BYTES_GPL_ONLY */
	181
8326eb46 BB	182	/*
	183	* Most of the blk_* macros were removed in 2.6.36. Ostensibly this was
	184	* done to improve readability and allow easier grepping. However, from
	185	* a portability stand point the macros are helpful. Therefore the needed
	186	* macros are redefined here if they are missing from the kernel.
	187	*/
	188	#ifndef blk_fs_request
a08ee875	189	#define blk_fs_request(rq) ((rq)->cmd_type == REQ_TYPE_FS)
8326eb46 BB	190	#endif
8326eb46 BB	191
3517f0b7 BB	192	/*
	193	* 2.6.27 API change,
	194	* The blk_queue_stackable() queue flag was added in 2.6.27 to handle dm
	195	* stacking drivers. Prior to this request stacking drivers were detected
	196	* by checking (q->request_fn == NULL), for earlier kernels we revert to
	197	* this legacy behavior.
	198	*/
	199	#ifndef blk_queue_stackable
a08ee875	200	#define blk_queue_stackable(q) ((q)->request_fn == NULL)
3517f0b7 BB	201	#endif
3517f0b7 BB	202
34037afe ED	203	/*
	204	* 2.6.34 API change,
	205	* The blk_queue_max_hw_sectors() function replaces blk_queue_max_sectors().
	206	*/
	207	#ifndef HAVE_BLK_QUEUE_MAX_HW_SECTORS
a08ee875	208	#define blk_queue_max_hw_sectors __blk_queue_max_hw_sectors
34037afe ED	209	static inline void
	210	__blk_queue_max_hw_sectors(struct request_queue *q, unsigned int max_hw_sectors)
	211	{
	212	blk_queue_max_sectors(q, max_hw_sectors);
	213	}
	214	#endif
	215
	216	/*
	217	* 2.6.34 API change,
	218	* The blk_queue_max_segments() function consolidates
	219	* blk_queue_max_hw_segments() and blk_queue_max_phys_segments().
	220	*/
	221	#ifndef HAVE_BLK_QUEUE_MAX_SEGMENTS
a08ee875	222	#define blk_queue_max_segments __blk_queue_max_segments
34037afe ED	223	static inline void
	224	__blk_queue_max_segments(struct request_queue *q, unsigned short max_segments)
	225	{
	226	blk_queue_max_phys_segments(q, max_segments);
	227	blk_queue_max_hw_segments(q, max_segments);
	228	}
	229	#endif
	230
	231	/*
	232	* 2.6.30 API change,
	233	* The blk_queue_physical_block_size() function was introduced to
	234	* indicate the smallest I/O the device can write without incurring
	235	* a read-modify-write penalty. For older kernels this is a no-op.
	236	*/
	237	#ifndef HAVE_BLK_QUEUE_PHYSICAL_BLOCK_SIZE
a08ee875	238	#define blk_queue_physical_block_size(q, x) ((void)(0))
34037afe ED	239	#endif
	240
	241	/*
	242	* 2.6.30 API change,
	243	* The blk_queue_io_opt() function was added to indicate the optimal
	244	* I/O size for the device. For older kernels this is a no-op.
	245	*/
	246	#ifndef HAVE_BLK_QUEUE_IO_OPT
a08ee875	247	#define blk_queue_io_opt(q, x) ((void)(0))
34037afe ED	248	#endif
34037afe ED	249
60101509 BB	250	#ifndef HAVE_GET_DISK_RO
	251	static inline int
	252	get_disk_ro(struct gendisk *disk)
	253	{
	254	int policy = 0;
	255
	256	if (disk->part[0])
	257	policy = disk->part[0]->policy;
	258
a08ee875	259	return (policy);
60101509 BB	260	}
	261	#endif /* HAVE_GET_DISK_RO */
	262
	263	#ifndef HAVE_RQ_IS_SYNC
	264	static inline bool
	265	rq_is_sync(struct request *req)
	266	{
	267	return (req->flags & REQ_RW_SYNC);
	268	}
	269	#endif /* HAVE_RQ_IS_SYNC */
	270
	271	#ifndef HAVE_RQ_FOR_EACH_SEGMENT
	272	struct req_iterator {
	273	int i;
	274	struct bio *bio;
	275	};
	276
a08ee875	277	#define for_each_bio(_bio) \
60101509 BB	278	for (; _bio; _bio = _bio->bi_next)
60101509 BB	279
a08ee875	280	#define __rq_for_each_bio(_bio, rq) \
60101509 BB	281	if ((rq->bio)) \
	282	for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next)
	283
a08ee875	284	#define rq_for_each_segment(bvl, _rq, _iter) \
60101509 BB	285	__rq_for_each_bio(_iter.bio, _rq) \
60101509 BB	286	bio_for_each_segment(bvl, _iter.bio, _iter.i)
ea04106b AX	287
ea04106b AX	288	#define HAVE_RQ_FOR_EACH_SEGMENT_BVP 1
60101509 BB	289	#endif /* HAVE_RQ_FOR_EACH_SEGMENT */
60101509 BB	290
ea04106b AX	291	/*
	292	* 3.14 API change
	293	* rq_for_each_segment changed from taking bio_vec * to taking bio_vec.
	294	* We provide rq_for_each_segment4 which takes both.
	295	* You should not modify the fields in @bv and @bvp.
	296	*
	297	* Note: the if-else is just to inject the assignment before the loop body.
	298	*/
	299	#ifdef HAVE_RQ_FOR_EACH_SEGMENT_BVP
	300	#define rq_for_each_segment4(bv, bvp, rq, iter) \
	301	rq_for_each_segment(bvp, rq, iter) \
	302	if ((bv = *bvp), 0) \
	303	; \
	304	else
	305	#else
	306	#define rq_for_each_segment4(bv, bvp, rq, iter) \
	307	rq_for_each_segment(bv, rq, iter) \
	308	if ((bvp = &bv), 0) \
	309	; \
	310	else
	311	#endif
	312
	313	#ifdef HAVE_BIO_BVEC_ITER
	314	#define BIO_BI_SECTOR(bio) (bio)->bi_iter.bi_sector
	315	#define BIO_BI_SIZE(bio) (bio)->bi_iter.bi_size
	316	#define BIO_BI_IDX(bio) (bio)->bi_iter.bi_idx
	317	#else
	318	#define BIO_BI_SECTOR(bio) (bio)->bi_sector
	319	#define BIO_BI_SIZE(bio) (bio)->bi_size
	320	#define BIO_BI_IDX(bio) (bio)->bi_idx
	321	#endif
	322
61e90960 BB	323	/*
	324	* Portable helper for correctly setting the FAILFAST flags. The
	325	* correct usage has changed 3 times from 2.6.12 to 2.6.38.
	326	*/
2959d94a BB	327	static inline void
	328	bio_set_flags_failfast(struct block_device bdev, int flags)
	329	{
f4af6bb7	330	#ifdef CONFIG_BUG
2959d94a	331	/*
f4af6bb7 BB	332	* Disable FAILFAST for loopback devices because of the
f4af6bb7 BB	333	* following incorrect BUG_ON() in loop_make_request().
2959d94a BB	334	* This support is also disabled for md devices because the
	335	* test suite layers md devices on top of loopback devices.
	336	* This may be removed when the loopback driver is fixed.
	337	*
	338	* BUG_ON(!lo \|\| (rw != READ && rw != WRITE));
	339	*/
2959d94a BB	340	if ((MAJOR(bdev->bd_dev) == LOOP_MAJOR) \|\|
	341	(MAJOR(bdev->bd_dev) == MD_MAJOR))
	342	return;
	343
	344	#ifdef BLOCK_EXT_MAJOR
	345	if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR)
	346	return;
	347	#endif /* BLOCK_EXT_MAJOR */
	348	#endif /* CONFIG_BUG */
f4af6bb7 BB	349
	350	#ifdef HAVE_BIO_RW_FAILFAST_DTD
	351	/* BIO_RW_FAILFAST_* preferred interface from 2.6.28 - 2.6.35 */
a08ee875 LG	352	*flags \|= (
	353	(1 << BIO_RW_FAILFAST_DEV) \|
	354	(1 << BIO_RW_FAILFAST_TRANSPORT) \|
	355	(1 << BIO_RW_FAILFAST_DRIVER));
f4af6bb7	356	#else
a08ee875	357	#ifdef HAVE_BIO_RW_FAILFAST
f4af6bb7	358	/* BIO_RW_FAILFAST preferred interface from 2.6.12 - 2.6.27 */
2959d94a	359	*flags \|= (1 << BIO_RW_FAILFAST);
a08ee875 LG	360	#else
	361	#ifdef HAVE_REQ_FAILFAST_MASK
	362	/*
	363	* REQ_FAILFAST_* preferred interface from 2.6.36 - 2.6.xx,
	364	* the BIO_* and REQ_* flags were unified under REQ_* flags.
	365	*/
f4af6bb7	366	*flags \|= REQ_FAILFAST_MASK;
a08ee875 LG	367	#endif /* HAVE_REQ_FAILFAST_MASK */
a08ee875 LG	368	#endif /* HAVE_BIO_RW_FAILFAST */
f4af6bb7	369	#endif /* HAVE_BIO_RW_FAILFAST_DTD */
2959d94a BB	370	}
2959d94a BB	371
61e90960 BB	372	/*
	373	* Maximum disk label length, it may be undefined for some kernels.
	374	*/
60101509	375	#ifndef DISK_NAME_LEN
a08ee875	376	#define DISK_NAME_LEN 32
60101509 BB	377	#endif /* DISK_NAME_LEN */
60101509 BB	378
61e90960 BB	379	/*
	380	* 2.6.24 API change,
	381	* The bio_end_io() prototype changed slightly. These are helper
	382	* macro's to ensure the prototype and return value are handled.
	383	*/
	384	#ifdef HAVE_2ARGS_BIO_END_IO_T
a08ee875 LG	385	#define BIO_END_IO_PROTO(fn, x, y, z) static void fn(struct bio *x, int z)
a08ee875 LG	386	#define BIO_END_IO_RETURN(rc) return
61e90960	387	#else
a08ee875 LG	388	#define BIO_END_IO_PROTO(fn, x, y, z) static int fn( \
	389	struct bio *x, \
	390	unsigned int y, \
	391	int z)
	392	#define BIO_END_IO_RETURN(rc) return rc
61e90960 BB	393	#endif /* HAVE_2ARGS_BIO_END_IO_T */
	394
	395	/*
45066d1f BB	396	* 2.6.38 - 2.6.x API,
	397	* blkdev_get_by_path()
	398	* blkdev_put()
	399	*
	400	* 2.6.28 - 2.6.37 API,
	401	* open_bdev_exclusive()
	402	* close_bdev_exclusive()
	403	*
	404	* 2.6.12 - 2.6.27 API,
	405	* open_bdev_excl()
	406	* close_bdev_excl()
	407	*
61e90960 BB	408	* Used to exclusively open a block device from within the kernel.
61e90960 BB	409	*/
45066d1f	410	#if defined(HAVE_BLKDEV_GET_BY_PATH)
a08ee875	411	#define vdev_bdev_open(path, md, hld) blkdev_get_by_path(path, \
45066d1f	412	(md) \| FMODE_EXCL, hld)
a08ee875	413	#define vdev_bdev_close(bdev, md) blkdev_put(bdev, (md) \| FMODE_EXCL)
45066d1f	414	#elif defined(HAVE_OPEN_BDEV_EXCLUSIVE)
a08ee875 LG	415	#define vdev_bdev_open(path, md, hld) open_bdev_exclusive(path, md, hld)
a08ee875 LG	416	#define vdev_bdev_close(bdev, md) close_bdev_exclusive(bdev, md)
61e90960	417	#else
a08ee875 LG	418	#define vdev_bdev_open(path, md, hld) open_bdev_excl(path, md, hld)
a08ee875 LG	419	#define vdev_bdev_close(bdev, md) close_bdev_excl(bdev)
45066d1f	420	#endif /* HAVE_BLKDEV_GET_BY_PATH \| HAVE_OPEN_BDEV_EXCLUSIVE */
61e90960 BB	421
	422	/*
	423	* 2.6.22 API change
	424	* The function invalidate_bdev() lost it's second argument because
	425	* it was unused.
	426	*/
	427	#ifdef HAVE_1ARG_INVALIDATE_BDEV
a08ee875	428	#define vdev_bdev_invalidate(bdev) invalidate_bdev(bdev)
61e90960	429	#else
a08ee875	430	#define vdev_bdev_invalidate(bdev) invalidate_bdev(bdev, 1)
61e90960 BB	431	#endif /* HAVE_1ARG_INVALIDATE_BDEV */
61e90960 BB	432
2b7ab9d4 BB	433	/*
	434	* 2.6.27 API change
	435	* The function was exported for use, prior to this it existed by the
	436	* symbol was not exported.
	437	*/
	438	#ifndef HAVE_LOOKUP_BDEV
a08ee875	439	#define lookup_bdev(path) ERR_PTR(-ENOTSUP)
2b7ab9d4 BB	440	#endif
2b7ab9d4 BB	441
61e90960 BB	442	/*
61e90960 BB	443	* 2.6.30 API change
2404b014 BB	444	* To ensure good performance preferentially use the physical block size
	445	* for proper alignment. The physical size is supposed to be the internal
	446	* sector size used by the device. This is often 4096 byte for AF devices,
	447	* while a smaller 512 byte logical size is supported for compatibility.
	448	*
	449	* Unfortunately, many drives still misreport their physical sector size.
	450	* For devices which are known to lie you may need to manually set this
	451	* at pool creation time with 'zpool create -o ashift=12 ...'.
	452	*
	453	* When the physical block size interface isn't available, we fall back to
	454	* the logical block size interface and then the older hard sector size.
61e90960	455	*/
2404b014	456	#ifdef HAVE_BDEV_PHYSICAL_BLOCK_SIZE
a08ee875 LG	457	#define vdev_bdev_block_size(bdev) bdev_physical_block_size(bdev)
	458	#else
	459	#ifdef HAVE_BDEV_LOGICAL_BLOCK_SIZE
	460	#define vdev_bdev_block_size(bdev) bdev_logical_block_size(bdev)
61e90960	461	#else
a08ee875 LG	462	#define vdev_bdev_block_size(bdev) bdev_hardsect_size(bdev)
a08ee875 LG	463	#endif /* HAVE_BDEV_LOGICAL_BLOCK_SIZE */
2404b014	464	#endif /* HAVE_BDEV_PHYSICAL_BLOCK_SIZE */
61e90960	465
96801d29 BB	466	/*
	467	* 2.6.37 API change
	468	* The WRITE_FLUSH, WRITE_FUA, and WRITE_FLUSH_FUA flags have been
	469	* introduced as a replacement for WRITE_BARRIER. This was done to
	470	* allow richer semantics to be expressed to the block layer. It is
	471	* the block layers responsibility to choose the correct way to
	472	* implement these semantics.
b18019d2 ED	473	*
	474	* The existence of these flags implies that REQ_FLUSH an REQ_FUA are
	475	* defined. Thus we can safely define VDEV_REQ_FLUSH and VDEV_REQ_FUA
	476	* compatibility macros.
96801d29 BB	477	*/
96801d29 BB	478	#ifdef WRITE_FLUSH_FUA
a08ee875 LG	479	#define VDEV_WRITE_FLUSH_FUA WRITE_FLUSH_FUA
	480	#define VDEV_REQ_FLUSH REQ_FLUSH
	481	#define VDEV_REQ_FUA REQ_FUA
96801d29	482	#else
a08ee875 LG	483	#define VDEV_WRITE_FLUSH_FUA WRITE_BARRIER
	484	#define VDEV_REQ_FLUSH REQ_HARDBARRIER
	485	#define VDEV_REQ_FUA REQ_HARDBARRIER
96801d29 BB	486	#endif
96801d29 BB	487
30930fba ED	488	/*
	489	* 2.6.32 API change
	490	* Use the normal I/O patch for discards.
	491	*/
	492	#ifdef REQ_DISCARD
a08ee875	493	#define VDEV_REQ_DISCARD REQ_DISCARD
30930fba ED	494	#endif
30930fba ED	495
ee5fd0bb ED	496	/*
	497	* 2.6.33 API change
	498	* Discard granularity and alignment restrictions may now be set. For
	499	* older kernels which do not support this it is safe to skip it.
	500	*/
	501	#ifdef HAVE_DISCARD_GRANULARITY
	502	static inline void
	503	blk_queue_discard_granularity(struct request_queue *q, unsigned int dg)
	504	{
	505	q->limits.discard_granularity = dg;
	506	}
	507	#else
a08ee875	508	#define blk_queue_discard_granularity(x, dg) ((void)0)
ee5fd0bb ED	509	#endif /* HAVE_DISCARD_GRANULARITY */
ee5fd0bb ED	510
61e90960 BB	511	/*
	512	* Default Linux IO Scheduler,
	513	* Setting the scheduler to noop will allow the Linux IO scheduler to
	514	* still perform front and back merging, while leaving the request
	515	* ordering and prioritization to the ZFS IO scheduler.
	516	*/
	517	#define VDEV_SCHEDULER "noop"
60101509	518
8128bd89 BB	519	/*
	520	* A common holder for vdev_bdev_open() is used to relax the exclusive open
	521	* semantics slightly. Internal vdev disk callers may pass VDEV_HOLDER to
	522	* allow them to open the device multiple times. Other kernel callers and
	523	* user space processes which don't pass this value will get EBUSY. This is
	524	* currently required for the correct operation of hot spares.
	525	*/
a08ee875	526	#define VDEV_HOLDER ((void *)0x2401de7)
8128bd89	527
61e90960	528	#endif /* _ZFS_BLKDEV_H */