4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright (C) 2011 Lawrence Livermore National Security, LLC.
24 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
25 * Written by Brian Behlendorf <behlendorf1@llnl.gov>.
32 #include <linux/blkdev.h>
33 #include <linux/elevator.h>
36 typedef unsigned __bitwise__ fmode_t
;
37 #endif /* HAVE_FMODE_T */
39 #ifndef HAVE_BLK_FETCH_REQUEST
40 static inline struct request
*
41 blk_fetch_request(struct request_queue
*q
)
45 req
= elv_next_request(q
);
47 blkdev_dequeue_request(req
);
51 #endif /* HAVE_BLK_FETCH_REQUEST */
53 #ifndef HAVE_BLK_REQUEUE_REQUEST
55 blk_requeue_request(request_queue_t
*q
, struct request
*req
)
57 elv_requeue_request(q
, req
);
59 #endif /* HAVE_BLK_REQUEUE_REQUEST */
61 #ifndef HAVE_BLK_END_REQUEST
63 __blk_end_request(struct request
*req
, int error
, unsigned int nr_bytes
)
68 * Request has already been dequeued but 2.6.18 version of
69 * end_request() unconditionally dequeues the request so we
70 * add it to a local list to prevent hitting the BUG_ON.
72 list_add(&req
->queuelist
, &list
);
75 * The old API required the driver to end each segment and not
76 * the entire request. In our case we always need to end the
77 * entire request partial requests are not supported.
79 req
->hard_cur_sectors
= nr_bytes
>> 9;
80 end_request(req
, ((error
== 0) ? 1 : error
));
86 blk_end_request(struct request
*req
, int error
, unsigned int nr_bytes
)
88 struct request_queue
*q
= req
->q
;
91 spin_lock_irq(q
->queue_lock
);
92 rc
= __blk_end_request(req
, error
, nr_bytes
);
93 spin_unlock_irq(q
->queue_lock
);
98 # ifdef HAVE_BLK_END_REQUEST_GPL_ONLY
100 * Define required to avoid conflicting 2.6.29 non-static prototype for a
101 * GPL-only version of the helper. As of 2.6.31 the helper is available
102 * to non-GPL modules and is not explicitly exported GPL-only.
104 # define __blk_end_request __blk_end_request_x
105 # define blk_end_request blk_end_request_x
108 __blk_end_request_x(struct request
*req
, int error
, unsigned int nr_bytes
)
111 * The old API required the driver to end each segment and not
112 * the entire request. In our case we always need to end the
113 * entire request partial requests are not supported.
115 req
->hard_cur_sectors
= nr_bytes
>> 9;
116 end_request(req
, ((error
== 0) ? 1 : error
));
121 blk_end_request_x(struct request
*req
, int error
, unsigned int nr_bytes
)
123 struct request_queue
*q
= req
->q
;
126 spin_lock_irq(q
->queue_lock
);
127 rc
= __blk_end_request_x(req
, error
, nr_bytes
);
128 spin_unlock_irq(q
->queue_lock
);
132 # endif /* HAVE_BLK_END_REQUEST_GPL_ONLY */
133 #endif /* HAVE_BLK_END_REQUEST */
137 * The blk_queue_flush() interface has replaced blk_queue_ordered()
138 * interface. However, while the old interface was available to all the
139 * new one is GPL-only. Thus if the GPL-only version is detected we
140 * implement our own trivial helper compatibility funcion. The hope is
141 * that long term this function will be opened up.
143 #if defined(HAVE_BLK_QUEUE_FLUSH) && defined(HAVE_BLK_QUEUE_FLUSH_GPL_ONLY)
144 #define blk_queue_flush __blk_queue_flush
146 __blk_queue_flush(struct request_queue
*q
, unsigned int flags
)
148 q
->flush_flags
= flags
& (REQ_FLUSH
| REQ_FUA
);
150 #endif /* HAVE_BLK_QUEUE_FLUSH && HAVE_BLK_QUEUE_FLUSH_GPL_ONLY */
152 #ifndef HAVE_BLK_RQ_POS
153 static inline sector_t
154 blk_rq_pos(struct request
*req
)
158 #endif /* HAVE_BLK_RQ_POS */
160 #ifndef HAVE_BLK_RQ_SECTORS
161 static inline unsigned int
162 blk_rq_sectors(struct request
*req
)
164 return req
->nr_sectors
;
166 #endif /* HAVE_BLK_RQ_SECTORS */
168 #if !defined(HAVE_BLK_RQ_BYTES) || defined(HAVE_BLK_RQ_BYTES_GPL_ONLY)
170 * Define required to avoid conflicting 2.6.29 non-static prototype for a
171 * GPL-only version of the helper. As of 2.6.31 the helper is available
172 * to non-GPL modules in the form of a static inline in the header.
174 #define blk_rq_bytes __blk_rq_bytes
175 static inline unsigned int
176 __blk_rq_bytes(struct request
*req
)
178 return blk_rq_sectors(req
) << 9;
180 #endif /* !HAVE_BLK_RQ_BYTES || HAVE_BLK_RQ_BYTES_GPL_ONLY */
183 * Most of the blk_* macros were removed in 2.6.36. Ostensibly this was
184 * done to improve readability and allow easier grepping. However, from
185 * a portability stand point the macros are helpful. Therefore the needed
186 * macros are redefined here if they are missing from the kernel.
188 #ifndef blk_fs_request
189 #define blk_fs_request(rq) ((rq)->cmd_type == REQ_TYPE_FS)
194 * The blk_queue_stackable() queue flag was added in 2.6.27 to handle dm
195 * stacking drivers. Prior to this request stacking drivers were detected
196 * by checking (q->request_fn == NULL), for earlier kernels we revert to
197 * this legacy behavior.
199 #ifndef blk_queue_stackable
200 #define blk_queue_stackable(q) ((q)->request_fn == NULL)
205 * The blk_queue_max_hw_sectors() function replaces blk_queue_max_sectors().
207 #ifndef HAVE_BLK_QUEUE_MAX_HW_SECTORS
208 #define blk_queue_max_hw_sectors __blk_queue_max_hw_sectors
210 __blk_queue_max_hw_sectors(struct request_queue
*q
, unsigned int max_hw_sectors
)
212 blk_queue_max_sectors(q
, max_hw_sectors
);
218 * The blk_queue_max_segments() function consolidates
219 * blk_queue_max_hw_segments() and blk_queue_max_phys_segments().
221 #ifndef HAVE_BLK_QUEUE_MAX_SEGMENTS
222 #define blk_queue_max_segments __blk_queue_max_segments
224 __blk_queue_max_segments(struct request_queue
*q
, unsigned short max_segments
)
226 blk_queue_max_phys_segments(q
, max_segments
);
227 blk_queue_max_hw_segments(q
, max_segments
);
233 * The blk_queue_physical_block_size() function was introduced to
234 * indicate the smallest I/O the device can write without incurring
235 * a read-modify-write penalty. For older kernels this is a no-op.
237 #ifndef HAVE_BLK_QUEUE_PHYSICAL_BLOCK_SIZE
238 #define blk_queue_physical_block_size(q, x) ((void)(0))
243 * The blk_queue_io_opt() function was added to indicate the optimal
244 * I/O size for the device. For older kernels this is a no-op.
246 #ifndef HAVE_BLK_QUEUE_IO_OPT
247 #define blk_queue_io_opt(q, x) ((void)(0))
250 #ifndef HAVE_GET_DISK_RO
252 get_disk_ro(struct gendisk
*disk
)
257 policy
= disk
->part
[0]->policy
;
261 #endif /* HAVE_GET_DISK_RO */
263 #ifndef HAVE_RQ_IS_SYNC
265 rq_is_sync(struct request
*req
)
267 return (req
->flags
& REQ_RW_SYNC
);
269 #endif /* HAVE_RQ_IS_SYNC */
271 #ifndef HAVE_RQ_FOR_EACH_SEGMENT
272 struct req_iterator
{
277 # define for_each_bio(_bio) \
278 for (; _bio; _bio = _bio->bi_next)
280 # define __rq_for_each_bio(_bio, rq) \
282 for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next)
284 # define rq_for_each_segment(bvl, _rq, _iter) \
285 __rq_for_each_bio(_iter.bio, _rq) \
286 bio_for_each_segment(bvl, _iter.bio, _iter.i)
287 #endif /* HAVE_RQ_FOR_EACH_SEGMENT */
290 * Portable helper for correctly setting the FAILFAST flags. The
291 * correct usage has changed 3 times from 2.6.12 to 2.6.38.
294 bio_set_flags_failfast(struct block_device
*bdev
, int *flags
)
298 * Disable FAILFAST for loopback devices because of the
299 * following incorrect BUG_ON() in loop_make_request().
300 * This support is also disabled for md devices because the
301 * test suite layers md devices on top of loopback devices.
302 * This may be removed when the loopback driver is fixed.
304 * BUG_ON(!lo || (rw != READ && rw != WRITE));
306 if ((MAJOR(bdev
->bd_dev
) == LOOP_MAJOR
) ||
307 (MAJOR(bdev
->bd_dev
) == MD_MAJOR
))
310 #ifdef BLOCK_EXT_MAJOR
311 if (MAJOR(bdev
->bd_dev
) == BLOCK_EXT_MAJOR
)
313 #endif /* BLOCK_EXT_MAJOR */
314 #endif /* CONFIG_BUG */
316 #ifdef HAVE_BIO_RW_FAILFAST_DTD
317 /* BIO_RW_FAILFAST_* preferred interface from 2.6.28 - 2.6.35 */
319 ((1 << BIO_RW_FAILFAST_DEV
) |
320 (1 << BIO_RW_FAILFAST_TRANSPORT
) |
321 (1 << BIO_RW_FAILFAST_DRIVER
));
323 # ifdef HAVE_BIO_RW_FAILFAST
324 /* BIO_RW_FAILFAST preferred interface from 2.6.12 - 2.6.27 */
325 *flags
|= (1 << BIO_RW_FAILFAST
);
327 # ifdef HAVE_REQ_FAILFAST_MASK
328 /* REQ_FAILFAST_* preferred interface from 2.6.36 - 2.6.xx,
329 * the BIO_* and REQ_* flags were unified under REQ_* flags. */
330 *flags
|= REQ_FAILFAST_MASK
;
331 # endif /* HAVE_REQ_FAILFAST_MASK */
332 # endif /* HAVE_BIO_RW_FAILFAST */
333 #endif /* HAVE_BIO_RW_FAILFAST_DTD */
337 * Maximum disk label length, it may be undefined for some kernels.
339 #ifndef DISK_NAME_LEN
340 #define DISK_NAME_LEN 32
341 #endif /* DISK_NAME_LEN */
345 * The bio_end_io() prototype changed slightly. These are helper
346 * macro's to ensure the prototype and return value are handled.
348 #ifdef HAVE_2ARGS_BIO_END_IO_T
349 # define BIO_END_IO_PROTO(fn, x, y, z) static void fn(struct bio *x, int z)
350 # define BIO_END_IO_RETURN(rc) return
352 # define BIO_END_IO_PROTO(fn, x, y, z) static int fn(struct bio *x, \
353 unsigned int y, int z)
354 # define BIO_END_IO_RETURN(rc) return rc
355 #endif /* HAVE_2ARGS_BIO_END_IO_T */
358 * 2.6.38 - 2.6.x API,
359 * blkdev_get_by_path()
362 * 2.6.28 - 2.6.37 API,
363 * open_bdev_exclusive()
364 * close_bdev_exclusive()
366 * 2.6.12 - 2.6.27 API,
370 * Used to exclusively open a block device from within the kernel.
372 #if defined(HAVE_BLKDEV_GET_BY_PATH)
373 # define vdev_bdev_open(path, md, hld) blkdev_get_by_path(path, \
374 (md) | FMODE_EXCL, hld)
375 # define vdev_bdev_close(bdev, md) blkdev_put(bdev, (md) | FMODE_EXCL)
376 #elif defined(HAVE_OPEN_BDEV_EXCLUSIVE)
377 # define vdev_bdev_open(path, md, hld) open_bdev_exclusive(path, md, hld)
378 # define vdev_bdev_close(bdev, md) close_bdev_exclusive(bdev, md)
380 # define vdev_bdev_open(path, md, hld) open_bdev_excl(path, md, hld)
381 # define vdev_bdev_close(bdev, md) close_bdev_excl(bdev)
382 #endif /* HAVE_BLKDEV_GET_BY_PATH | HAVE_OPEN_BDEV_EXCLUSIVE */
386 * The function invalidate_bdev() lost it's second argument because
389 #ifdef HAVE_1ARG_INVALIDATE_BDEV
390 # define vdev_bdev_invalidate(bdev) invalidate_bdev(bdev)
392 # define vdev_bdev_invalidate(bdev) invalidate_bdev(bdev, 1)
393 #endif /* HAVE_1ARG_INVALIDATE_BDEV */
397 * The function was exported for use, prior to this it existed by the
398 * symbol was not exported.
400 #ifndef HAVE_LOOKUP_BDEV
401 # define lookup_bdev(path) ERR_PTR(-ENOTSUP)
406 * To ensure good performance preferentially use the physical block size
407 * for proper alignment. The physical size is supposed to be the internal
408 * sector size used by the device. This is often 4096 byte for AF devices,
409 * while a smaller 512 byte logical size is supported for compatibility.
411 * Unfortunately, many drives still misreport their physical sector size.
412 * For devices which are known to lie you may need to manually set this
413 * at pool creation time with 'zpool create -o ashift=12 ...'.
415 * When the physical block size interface isn't available, we fall back to
416 * the logical block size interface and then the older hard sector size.
418 #ifdef HAVE_BDEV_PHYSICAL_BLOCK_SIZE
419 # define vdev_bdev_block_size(bdev) bdev_physical_block_size(bdev)
421 # ifdef HAVE_BDEV_LOGICAL_BLOCK_SIZE
422 # define vdev_bdev_block_size(bdev) bdev_logical_block_size(bdev)
424 # define vdev_bdev_block_size(bdev) bdev_hardsect_size(bdev)
425 # endif /* HAVE_BDEV_LOGICAL_BLOCK_SIZE */
426 #endif /* HAVE_BDEV_PHYSICAL_BLOCK_SIZE */
430 * The WRITE_FLUSH, WRITE_FUA, and WRITE_FLUSH_FUA flags have been
431 * introduced as a replacement for WRITE_BARRIER. This was done to
432 * allow richer semantics to be expressed to the block layer. It is
433 * the block layers responsibility to choose the correct way to
434 * implement these semantics.
436 * The existence of these flags implies that REQ_FLUSH an REQ_FUA are
437 * defined. Thus we can safely define VDEV_REQ_FLUSH and VDEV_REQ_FUA
438 * compatibility macros.
440 #ifdef WRITE_FLUSH_FUA
441 # define VDEV_WRITE_FLUSH_FUA WRITE_FLUSH_FUA
442 # define VDEV_REQ_FLUSH REQ_FLUSH
443 # define VDEV_REQ_FUA REQ_FUA
445 # define VDEV_WRITE_FLUSH_FUA WRITE_BARRIER
446 # define VDEV_REQ_FLUSH REQ_HARDBARRIER
447 # define VDEV_REQ_FUA REQ_HARDBARRIER
452 * Use the normal I/O patch for discards.
455 # define VDEV_REQ_DISCARD REQ_DISCARD
460 * Discard granularity and alignment restrictions may now be set. For
461 * older kernels which do not support this it is safe to skip it.
463 #ifdef HAVE_DISCARD_GRANULARITY
465 blk_queue_discard_granularity(struct request_queue
*q
, unsigned int dg
)
467 q
->limits
.discard_granularity
= dg
;
470 #define blk_queue_discard_granularity(x, dg) ((void)0)
471 #endif /* HAVE_DISCARD_GRANULARITY */
474 * Default Linux IO Scheduler,
475 * Setting the scheduler to noop will allow the Linux IO scheduler to
476 * still perform front and back merging, while leaving the request
477 * ordering and prioritization to the ZFS IO scheduler.
479 #define VDEV_SCHEDULER "noop"
482 * A common holder for vdev_bdev_open() is used to relax the exclusive open
483 * semantics slightly. Internal vdev disk callers may pass VDEV_HOLDER to
484 * allow them to open the device multiple times. Other kernel callers and
485 * user space processes which don't pass this value will get EBUSY. This is
486 * currently required for the correct operation of hot spares.
488 #define VDEV_HOLDER ((void *)0x2401de7)
490 #endif /* _ZFS_BLKDEV_H */