include/os/linux/kernel/linux/blkdev_compat.h

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or https://opensource.org/licenses/CDDL-1.0.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright (C) 2011 Lawrence Livermore National Security, LLC.
  24  * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
  25  * Written by Brian Behlendorf <behlendorf1@llnl.gov>.
  26  * LLNL-CODE-403049.
  27  */
  28
  29 #ifndef _ZFS_BLKDEV_H
  30 #define _ZFS_BLKDEV_H
  31
  32 #include <linux/blkdev.h>
  33 #include <linux/backing-dev.h>
  34 #include <linux/hdreg.h>
  35 #include <linux/major.h>
  36 #include <linux/msdos_fs.h>     /* for SECTOR_* */
  37 #include <linux/bio.h>
  38
  39 #ifdef HAVE_BLK_MQ
  40 #include <linux/blk-mq.h>
  41 #endif
  42
  43 #ifndef HAVE_BLK_QUEUE_FLAG_SET
  44 static inline void
  45 blk_queue_flag_set(unsigned int flag, struct request_queue *q)
  46 {
  47         queue_flag_set(flag, q);
  48 }
  49 #endif
  50
  51 #ifndef HAVE_BLK_QUEUE_FLAG_CLEAR
  52 static inline void
  53 blk_queue_flag_clear(unsigned int flag, struct request_queue *q)
  54 {
  55         queue_flag_clear(flag, q);
  56 }
  57 #endif
  58
  59 /*
  60  * 4.7 API,
  61  * The blk_queue_write_cache() interface has replaced blk_queue_flush()
  62  * interface.  However, the new interface is GPL-only thus we implement
  63  * our own trivial wrapper when the GPL-only version is detected.
  64  *
  65  * 2.6.36 - 4.6 API,
  66  * The blk_queue_flush() interface has replaced blk_queue_ordered()
  67  * interface.  However, while the old interface was available to all the
  68  * new one is GPL-only.   Thus if the GPL-only version is detected we
  69  * implement our own trivial helper.
  70  */
  71 static inline void
  72 blk_queue_set_write_cache(struct request_queue *q, bool wc, bool fua)
  73 {
  74 #if defined(HAVE_BLK_QUEUE_WRITE_CACHE_GPL_ONLY)
  75         if (wc)
  76                 blk_queue_flag_set(QUEUE_FLAG_WC, q);
  77         else
  78                 blk_queue_flag_clear(QUEUE_FLAG_WC, q);
  79         if (fua)
  80                 blk_queue_flag_set(QUEUE_FLAG_FUA, q);
  81         else
  82                 blk_queue_flag_clear(QUEUE_FLAG_FUA, q);
  83 #elif defined(HAVE_BLK_QUEUE_WRITE_CACHE)
  84         blk_queue_write_cache(q, wc, fua);
  85 #elif defined(HAVE_BLK_QUEUE_FLUSH_GPL_ONLY)
  86         if (wc)
  87                 q->flush_flags |= REQ_FLUSH;
  88         if (fua)
  89                 q->flush_flags |= REQ_FUA;
  90 #elif defined(HAVE_BLK_QUEUE_FLUSH)
  91         blk_queue_flush(q, (wc ? REQ_FLUSH : 0) | (fua ? REQ_FUA : 0));
  92 #else
  93 #error "Unsupported kernel"
  94 #endif
  95 }
  96
  97 static inline void
  98 blk_queue_set_read_ahead(struct request_queue *q, unsigned long ra_pages)
  99 {
 100 #if !defined(HAVE_BLK_QUEUE_UPDATE_READAHEAD) && \
 101         !defined(HAVE_DISK_UPDATE_READAHEAD)
 102 #ifdef HAVE_BLK_QUEUE_BDI_DYNAMIC
 103         q->backing_dev_info->ra_pages = ra_pages;
 104 #else
 105         q->backing_dev_info.ra_pages = ra_pages;
 106 #endif
 107 #endif
 108 }
 109
 110 #ifdef HAVE_BIO_BVEC_ITER
 111 #define BIO_BI_SECTOR(bio)      (bio)->bi_iter.bi_sector
 112 #define BIO_BI_SIZE(bio)        (bio)->bi_iter.bi_size
 113 #define BIO_BI_IDX(bio)         (bio)->bi_iter.bi_idx
 114 #define BIO_BI_SKIP(bio)        (bio)->bi_iter.bi_bvec_done
 115 #define bio_for_each_segment4(bv, bvp, b, i)    \
 116         bio_for_each_segment((bv), (b), (i))
 117 typedef struct bvec_iter bvec_iterator_t;
 118 #else
 119 #define BIO_BI_SECTOR(bio)      (bio)->bi_sector
 120 #define BIO_BI_SIZE(bio)        (bio)->bi_size
 121 #define BIO_BI_IDX(bio)         (bio)->bi_idx
 122 #define BIO_BI_SKIP(bio)        (0)
 123 #define bio_for_each_segment4(bv, bvp, b, i)    \
 124         bio_for_each_segment((bvp), (b), (i))
 125 typedef int bvec_iterator_t;
 126 #endif
 127
 128 static inline void
 129 bio_set_flags_failfast(struct block_device *bdev, int *flags, bool dev,
 130     bool transport, bool driver)
 131 {
 132 #ifdef CONFIG_BUG
 133         /*
 134          * Disable FAILFAST for loopback devices because of the
 135          * following incorrect BUG_ON() in loop_make_request().
 136          * This support is also disabled for md devices because the
 137          * test suite layers md devices on top of loopback devices.
 138          * This may be removed when the loopback driver is fixed.
 139          *
 140          *   BUG_ON(!lo || (rw != READ && rw != WRITE));
 141          */
 142         if ((MAJOR(bdev->bd_dev) == LOOP_MAJOR) ||
 143             (MAJOR(bdev->bd_dev) == MD_MAJOR))
 144                 return;
 145
 146 #ifdef BLOCK_EXT_MAJOR
 147         if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR)
 148                 return;
 149 #endif /* BLOCK_EXT_MAJOR */
 150 #endif /* CONFIG_BUG */
 151
 152         if (dev)
 153                 *flags |= REQ_FAILFAST_DEV;
 154         if (transport)
 155                 *flags |= REQ_FAILFAST_TRANSPORT;
 156         if (driver)
 157                 *flags |= REQ_FAILFAST_DRIVER;
 158 }
 159
 160 /*
 161  * Maximum disk label length, it may be undefined for some kernels.
 162  */
 163 #if !defined(DISK_NAME_LEN)
 164 #define DISK_NAME_LEN   32
 165 #endif /* DISK_NAME_LEN */
 166
 167 #ifdef HAVE_BIO_BI_STATUS
 168 static inline int
 169 bi_status_to_errno(blk_status_t status)
 170 {
 171         switch (status) {
 172         case BLK_STS_OK:
 173                 return (0);
 174         case BLK_STS_NOTSUPP:
 175                 return (EOPNOTSUPP);
 176         case BLK_STS_TIMEOUT:
 177                 return (ETIMEDOUT);
 178         case BLK_STS_NOSPC:
 179                 return (ENOSPC);
 180         case BLK_STS_TRANSPORT:
 181                 return (ENOLINK);
 182         case BLK_STS_TARGET:
 183                 return (EREMOTEIO);
 184         case BLK_STS_NEXUS:
 185                 return (EBADE);
 186         case BLK_STS_MEDIUM:
 187                 return (ENODATA);
 188         case BLK_STS_PROTECTION:
 189                 return (EILSEQ);
 190         case BLK_STS_RESOURCE:
 191                 return (ENOMEM);
 192         case BLK_STS_AGAIN:
 193                 return (EAGAIN);
 194         case BLK_STS_IOERR:
 195                 return (EIO);
 196         default:
 197                 return (EIO);
 198         }
 199 }
 200
 201 static inline blk_status_t
 202 errno_to_bi_status(int error)
 203 {
 204         switch (error) {
 205         case 0:
 206                 return (BLK_STS_OK);
 207         case EOPNOTSUPP:
 208                 return (BLK_STS_NOTSUPP);
 209         case ETIMEDOUT:
 210                 return (BLK_STS_TIMEOUT);
 211         case ENOSPC:
 212                 return (BLK_STS_NOSPC);
 213         case ENOLINK:
 214                 return (BLK_STS_TRANSPORT);
 215         case EREMOTEIO:
 216                 return (BLK_STS_TARGET);
 217         case EBADE:
 218                 return (BLK_STS_NEXUS);
 219         case ENODATA:
 220                 return (BLK_STS_MEDIUM);
 221         case EILSEQ:
 222                 return (BLK_STS_PROTECTION);
 223         case ENOMEM:
 224                 return (BLK_STS_RESOURCE);
 225         case EAGAIN:
 226                 return (BLK_STS_AGAIN);
 227         case EIO:
 228                 return (BLK_STS_IOERR);
 229         default:
 230                 return (BLK_STS_IOERR);
 231         }
 232 }
 233 #endif /* HAVE_BIO_BI_STATUS */
 234
 235 /*
 236  * 4.3 API change
 237  * The bio_endio() prototype changed slightly.  These are helper
 238  * macro's to ensure the prototype and invocation are handled.
 239  */
 240 #ifdef HAVE_1ARG_BIO_END_IO_T
 241 #ifdef HAVE_BIO_BI_STATUS
 242 #define BIO_END_IO_ERROR(bio)           bi_status_to_errno(bio->bi_status)
 243 #define BIO_END_IO_PROTO(fn, x, z)      static void fn(struct bio *x)
 244 #define BIO_END_IO(bio, error)          bio_set_bi_status(bio, error)
 245 static inline void
 246 bio_set_bi_status(struct bio *bio, int error)
 247 {
 248         ASSERT3S(error, <=, 0);
 249         bio->bi_status = errno_to_bi_status(-error);
 250         bio_endio(bio);
 251 }
 252 #else
 253 #define BIO_END_IO_ERROR(bio)           (-(bio->bi_error))
 254 #define BIO_END_IO_PROTO(fn, x, z)      static void fn(struct bio *x)
 255 #define BIO_END_IO(bio, error)          bio_set_bi_error(bio, error)
 256 static inline void
 257 bio_set_bi_error(struct bio *bio, int error)
 258 {
 259         ASSERT3S(error, <=, 0);
 260         bio->bi_error = error;
 261         bio_endio(bio);
 262 }
 263 #endif /* HAVE_BIO_BI_STATUS */
 264
 265 #else
 266 #define BIO_END_IO_PROTO(fn, x, z)      static void fn(struct bio *x, int z)
 267 #define BIO_END_IO(bio, error)          bio_endio(bio, error);
 268 #endif /* HAVE_1ARG_BIO_END_IO_T */
 269
 270 /*
 271  * 5.15 MACRO,
 272  *   GD_DEAD
 273  *
 274  * 2.6.36 - 5.14 MACRO,
 275  *   GENHD_FL_UP
 276  *
 277  * Check the disk status and return B_TRUE if alive
 278  * otherwise B_FALSE
 279  */
 280 static inline boolean_t
 281 zfs_check_disk_status(struct block_device *bdev)
 282 {
 283 #if defined(GENHD_FL_UP)
 284         return (!!(bdev->bd_disk->flags & GENHD_FL_UP));
 285 #elif defined(GD_DEAD)
 286         return (!test_bit(GD_DEAD, &bdev->bd_disk->state));
 287 #else
 288 /*
 289  * This is encountered if neither GENHD_FL_UP nor GD_DEAD is available in
 290  * the kernel - likely due to an MACRO change that needs to be chased down.
 291  */
 292 #error "Unsupported kernel: no usable disk status check"
 293 #endif
 294 }
 295
 296 /*
 297  * 4.1 API,
 298  * 3.10.0 CentOS 7.x API,
 299  *   blkdev_reread_part()
 300  *
 301  * For older kernels trigger a re-reading of the partition table by calling
 302  * check_disk_change() which calls flush_disk() to invalidate the device.
 303  *
 304  * For newer kernels (as of 5.10), bdev_check_media_change is used, in favor of
 305  * check_disk_change(), with the modification that invalidation is no longer
 306  * forced.
 307  */
 308 #ifdef HAVE_CHECK_DISK_CHANGE
 309 #define zfs_check_media_change(bdev)    check_disk_change(bdev)
 310 #ifdef HAVE_BLKDEV_REREAD_PART
 311 #define vdev_bdev_reread_part(bdev)     blkdev_reread_part(bdev)
 312 #else
 313 #define vdev_bdev_reread_part(bdev)     check_disk_change(bdev)
 314 #endif /* HAVE_BLKDEV_REREAD_PART */
 315 #else
 316 #ifdef HAVE_BDEV_CHECK_MEDIA_CHANGE
 317 static inline int
 318 zfs_check_media_change(struct block_device *bdev)
 319 {
 320 #ifdef HAVE_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK
 321         struct gendisk *gd = bdev->bd_disk;
 322         const struct block_device_operations *bdo = gd->fops;
 323 #endif
 324
 325         if (!bdev_check_media_change(bdev))
 326                 return (0);
 327
 328 #ifdef HAVE_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK
 329         /*
 330          * Force revalidation, to mimic the old behavior of
 331          * check_disk_change()
 332          */
 333         if (bdo->revalidate_disk)
 334                 bdo->revalidate_disk(gd);
 335 #endif
 336
 337         return (0);
 338 }
 339 #define vdev_bdev_reread_part(bdev)     zfs_check_media_change(bdev)
 340 #else
 341 /*
 342  * This is encountered if check_disk_change() and bdev_check_media_change()
 343  * are not available in the kernel - likely due to an API change that needs
 344  * to be chased down.
 345  */
 346 #error "Unsupported kernel: no usable disk change check"
 347 #endif /* HAVE_BDEV_CHECK_MEDIA_CHANGE */
 348 #endif /* HAVE_CHECK_DISK_CHANGE */
 349
 350 /*
 351  * 2.6.27 API change
 352  * The function was exported for use, prior to this it existed but the
 353  * symbol was not exported.
 354  *
 355  * 4.4.0-6.21 API change for Ubuntu
 356  * lookup_bdev() gained a second argument, FMODE_*, to check inode permissions.
 357  *
 358  * 5.11 API change
 359  * Changed to take a dev_t argument which is set on success and return a
 360  * non-zero error code on failure.
 361  */
 362 static inline int
 363 vdev_lookup_bdev(const char *path, dev_t *dev)
 364 {
 365 #if defined(HAVE_DEVT_LOOKUP_BDEV)
 366         return (lookup_bdev(path, dev));
 367 #elif defined(HAVE_1ARG_LOOKUP_BDEV)
 368         struct block_device *bdev = lookup_bdev(path);
 369         if (IS_ERR(bdev))
 370                 return (PTR_ERR(bdev));
 371
 372         *dev = bdev->bd_dev;
 373         bdput(bdev);
 374
 375         return (0);
 376 #elif defined(HAVE_MODE_LOOKUP_BDEV)
 377         struct block_device *bdev = lookup_bdev(path, FMODE_READ);
 378         if (IS_ERR(bdev))
 379                 return (PTR_ERR(bdev));
 380
 381         *dev = bdev->bd_dev;
 382         bdput(bdev);
 383
 384         return (0);
 385 #else
 386 #error "Unsupported kernel"
 387 #endif
 388 }
 389
 390 /*
 391  * Kernels without bio_set_op_attrs use bi_rw for the bio flags.
 392  */
 393 #if !defined(HAVE_BIO_SET_OP_ATTRS)
 394 static inline void
 395 bio_set_op_attrs(struct bio *bio, unsigned rw, unsigned flags)
 396 {
 397 #if defined(HAVE_BIO_BI_OPF)
 398         bio->bi_opf = rw | flags;
 399 #else
 400         bio->bi_rw |= rw | flags;
 401 #endif /* HAVE_BIO_BI_OPF */
 402 }
 403 #endif
 404
 405 /*
 406  * bio_set_flush - Set the appropriate flags in a bio to guarantee
 407  * data are on non-volatile media on completion.
 408  *
 409  * 2.6.37 - 4.8 API,
 410  *   Introduce WRITE_FLUSH, WRITE_FUA, and WRITE_FLUSH_FUA flags as a
 411  *   replacement for WRITE_BARRIER to allow expressing richer semantics
 412  *   to the block layer.  It's up to the block layer to implement the
 413  *   semantics correctly. Use the WRITE_FLUSH_FUA flag combination.
 414  *
 415  * 4.8 - 4.9 API,
 416  *   REQ_FLUSH was renamed to REQ_PREFLUSH.  For consistency with previous
 417  *   OpenZFS releases, prefer the WRITE_FLUSH_FUA flag set if it's available.
 418  *
 419  * 4.10 API,
 420  *   The read/write flags and their modifiers, including WRITE_FLUSH,
 421  *   WRITE_FUA and WRITE_FLUSH_FUA were removed from fs.h in
 422  *   torvalds/linux@70fd7614 and replaced by direct flag modification
 423  *   of the REQ_ flags in bio->bi_opf.  Use REQ_PREFLUSH.
 424  */
 425 static inline void
 426 bio_set_flush(struct bio *bio)
 427 {
 428 #if defined(HAVE_REQ_PREFLUSH)  /* >= 4.10 */
 429         bio_set_op_attrs(bio, 0, REQ_PREFLUSH | REQ_OP_WRITE);
 430 #elif defined(WRITE_FLUSH_FUA)  /* >= 2.6.37 and <= 4.9 */
 431         bio_set_op_attrs(bio, 0, WRITE_FLUSH_FUA);
 432 #else
 433 #error  "Allowing the build will cause bio_set_flush requests to be ignored."
 434 #endif
 435 }
 436
 437 /*
 438  * 4.8 API,
 439  *   REQ_OP_FLUSH
 440  *
 441  * 4.8-rc0 - 4.8-rc1,
 442  *   REQ_PREFLUSH
 443  *
 444  * 2.6.36 - 4.7 API,
 445  *   REQ_FLUSH
 446  *
 447  * in all cases but may have a performance impact for some kernels.  It
 448  * has the advantage of minimizing kernel specific changes in the zvol code.
 449  *
 450  */
 451 static inline boolean_t
 452 bio_is_flush(struct bio *bio)
 453 {
 454 #if defined(HAVE_REQ_OP_FLUSH) && defined(HAVE_BIO_BI_OPF)
 455         return ((bio_op(bio) == REQ_OP_FLUSH) || (bio->bi_opf & REQ_PREFLUSH));
 456 #elif defined(HAVE_REQ_PREFLUSH) && defined(HAVE_BIO_BI_OPF)
 457         return (bio->bi_opf & REQ_PREFLUSH);
 458 #elif defined(HAVE_REQ_PREFLUSH) && !defined(HAVE_BIO_BI_OPF)
 459         return (bio->bi_rw & REQ_PREFLUSH);
 460 #elif defined(HAVE_REQ_FLUSH)
 461         return (bio->bi_rw & REQ_FLUSH);
 462 #else
 463 #error  "Unsupported kernel"
 464 #endif
 465 }
 466
 467 /*
 468  * 4.8 API,
 469  *   REQ_FUA flag moved to bio->bi_opf
 470  *
 471  * 2.6.x - 4.7 API,
 472  *   REQ_FUA
 473  */
 474 static inline boolean_t
 475 bio_is_fua(struct bio *bio)
 476 {
 477 #if defined(HAVE_BIO_BI_OPF)
 478         return (bio->bi_opf & REQ_FUA);
 479 #elif defined(REQ_FUA)
 480         return (bio->bi_rw & REQ_FUA);
 481 #else
 482 #error  "Allowing the build will cause fua requests to be ignored."
 483 #endif
 484 }
 485
 486 /*
 487  * 4.8 API,
 488  *   REQ_OP_DISCARD
 489  *
 490  * 2.6.36 - 4.7 API,
 491  *   REQ_DISCARD
 492  *
 493  * In all cases the normal I/O path is used for discards.  The only
 494  * difference is how the kernel tags individual I/Os as discards.
 495  */
 496 static inline boolean_t
 497 bio_is_discard(struct bio *bio)
 498 {
 499 #if defined(HAVE_REQ_OP_DISCARD)
 500         return (bio_op(bio) == REQ_OP_DISCARD);
 501 #elif defined(HAVE_REQ_DISCARD)
 502         return (bio->bi_rw & REQ_DISCARD);
 503 #else
 504 #error "Unsupported kernel"
 505 #endif
 506 }
 507
 508 /*
 509  * 4.8 API,
 510  *   REQ_OP_SECURE_ERASE
 511  *
 512  * 2.6.36 - 4.7 API,
 513  *   REQ_SECURE
 514  */
 515 static inline boolean_t
 516 bio_is_secure_erase(struct bio *bio)
 517 {
 518 #if defined(HAVE_REQ_OP_SECURE_ERASE)
 519         return (bio_op(bio) == REQ_OP_SECURE_ERASE);
 520 #elif defined(REQ_SECURE)
 521         return (bio->bi_rw & REQ_SECURE);
 522 #else
 523         return (0);
 524 #endif
 525 }
 526
 527 /*
 528  * 2.6.33 API change
 529  * Discard granularity and alignment restrictions may now be set.  For
 530  * older kernels which do not support this it is safe to skip it.
 531  */
 532 static inline void
 533 blk_queue_discard_granularity(struct request_queue *q, unsigned int dg)
 534 {
 535         q->limits.discard_granularity = dg;
 536 }
 537
 538 /*
 539  * 5.19 API,
 540  *   bdev_max_discard_sectors()
 541  *
 542  * 2.6.32 API,
 543  *   blk_queue_discard()
 544  */
 545 static inline boolean_t
 546 bdev_discard_supported(struct block_device *bdev)
 547 {
 548 #if defined(HAVE_BDEV_MAX_DISCARD_SECTORS)
 549         return (!!bdev_max_discard_sectors(bdev));
 550 #elif defined(HAVE_BLK_QUEUE_DISCARD)
 551         return (!!blk_queue_discard(bdev_get_queue(bdev)));
 552 #else
 553 #error "Unsupported kernel"
 554 #endif
 555 }
 556
 557 /*
 558  * 5.19 API,
 559  *   bdev_max_secure_erase_sectors()
 560  *
 561  * 4.8 API,
 562  *   blk_queue_secure_erase()
 563  *
 564  * 2.6.36 - 4.7 API,
 565  *   blk_queue_secdiscard()
 566  */
 567 static inline boolean_t
 568 bdev_secure_discard_supported(struct block_device *bdev)
 569 {
 570 #if defined(HAVE_BDEV_MAX_SECURE_ERASE_SECTORS)
 571         return (!!bdev_max_secure_erase_sectors(bdev));
 572 #elif defined(HAVE_BLK_QUEUE_SECURE_ERASE)
 573         return (!!blk_queue_secure_erase(bdev_get_queue(bdev)));
 574 #elif defined(HAVE_BLK_QUEUE_SECDISCARD)
 575         return (!!blk_queue_secdiscard(bdev_get_queue(bdev)));
 576 #else
 577 #error "Unsupported kernel"
 578 #endif
 579 }
 580
 581 /*
 582  * A common holder for vdev_bdev_open() is used to relax the exclusive open
 583  * semantics slightly.  Internal vdev disk callers may pass VDEV_HOLDER to
 584  * allow them to open the device multiple times.  Other kernel callers and
 585  * user space processes which don't pass this value will get EBUSY.  This is
 586  * currently required for the correct operation of hot spares.
 587  */
 588 #define VDEV_HOLDER                     ((void *)0x2401de7)
 589
 590 static inline unsigned long
 591 blk_generic_start_io_acct(struct request_queue *q __attribute__((unused)),
 592     struct gendisk *disk __attribute__((unused)),
 593     int rw __attribute__((unused)), struct bio *bio)
 594 {
 595 #if defined(HAVE_BDEV_IO_ACCT_63)
 596         return (bdev_start_io_acct(bio->bi_bdev, bio_op(bio),
 597             jiffies));
 598 #elif defined(HAVE_BDEV_IO_ACCT_OLD)
 599         return (bdev_start_io_acct(bio->bi_bdev, bio_sectors(bio),
 600             bio_op(bio), jiffies));
 601 #elif defined(HAVE_DISK_IO_ACCT)
 602         return (disk_start_io_acct(disk, bio_sectors(bio), bio_op(bio)));
 603 #elif defined(HAVE_BIO_IO_ACCT)
 604         return (bio_start_io_acct(bio));
 605 #elif defined(HAVE_GENERIC_IO_ACCT_3ARG)
 606         unsigned long start_time = jiffies;
 607         generic_start_io_acct(rw, bio_sectors(bio), &disk->part0);
 608         return (start_time);
 609 #elif defined(HAVE_GENERIC_IO_ACCT_4ARG)
 610         unsigned long start_time = jiffies;
 611         generic_start_io_acct(q, rw, bio_sectors(bio), &disk->part0);
 612         return (start_time);
 613 #else
 614         /* Unsupported */
 615         return (0);
 616 #endif
 617 }
 618
 619 static inline void
 620 blk_generic_end_io_acct(struct request_queue *q __attribute__((unused)),
 621     struct gendisk *disk __attribute__((unused)),
 622     int rw __attribute__((unused)), struct bio *bio, unsigned long start_time)
 623 {
 624 #if defined(HAVE_BDEV_IO_ACCT_63)
 625         bdev_end_io_acct(bio->bi_bdev, bio_op(bio), bio_sectors(bio),
 626             start_time);
 627 #elif defined(HAVE_BDEV_IO_ACCT_OLD)
 628         bdev_end_io_acct(bio->bi_bdev, bio_op(bio), start_time);
 629 #elif defined(HAVE_DISK_IO_ACCT)
 630         disk_end_io_acct(disk, bio_op(bio), start_time);
 631 #elif defined(HAVE_BIO_IO_ACCT)
 632         bio_end_io_acct(bio, start_time);
 633 #elif defined(HAVE_GENERIC_IO_ACCT_3ARG)
 634         generic_end_io_acct(rw, &disk->part0, start_time);
 635 #elif defined(HAVE_GENERIC_IO_ACCT_4ARG)
 636         generic_end_io_acct(q, rw, &disk->part0, start_time);
 637 #endif
 638 }
 639
 640 #ifndef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS
 641 static inline struct request_queue *
 642 blk_generic_alloc_queue(make_request_fn make_request, int node_id)
 643 {
 644 #if defined(HAVE_BLK_ALLOC_QUEUE_REQUEST_FN)
 645         return (blk_alloc_queue(make_request, node_id));
 646 #elif defined(HAVE_BLK_ALLOC_QUEUE_REQUEST_FN_RH)
 647         return (blk_alloc_queue_rh(make_request, node_id));
 648 #else
 649         struct request_queue *q = blk_alloc_queue(GFP_KERNEL);
 650         if (q != NULL)
 651                 blk_queue_make_request(q, make_request);
 652
 653         return (q);
 654 #endif
 655 }
 656 #endif /* !HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS */
 657
 658 /*
 659  * All the io_*() helper functions below can operate on a bio, or a rq, but
 660  * not both.  The older submit_bio() codepath will pass a bio, and the
 661  * newer blk-mq codepath will pass a rq.
 662  */
 663 static inline int
 664 io_data_dir(struct bio *bio, struct request *rq)
 665 {
 666 #ifdef HAVE_BLK_MQ
 667         if (rq != NULL) {
 668                 if (op_is_write(req_op(rq))) {
 669                         return (WRITE);
 670                 } else {
 671                         return (READ);
 672                 }
 673         }
 674 #else
 675         ASSERT3P(rq, ==, NULL);
 676 #endif
 677         return (bio_data_dir(bio));
 678 }
 679
 680 static inline int
 681 io_is_flush(struct bio *bio, struct request *rq)
 682 {
 683 #ifdef HAVE_BLK_MQ
 684         if (rq != NULL)
 685                 return (req_op(rq) == REQ_OP_FLUSH);
 686 #else
 687         ASSERT3P(rq, ==, NULL);
 688 #endif
 689         return (bio_is_flush(bio));
 690 }
 691
 692 static inline int
 693 io_is_discard(struct bio *bio, struct request *rq)
 694 {
 695 #ifdef HAVE_BLK_MQ
 696         if (rq != NULL)
 697                 return (req_op(rq) == REQ_OP_DISCARD);
 698 #else
 699         ASSERT3P(rq, ==, NULL);
 700 #endif
 701         return (bio_is_discard(bio));
 702 }
 703
 704 static inline int
 705 io_is_secure_erase(struct bio *bio, struct request *rq)
 706 {
 707 #ifdef HAVE_BLK_MQ
 708         if (rq != NULL)
 709                 return (req_op(rq) == REQ_OP_SECURE_ERASE);
 710 #else
 711         ASSERT3P(rq, ==, NULL);
 712 #endif
 713         return (bio_is_secure_erase(bio));
 714 }
 715
 716 static inline int
 717 io_is_fua(struct bio *bio, struct request *rq)
 718 {
 719 #ifdef HAVE_BLK_MQ
 720         if (rq != NULL)
 721                 return (rq->cmd_flags & REQ_FUA);
 722 #else
 723         ASSERT3P(rq, ==, NULL);
 724 #endif
 725         return (bio_is_fua(bio));
 726 }
 727
 728
 729 static inline uint64_t
 730 io_offset(struct bio *bio, struct request *rq)
 731 {
 732 #ifdef HAVE_BLK_MQ
 733         if (rq != NULL)
 734                 return (blk_rq_pos(rq) << 9);
 735 #else
 736         ASSERT3P(rq, ==, NULL);
 737 #endif
 738         return (BIO_BI_SECTOR(bio) << 9);
 739 }
 740
 741 static inline uint64_t
 742 io_size(struct bio *bio, struct request *rq)
 743 {
 744 #ifdef HAVE_BLK_MQ
 745         if (rq != NULL)
 746                 return (blk_rq_bytes(rq));
 747 #else
 748         ASSERT3P(rq, ==, NULL);
 749 #endif
 750         return (BIO_BI_SIZE(bio));
 751 }
 752
 753 static inline int
 754 io_has_data(struct bio *bio, struct request *rq)
 755 {
 756 #ifdef HAVE_BLK_MQ
 757         if (rq != NULL)
 758                 return (bio_has_data(rq->bio));
 759 #else
 760         ASSERT3P(rq, ==, NULL);
 761 #endif
 762         return (bio_has_data(bio));
 763 }
 764 #endif /* _ZFS_BLKDEV_H */