vdev_disk: ensure trim errors are returned immediately

[mirror_zfs.git] / module / os / linux / zfs / vdev_disk.c

diff --git a/module/os/linux/zfs/vdev_disk.c b/module/os/linux/zfs/vdev_disk.c
index a710bb91004ed9aae79291e72aa77844cfaf0412..a560bca918a82d329c00901647702845f0181439 100644 (file)
--- a/module/os/linux/zfs/vdev_disk.c
+++ b/module/os/linux/zfs/vdev_disk.c
@@ -1252,8 +1252,6 @@ vdev_disk_io_flush(struct block_device *bdev, zio_t *zio)
        return (0);
 }
 
-#if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE) || \
-       defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC)
 BIO_END_IO_PROTO(vdev_disk_discard_end_io, bio, error)
 {
        zio_t *zio = bio->bi_private;
@@ -1268,54 +1266,99 @@ BIO_END_IO_PROTO(vdev_disk_discard_end_io, bio, error)
        zio_interrupt(zio);
 }
 
+/*
+ * Wrappers for the different secure erase and discard APIs. We use async
+ * when available; in this case, *biop is set to the last bio in the chain.
+ */
 static int
-vdev_issue_discard_trim(zio_t *zio, unsigned long flags)
+vdev_bdev_issue_secure_erase(zfs_bdev_handle_t *bdh, sector_t sector,
+    sector_t nsect, struct bio **biop)
 {
-       int ret;
-       struct bio *bio = NULL;
+       *biop = NULL;
+       int error;
 
-#if defined(BLKDEV_DISCARD_SECURE)
-       ret = - __blkdev_issue_discard(
-           BDH_BDEV(((vdev_disk_t *)zio->io_vd->vdev_tsd)->vd_bdh),
-           zio->io_offset >> 9, zio->io_size >> 9, GFP_NOFS, flags, &bio);
+#if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE)
+       error = blkdev_issue_secure_erase(BDH_BDEV(bdh),
+           sector, nsect, GFP_NOFS);
+#elif defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_FLAGS)
+       error = __blkdev_issue_discard(BDH_BDEV(bdh),
+           sector, nsect, GFP_NOFS, BLKDEV_DISCARD_SECURE, biop);
+#elif defined(HAVE_BLKDEV_ISSUE_DISCARD_FLAGS)
+       error = blkdev_issue_discard(BDH_BDEV(bdh),
+           sector, nsect, GFP_NOFS, BLKDEV_DISCARD_SECURE);
 #else
-       (void) flags;
-       ret = - __blkdev_issue_discard(
-           BDH_BDEV(((vdev_disk_t *)zio->io_vd->vdev_tsd)->vd_bdh),
-           zio->io_offset >> 9, zio->io_size >> 9, GFP_NOFS, &bio);
+#error "unsupported kernel"
 #endif
-       if (!ret && bio) {
-               bio->bi_private = zio;
-               bio->bi_end_io = vdev_disk_discard_end_io;
-               vdev_submit_bio(bio);
-       }
-       return (ret);
+
+       return (error);
 }
+
+static int
+vdev_bdev_issue_discard(zfs_bdev_handle_t *bdh, sector_t sector,
+    sector_t nsect, struct bio **biop)
+{
+       *biop = NULL;
+       int error;
+
+#if defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_FLAGS)
+       error = __blkdev_issue_discard(BDH_BDEV(bdh),
+           sector, nsect, GFP_NOFS, 0, biop);
+#elif defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_NOFLAGS)
+       error = __blkdev_issue_discard(BDH_BDEV(bdh),
+           sector, nsect, GFP_NOFS, biop);
+#elif defined(HAVE_BLKDEV_ISSUE_DISCARD_FLAGS)
+       error = blkdev_issue_discard(BDH_BDEV(bdh),
+           sector, nsect, GFP_NOFS, 0);
+#elif defined(HAVE_BLKDEV_ISSUE_DISCARD_NOFLAGS)
+       error = blkdev_issue_discard(BDH_BDEV(bdh),
+           sector, nsect, GFP_NOFS);
+#else
+#error "unsupported kernel"
 #endif
 
+       return (error);
+}
+
+/*
+ * Entry point for TRIM ops. This calls the right wrapper for secure erase or
+ * discard, and then does the appropriate finishing work for error vs success
+ * and async vs sync.
+ */
 static int
 vdev_disk_io_trim(zio_t *zio)
 {
-       unsigned long trim_flags = 0;
-       if (zio->io_trim_flags & ZIO_TRIM_SECURE) {
-#if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE)
-               return (-blkdev_issue_secure_erase(
-                   BDH_BDEV(((vdev_disk_t *)zio->io_vd->vdev_tsd)->vd_bdh),
-                   zio->io_offset >> 9, zio->io_size >> 9, GFP_NOFS));
-#elif defined(BLKDEV_DISCARD_SECURE)
-               trim_flags |= BLKDEV_DISCARD_SECURE;
-#endif
+       int error;
+       struct bio *bio;
+
+       zfs_bdev_handle_t *bdh = ((vdev_disk_t *)zio->io_vd->vdev_tsd)->vd_bdh;
+       sector_t sector = zio->io_offset >> 9;
+       sector_t nsects = zio->io_size >> 9;
+
+       if (zio->io_trim_flags & ZIO_TRIM_SECURE)
+               error = vdev_bdev_issue_secure_erase(bdh, sector, nsects, &bio);
+       else
+               error = vdev_bdev_issue_discard(bdh, sector, nsects, &bio);
+
+       if (error != 0)
+               return (SET_ERROR(-error));
+
+       if (bio == NULL) {
+               /*
+                * This was a synchronous op that completed successfully, so
+                * return it to ZFS immediately.
+                */
+               zio_interrupt(zio);
+       } else {
+               /*
+                * This was an asynchronous op; set up completion callback and
+                * issue it.
+                */
+               bio->bi_private = zio;
+               bio->bi_end_io = vdev_disk_discard_end_io;
+               vdev_submit_bio(bio);
        }
-#if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE) || \
-       defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC)
-       return (vdev_issue_discard_trim(zio, trim_flags));
-#elif defined(HAVE_BLKDEV_ISSUE_DISCARD)
-       return (-blkdev_issue_discard(
-           BDH_BDEV(((vdev_disk_t *)zio->io_vd->vdev_tsd)->vd_bdh),
-           zio->io_offset >> 9, zio->io_size >> 9, GFP_NOFS, trim_flags));
-#else
-#error "Unsupported kernel"
-#endif
+
+       return (0);
 }
 
 int (*vdev_disk_io_rw_fn)(zio_t *zio) = NULL;
@@ -1390,14 +1433,12 @@ vdev_disk_io_start(zio_t *zio)
                return;
 
        case ZIO_TYPE_TRIM:
-               zio->io_error = vdev_disk_io_trim(zio);
+               error = vdev_disk_io_trim(zio);
                rw_exit(&vd->vd_lock);
-#if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE)
-               if (zio->io_trim_flags & ZIO_TRIM_SECURE)
-                       zio_interrupt(zio);
-#elif defined(HAVE_BLKDEV_ISSUE_DISCARD)
-               zio_interrupt(zio);
-#endif
+               if (error) {
+                       zio->io_error = error;
+                       zio_execute(zio);
+               }
                return;
 
        case ZIO_TYPE_READ: