OpenZFS 7614, 9064 - zfs device evacuation/removal

[mirror_zfs.git] / module / zfs / vdev_queue.c

diff --git a/module/zfs/vdev_queue.c b/module/zfs/vdev_queue.c
index 36a4bf629d61cf1ea48cdde73066682c66fc1bc3..3ac31a87278a20207d276acb36e14b24b57b8c64 100644 (file)
--- a/module/zfs/vdev_queue.c
+++ b/module/zfs/vdev_queue.c
@@ -152,6 +152,8 @@ uint32_t zfs_vdev_async_write_min_active = 2;
 uint32_t zfs_vdev_async_write_max_active = 10;
 uint32_t zfs_vdev_scrub_min_active = 1;
 uint32_t zfs_vdev_scrub_max_active = 2;
+uint32_t zfs_vdev_removal_min_active = 1;
+uint32_t zfs_vdev_removal_max_active = 2;
 
 /*
  * When the pool has less than zfs_vdev_async_write_active_min_dirty_percent
@@ -169,7 +171,7 @@ int zfs_vdev_async_write_active_max_dirty_percent = 60;
  * we include spans of optional I/Os to aid aggregation at the disk even when
  * they aren't able to help us aggregate at this level.
  */
-int zfs_vdev_aggregation_limit = SPA_OLD_MAXBLOCKSIZE;
+int zfs_vdev_aggregation_limit = 1 << 20;
 int zfs_vdev_read_gap_limit = 32 << 10;
 int zfs_vdev_write_gap_limit = 4 << 10;
 
@@ -248,6 +250,8 @@ vdev_queue_class_min_active(zio_priority_t p)
                return (zfs_vdev_async_write_min_active);
        case ZIO_PRIORITY_SCRUB:
                return (zfs_vdev_scrub_min_active);
+       case ZIO_PRIORITY_REMOVAL:
+               return (zfs_vdev_removal_min_active);
        default:
                panic("invalid priority %u", p);
                return (0);
@@ -316,6 +320,8 @@ vdev_queue_class_max_active(spa_t *spa, zio_priority_t p)
                return (vdev_queue_max_async_writes(spa));
        case ZIO_PRIORITY_SCRUB:
                return (zfs_vdev_scrub_max_active);
+       case ZIO_PRIORITY_REMOVAL:
+               return (zfs_vdev_removal_max_active);
        default:
                panic("invalid priority %u", p);
                return (0);
@@ -517,6 +523,7 @@ static zio_t *
 vdev_queue_aggregate(vdev_queue_t *vq, zio_t *zio)
 {
        zio_t *first, *last, *aio, *dio, *mandatory, *nio;
+       zio_link_t *zl = NULL;
        uint64_t maxgap = 0;
        uint64_t size;
        uint64_t limit;
@@ -559,7 +566,8 @@ vdev_queue_aggregate(vdev_queue_t *vq, zio_t *zio)
        while ((dio = AVL_PREV(t, first)) != NULL &&
            (dio->io_flags & ZIO_FLAG_AGG_INHERIT) == flags &&
            IO_SPAN(dio, last) <= limit &&
-           IO_GAP(dio, first) <= maxgap) {
+           IO_GAP(dio, first) <= maxgap &&
+           dio->io_type == zio->io_type) {
                first = dio;
                if (mandatory == NULL && !(first->io_flags & ZIO_FLAG_OPTIONAL))
                        mandatory = first;
@@ -585,7 +593,8 @@ vdev_queue_aggregate(vdev_queue_t *vq, zio_t *zio)
            (IO_SPAN(first, dio) <= limit ||
            (dio->io_flags & ZIO_FLAG_OPTIONAL)) &&
            IO_SPAN(first, dio) <= maxblocksize &&
-           IO_GAP(last, dio) <= maxgap) {
+           IO_GAP(last, dio) <= maxgap &&
+           dio->io_type == zio->io_type) {
                last = dio;
                if (!(last->io_flags & ZIO_FLAG_OPTIONAL))
                        mandatory = last;
@@ -665,9 +674,18 @@ vdev_queue_aggregate(vdev_queue_t *vq, zio_t *zio)
 
                zio_add_child(dio, aio);
                vdev_queue_io_remove(vq, dio);
+       } while (dio != last);
+
+       /*
+        * We need to drop the vdev queue's lock to avoid a deadlock that we
+        * could encounter since this I/O will complete immediately.
+        */
+       mutex_exit(&vq->vq_lock);
+       while ((dio = zio_walk_parents(aio, &zl)) != NULL) {
                zio_vdev_io_bypass(dio);
                zio_execute(dio);
-       } while (dio != last);
+       }
+       mutex_enter(&vq->vq_lock);
 
        return (aio);
 }
@@ -747,12 +765,14 @@ vdev_queue_io(zio_t *zio)
        if (zio->io_type == ZIO_TYPE_READ) {
                if (zio->io_priority != ZIO_PRIORITY_SYNC_READ &&
                    zio->io_priority != ZIO_PRIORITY_ASYNC_READ &&
-                   zio->io_priority != ZIO_PRIORITY_SCRUB)
+                   zio->io_priority != ZIO_PRIORITY_SCRUB &&
+                   zio->io_priority != ZIO_PRIORITY_REMOVAL)
                        zio->io_priority = ZIO_PRIORITY_ASYNC_READ;
        } else {
                ASSERT(zio->io_type == ZIO_TYPE_WRITE);
                if (zio->io_priority != ZIO_PRIORITY_SYNC_WRITE &&
-                   zio->io_priority != ZIO_PRIORITY_ASYNC_WRITE)
+                   zio->io_priority != ZIO_PRIORITY_ASYNC_WRITE &&
+                   zio->io_priority != ZIO_PRIORITY_REMOVAL)
                        zio->io_priority = ZIO_PRIORITY_ASYNC_WRITE;
        }
 
@@ -803,6 +823,48 @@ vdev_queue_io_done(zio_t *zio)
        mutex_exit(&vq->vq_lock);
 }
 
+void
+vdev_queue_change_io_priority(zio_t *zio, zio_priority_t priority)
+{
+       vdev_queue_t *vq = &zio->io_vd->vdev_queue;
+       avl_tree_t *tree;
+
+       ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);
+       ASSERT3U(priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);
+
+       if (zio->io_type == ZIO_TYPE_READ) {
+               if (priority != ZIO_PRIORITY_SYNC_READ &&
+                   priority != ZIO_PRIORITY_ASYNC_READ &&
+                   priority != ZIO_PRIORITY_SCRUB)
+                       priority = ZIO_PRIORITY_ASYNC_READ;
+       } else {
+               ASSERT(zio->io_type == ZIO_TYPE_WRITE);
+               if (priority != ZIO_PRIORITY_SYNC_WRITE &&
+                   priority != ZIO_PRIORITY_ASYNC_WRITE)
+                       priority = ZIO_PRIORITY_ASYNC_WRITE;
+       }
+
+       mutex_enter(&vq->vq_lock);
+
+       /*
+        * If the zio is in none of the queues we can simply change
+        * the priority. If the zio is waiting to be submitted we must
+        * remove it from the queue and re-insert it with the new priority.
+        * Otherwise, the zio is currently active and we cannot change its
+        * priority.
+        */
+       tree = vdev_queue_class_tree(vq, zio->io_priority);
+       if (avl_find(tree, zio, NULL) == zio) {
+               avl_remove(vdev_queue_class_tree(vq, zio->io_priority), zio);
+               zio->io_priority = priority;
+               avl_add(vdev_queue_class_tree(vq, zio->io_priority), zio);
+       } else if (avl_find(&vq->vq_active_tree, zio, NULL) != zio) {
+               zio->io_priority = priority;
+       }
+
+       mutex_exit(&vq->vq_lock);
+}
+
 /*
  * As these two methods are only used for load calculations we're not
  * concerned if we get an incorrect value on 32bit platforms due to lack of