2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
5 * This file is released under the GPL.
10 #include "dm-uevent.h"
12 #include <linux/init.h>
13 #include <linux/module.h>
14 #include <linux/mutex.h>
15 #include <linux/sched/mm.h>
16 #include <linux/sched/signal.h>
17 #include <linux/blkpg.h>
18 #include <linux/bio.h>
19 #include <linux/mempool.h>
20 #include <linux/dax.h>
21 #include <linux/slab.h>
22 #include <linux/idr.h>
23 #include <linux/uio.h>
24 #include <linux/hdreg.h>
25 #include <linux/delay.h>
26 #include <linux/wait.h>
28 #include <linux/refcount.h>
29 #include <linux/part_stat.h>
30 #include <linux/blk-crypto.h>
31 #include <linux/keyslot-manager.h>
33 #define DM_MSG_PREFIX "core"
36 * Cookies are numeric values sent with CHANGE and REMOVE
37 * uevents while resuming, removing or renaming the device.
39 #define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE"
40 #define DM_COOKIE_LENGTH 24
42 static const char *_name
= DM_NAME
;
44 static unsigned int major
= 0;
45 static unsigned int _major
= 0;
47 static DEFINE_IDR(_minor_idr
);
49 static DEFINE_SPINLOCK(_minor_lock
);
51 static void do_deferred_remove(struct work_struct
*w
);
53 static DECLARE_WORK(deferred_remove_work
, do_deferred_remove
);
55 static struct workqueue_struct
*deferred_remove_workqueue
;
57 atomic_t dm_global_event_nr
= ATOMIC_INIT(0);
58 DECLARE_WAIT_QUEUE_HEAD(dm_global_eventq
);
60 void dm_issue_global_event(void)
62 atomic_inc(&dm_global_event_nr
);
63 wake_up(&dm_global_eventq
);
67 * One of these is allocated (on-stack) per original bio.
74 unsigned sector_count
;
78 * One of these is allocated per clone bio.
80 #define DM_TIO_MAGIC 7282014
85 unsigned target_bio_nr
;
92 * One of these is allocated per original bio.
93 * It contains the first clone used for that original.
95 #define DM_IO_MAGIC 5191977
98 struct mapped_device
*md
;
101 struct bio
*orig_bio
;
102 unsigned long start_time
;
103 spinlock_t endio_lock
;
104 struct dm_stats_aux stats_aux
;
105 /* last member of dm_target_io is 'struct bio' */
106 struct dm_target_io tio
;
109 #define DM_TARGET_IO_BIO_OFFSET (offsetof(struct dm_target_io, clone))
110 #define DM_IO_BIO_OFFSET \
111 (offsetof(struct dm_target_io, clone) + offsetof(struct dm_io, tio))
113 void *dm_per_bio_data(struct bio
*bio
, size_t data_size
)
115 struct dm_target_io
*tio
= container_of(bio
, struct dm_target_io
, clone
);
116 if (!tio
->inside_dm_io
)
117 return (char *)bio
- DM_TARGET_IO_BIO_OFFSET
- data_size
;
118 return (char *)bio
- DM_IO_BIO_OFFSET
- data_size
;
120 EXPORT_SYMBOL_GPL(dm_per_bio_data
);
122 struct bio
*dm_bio_from_per_bio_data(void *data
, size_t data_size
)
124 struct dm_io
*io
= (struct dm_io
*)((char *)data
+ data_size
);
125 if (io
->magic
== DM_IO_MAGIC
)
126 return (struct bio
*)((char *)io
+ DM_IO_BIO_OFFSET
);
127 BUG_ON(io
->magic
!= DM_TIO_MAGIC
);
128 return (struct bio
*)((char *)io
+ DM_TARGET_IO_BIO_OFFSET
);
130 EXPORT_SYMBOL_GPL(dm_bio_from_per_bio_data
);
132 unsigned dm_bio_get_target_bio_nr(const struct bio
*bio
)
134 return container_of(bio
, struct dm_target_io
, clone
)->target_bio_nr
;
136 EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr
);
138 #define MINOR_ALLOCED ((void *)-1)
141 * Bits for the md->flags field.
143 #define DMF_BLOCK_IO_FOR_SUSPEND 0
144 #define DMF_SUSPENDED 1
146 #define DMF_FREEING 3
147 #define DMF_DELETING 4
148 #define DMF_NOFLUSH_SUSPENDING 5
149 #define DMF_DEFERRED_REMOVE 6
150 #define DMF_SUSPENDED_INTERNALLY 7
151 #define DMF_POST_SUSPENDING 8
153 #define DM_NUMA_NODE NUMA_NO_NODE
154 static int dm_numa_node
= DM_NUMA_NODE
;
156 #define DEFAULT_SWAP_BIOS (8 * 1048576 / PAGE_SIZE)
157 static int swap_bios
= DEFAULT_SWAP_BIOS
;
158 static int get_swap_bios(void)
160 int latch
= READ_ONCE(swap_bios
);
161 if (unlikely(latch
<= 0))
162 latch
= DEFAULT_SWAP_BIOS
;
167 * For mempools pre-allocation at the table loading time.
169 struct dm_md_mempools
{
171 struct bio_set io_bs
;
174 struct table_device
{
175 struct list_head list
;
177 struct dm_dev dm_dev
;
181 * Bio-based DM's mempools' reserved IOs set by the user.
183 #define RESERVED_BIO_BASED_IOS 16
184 static unsigned reserved_bio_based_ios
= RESERVED_BIO_BASED_IOS
;
186 static int __dm_get_module_param_int(int *module_param
, int min
, int max
)
188 int param
= READ_ONCE(*module_param
);
189 int modified_param
= 0;
190 bool modified
= true;
193 modified_param
= min
;
194 else if (param
> max
)
195 modified_param
= max
;
200 (void)cmpxchg(module_param
, param
, modified_param
);
201 param
= modified_param
;
207 unsigned __dm_get_module_param(unsigned *module_param
,
208 unsigned def
, unsigned max
)
210 unsigned param
= READ_ONCE(*module_param
);
211 unsigned modified_param
= 0;
214 modified_param
= def
;
215 else if (param
> max
)
216 modified_param
= max
;
218 if (modified_param
) {
219 (void)cmpxchg(module_param
, param
, modified_param
);
220 param
= modified_param
;
226 unsigned dm_get_reserved_bio_based_ios(void)
228 return __dm_get_module_param(&reserved_bio_based_ios
,
229 RESERVED_BIO_BASED_IOS
, DM_RESERVED_MAX_IOS
);
231 EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios
);
233 static unsigned dm_get_numa_node(void)
235 return __dm_get_module_param_int(&dm_numa_node
,
236 DM_NUMA_NODE
, num_online_nodes() - 1);
239 static int __init
local_init(void)
243 r
= dm_uevent_init();
247 deferred_remove_workqueue
= alloc_workqueue("kdmremove", WQ_UNBOUND
, 1);
248 if (!deferred_remove_workqueue
) {
250 goto out_uevent_exit
;
254 r
= register_blkdev(_major
, _name
);
256 goto out_free_workqueue
;
264 destroy_workqueue(deferred_remove_workqueue
);
271 static void local_exit(void)
273 flush_scheduled_work();
274 destroy_workqueue(deferred_remove_workqueue
);
276 unregister_blkdev(_major
, _name
);
281 DMINFO("cleaned up");
284 static int (*_inits
[])(void) __initdata
= {
295 static void (*_exits
[])(void) = {
306 static int __init
dm_init(void)
308 const int count
= ARRAY_SIZE(_inits
);
312 for (i
= 0; i
< count
; i
++) {
327 static void __exit
dm_exit(void)
329 int i
= ARRAY_SIZE(_exits
);
335 * Should be empty by this point.
337 idr_destroy(&_minor_idr
);
341 * Block device functions
343 int dm_deleting_md(struct mapped_device
*md
)
345 return test_bit(DMF_DELETING
, &md
->flags
);
348 static int dm_blk_open(struct block_device
*bdev
, fmode_t mode
)
350 struct mapped_device
*md
;
352 spin_lock(&_minor_lock
);
354 md
= bdev
->bd_disk
->private_data
;
358 if (test_bit(DMF_FREEING
, &md
->flags
) ||
359 dm_deleting_md(md
)) {
365 atomic_inc(&md
->open_count
);
367 spin_unlock(&_minor_lock
);
369 return md
? 0 : -ENXIO
;
372 static void dm_blk_close(struct gendisk
*disk
, fmode_t mode
)
374 struct mapped_device
*md
;
376 spin_lock(&_minor_lock
);
378 md
= disk
->private_data
;
382 if (atomic_dec_and_test(&md
->open_count
) &&
383 (test_bit(DMF_DEFERRED_REMOVE
, &md
->flags
)))
384 queue_work(deferred_remove_workqueue
, &deferred_remove_work
);
388 spin_unlock(&_minor_lock
);
391 int dm_open_count(struct mapped_device
*md
)
393 return atomic_read(&md
->open_count
);
397 * Guarantees nothing is using the device before it's deleted.
399 int dm_lock_for_deletion(struct mapped_device
*md
, bool mark_deferred
, bool only_deferred
)
403 spin_lock(&_minor_lock
);
405 if (dm_open_count(md
)) {
408 set_bit(DMF_DEFERRED_REMOVE
, &md
->flags
);
409 } else if (only_deferred
&& !test_bit(DMF_DEFERRED_REMOVE
, &md
->flags
))
412 set_bit(DMF_DELETING
, &md
->flags
);
414 spin_unlock(&_minor_lock
);
419 int dm_cancel_deferred_remove(struct mapped_device
*md
)
423 spin_lock(&_minor_lock
);
425 if (test_bit(DMF_DELETING
, &md
->flags
))
428 clear_bit(DMF_DEFERRED_REMOVE
, &md
->flags
);
430 spin_unlock(&_minor_lock
);
435 static void do_deferred_remove(struct work_struct
*w
)
437 dm_deferred_remove();
440 static int dm_blk_getgeo(struct block_device
*bdev
, struct hd_geometry
*geo
)
442 struct mapped_device
*md
= bdev
->bd_disk
->private_data
;
444 return dm_get_geometry(md
, geo
);
447 #ifdef CONFIG_BLK_DEV_ZONED
448 int dm_report_zones_cb(struct blk_zone
*zone
, unsigned int idx
, void *data
)
450 struct dm_report_zones_args
*args
= data
;
451 sector_t sector_diff
= args
->tgt
->begin
- args
->start
;
454 * Ignore zones beyond the target range.
456 if (zone
->start
>= args
->start
+ args
->tgt
->len
)
460 * Remap the start sector and write pointer position of the zone
461 * to match its position in the target range.
463 zone
->start
+= sector_diff
;
464 if (zone
->type
!= BLK_ZONE_TYPE_CONVENTIONAL
) {
465 if (zone
->cond
== BLK_ZONE_COND_FULL
)
466 zone
->wp
= zone
->start
+ zone
->len
;
467 else if (zone
->cond
== BLK_ZONE_COND_EMPTY
)
468 zone
->wp
= zone
->start
;
470 zone
->wp
+= sector_diff
;
473 args
->next_sector
= zone
->start
+ zone
->len
;
474 return args
->orig_cb(zone
, args
->zone_idx
++, args
->orig_data
);
476 EXPORT_SYMBOL_GPL(dm_report_zones_cb
);
478 static int dm_blk_report_zones(struct gendisk
*disk
, sector_t sector
,
479 unsigned int nr_zones
, report_zones_cb cb
, void *data
)
481 struct mapped_device
*md
= disk
->private_data
;
482 struct dm_table
*map
;
484 struct dm_report_zones_args args
= {
485 .next_sector
= sector
,
490 if (dm_suspended_md(md
))
493 map
= dm_get_live_table(md
, &srcu_idx
);
500 struct dm_target
*tgt
;
502 tgt
= dm_table_find_target(map
, args
.next_sector
);
503 if (WARN_ON_ONCE(!tgt
->type
->report_zones
)) {
509 ret
= tgt
->type
->report_zones(tgt
, &args
,
510 nr_zones
- args
.zone_idx
);
513 } while (args
.zone_idx
< nr_zones
&&
514 args
.next_sector
< get_capacity(disk
));
518 dm_put_live_table(md
, srcu_idx
);
522 #define dm_blk_report_zones NULL
523 #endif /* CONFIG_BLK_DEV_ZONED */
525 static int dm_prepare_ioctl(struct mapped_device
*md
, int *srcu_idx
,
526 struct block_device
**bdev
)
528 struct dm_target
*tgt
;
529 struct dm_table
*map
;
534 map
= dm_get_live_table(md
, srcu_idx
);
535 if (!map
|| !dm_table_get_size(map
))
538 /* We only support devices that have a single target */
539 if (dm_table_get_num_targets(map
) != 1)
542 tgt
= dm_table_get_target(map
, 0);
543 if (!tgt
->type
->prepare_ioctl
)
546 if (dm_suspended_md(md
))
549 r
= tgt
->type
->prepare_ioctl(tgt
, bdev
);
550 if (r
== -ENOTCONN
&& !fatal_signal_pending(current
)) {
551 dm_put_live_table(md
, *srcu_idx
);
559 static void dm_unprepare_ioctl(struct mapped_device
*md
, int srcu_idx
)
561 dm_put_live_table(md
, srcu_idx
);
564 static int dm_blk_ioctl(struct block_device
*bdev
, fmode_t mode
,
565 unsigned int cmd
, unsigned long arg
)
567 struct mapped_device
*md
= bdev
->bd_disk
->private_data
;
570 r
= dm_prepare_ioctl(md
, &srcu_idx
, &bdev
);
576 * Target determined this ioctl is being issued against a
577 * subset of the parent bdev; require extra privileges.
579 if (!capable(CAP_SYS_RAWIO
)) {
581 "%s: sending ioctl %x to DM device without required privilege.",
588 if (!bdev
->bd_disk
->fops
->ioctl
)
591 r
= bdev
->bd_disk
->fops
->ioctl(bdev
, mode
, cmd
, arg
);
593 dm_unprepare_ioctl(md
, srcu_idx
);
597 u64
dm_start_time_ns_from_clone(struct bio
*bio
)
599 struct dm_target_io
*tio
= container_of(bio
, struct dm_target_io
, clone
);
600 struct dm_io
*io
= tio
->io
;
602 return jiffies_to_nsecs(io
->start_time
);
604 EXPORT_SYMBOL_GPL(dm_start_time_ns_from_clone
);
606 static void start_io_acct(struct dm_io
*io
)
608 struct mapped_device
*md
= io
->md
;
609 struct bio
*bio
= io
->orig_bio
;
611 io
->start_time
= bio_start_io_acct(bio
);
612 if (unlikely(dm_stats_used(&md
->stats
)))
613 dm_stats_account_io(&md
->stats
, bio_data_dir(bio
),
614 bio
->bi_iter
.bi_sector
, bio_sectors(bio
),
615 false, 0, &io
->stats_aux
);
618 static void end_io_acct(struct dm_io
*io
)
620 struct mapped_device
*md
= io
->md
;
621 struct bio
*bio
= io
->orig_bio
;
622 unsigned long duration
= jiffies
- io
->start_time
;
624 bio_end_io_acct(bio
, io
->start_time
);
626 if (unlikely(dm_stats_used(&md
->stats
)))
627 dm_stats_account_io(&md
->stats
, bio_data_dir(bio
),
628 bio
->bi_iter
.bi_sector
, bio_sectors(bio
),
629 true, duration
, &io
->stats_aux
);
631 /* nudge anyone waiting on suspend queue */
632 if (unlikely(wq_has_sleeper(&md
->wait
)))
636 static struct dm_io
*alloc_io(struct mapped_device
*md
, struct bio
*bio
)
639 struct dm_target_io
*tio
;
642 clone
= bio_alloc_bioset(GFP_NOIO
, 0, &md
->io_bs
);
646 tio
= container_of(clone
, struct dm_target_io
, clone
);
647 tio
->inside_dm_io
= true;
650 io
= container_of(tio
, struct dm_io
, tio
);
651 io
->magic
= DM_IO_MAGIC
;
653 atomic_set(&io
->io_count
, 1);
656 spin_lock_init(&io
->endio_lock
);
663 static void free_io(struct mapped_device
*md
, struct dm_io
*io
)
665 bio_put(&io
->tio
.clone
);
668 static struct dm_target_io
*alloc_tio(struct clone_info
*ci
, struct dm_target
*ti
,
669 unsigned target_bio_nr
, gfp_t gfp_mask
)
671 struct dm_target_io
*tio
;
673 if (!ci
->io
->tio
.io
) {
674 /* the dm_target_io embedded in ci->io is available */
677 struct bio
*clone
= bio_alloc_bioset(gfp_mask
, 0, &ci
->io
->md
->bs
);
681 tio
= container_of(clone
, struct dm_target_io
, clone
);
682 tio
->inside_dm_io
= false;
685 tio
->magic
= DM_TIO_MAGIC
;
688 tio
->target_bio_nr
= target_bio_nr
;
693 static void free_tio(struct dm_target_io
*tio
)
695 if (tio
->inside_dm_io
)
697 bio_put(&tio
->clone
);
701 * Add the bio to the list of deferred io.
703 static void queue_io(struct mapped_device
*md
, struct bio
*bio
)
707 spin_lock_irqsave(&md
->deferred_lock
, flags
);
708 bio_list_add(&md
->deferred
, bio
);
709 spin_unlock_irqrestore(&md
->deferred_lock
, flags
);
710 queue_work(md
->wq
, &md
->work
);
714 * Everyone (including functions in this file), should use this
715 * function to access the md->map field, and make sure they call
716 * dm_put_live_table() when finished.
718 struct dm_table
*dm_get_live_table(struct mapped_device
*md
, int *srcu_idx
) __acquires(md
->io_barrier
)
720 *srcu_idx
= srcu_read_lock(&md
->io_barrier
);
722 return srcu_dereference(md
->map
, &md
->io_barrier
);
725 void dm_put_live_table(struct mapped_device
*md
, int srcu_idx
) __releases(md
->io_barrier
)
727 srcu_read_unlock(&md
->io_barrier
, srcu_idx
);
730 void dm_sync_table(struct mapped_device
*md
)
732 synchronize_srcu(&md
->io_barrier
);
733 synchronize_rcu_expedited();
737 * A fast alternative to dm_get_live_table/dm_put_live_table.
738 * The caller must not block between these two functions.
740 static struct dm_table
*dm_get_live_table_fast(struct mapped_device
*md
) __acquires(RCU
)
743 return rcu_dereference(md
->map
);
746 static void dm_put_live_table_fast(struct mapped_device
*md
) __releases(RCU
)
751 static char *_dm_claim_ptr
= "I belong to device-mapper";
754 * Open a table device so we can use it as a map destination.
756 static int open_table_device(struct table_device
*td
, dev_t dev
,
757 struct mapped_device
*md
)
759 struct block_device
*bdev
;
763 BUG_ON(td
->dm_dev
.bdev
);
765 bdev
= blkdev_get_by_dev(dev
, td
->dm_dev
.mode
| FMODE_EXCL
, _dm_claim_ptr
);
767 return PTR_ERR(bdev
);
769 r
= bd_link_disk_holder(bdev
, dm_disk(md
));
771 blkdev_put(bdev
, td
->dm_dev
.mode
| FMODE_EXCL
);
775 td
->dm_dev
.bdev
= bdev
;
776 td
->dm_dev
.dax_dev
= dax_get_by_host(bdev
->bd_disk
->disk_name
);
781 * Close a table device that we've been using.
783 static void close_table_device(struct table_device
*td
, struct mapped_device
*md
)
785 if (!td
->dm_dev
.bdev
)
788 bd_unlink_disk_holder(td
->dm_dev
.bdev
, dm_disk(md
));
789 blkdev_put(td
->dm_dev
.bdev
, td
->dm_dev
.mode
| FMODE_EXCL
);
790 put_dax(td
->dm_dev
.dax_dev
);
791 td
->dm_dev
.bdev
= NULL
;
792 td
->dm_dev
.dax_dev
= NULL
;
795 static struct table_device
*find_table_device(struct list_head
*l
, dev_t dev
,
798 struct table_device
*td
;
800 list_for_each_entry(td
, l
, list
)
801 if (td
->dm_dev
.bdev
->bd_dev
== dev
&& td
->dm_dev
.mode
== mode
)
807 int dm_get_table_device(struct mapped_device
*md
, dev_t dev
, fmode_t mode
,
808 struct dm_dev
**result
)
811 struct table_device
*td
;
813 mutex_lock(&md
->table_devices_lock
);
814 td
= find_table_device(&md
->table_devices
, dev
, mode
);
816 td
= kmalloc_node(sizeof(*td
), GFP_KERNEL
, md
->numa_node_id
);
818 mutex_unlock(&md
->table_devices_lock
);
822 td
->dm_dev
.mode
= mode
;
823 td
->dm_dev
.bdev
= NULL
;
825 if ((r
= open_table_device(td
, dev
, md
))) {
826 mutex_unlock(&md
->table_devices_lock
);
831 format_dev_t(td
->dm_dev
.name
, dev
);
833 refcount_set(&td
->count
, 1);
834 list_add(&td
->list
, &md
->table_devices
);
836 refcount_inc(&td
->count
);
838 mutex_unlock(&md
->table_devices_lock
);
840 *result
= &td
->dm_dev
;
843 EXPORT_SYMBOL_GPL(dm_get_table_device
);
845 void dm_put_table_device(struct mapped_device
*md
, struct dm_dev
*d
)
847 struct table_device
*td
= container_of(d
, struct table_device
, dm_dev
);
849 mutex_lock(&md
->table_devices_lock
);
850 if (refcount_dec_and_test(&td
->count
)) {
851 close_table_device(td
, md
);
855 mutex_unlock(&md
->table_devices_lock
);
857 EXPORT_SYMBOL(dm_put_table_device
);
859 static void free_table_devices(struct list_head
*devices
)
861 struct list_head
*tmp
, *next
;
863 list_for_each_safe(tmp
, next
, devices
) {
864 struct table_device
*td
= list_entry(tmp
, struct table_device
, list
);
866 DMWARN("dm_destroy: %s still exists with %d references",
867 td
->dm_dev
.name
, refcount_read(&td
->count
));
873 * Get the geometry associated with a dm device
875 int dm_get_geometry(struct mapped_device
*md
, struct hd_geometry
*geo
)
883 * Set the geometry of a device.
885 int dm_set_geometry(struct mapped_device
*md
, struct hd_geometry
*geo
)
887 sector_t sz
= (sector_t
)geo
->cylinders
* geo
->heads
* geo
->sectors
;
889 if (geo
->start
> sz
) {
890 DMWARN("Start sector is beyond the geometry limits.");
899 static int __noflush_suspending(struct mapped_device
*md
)
901 return test_bit(DMF_NOFLUSH_SUSPENDING
, &md
->flags
);
905 * Decrements the number of outstanding ios that a bio has been
906 * cloned into, completing the original io if necc.
908 static void dec_pending(struct dm_io
*io
, blk_status_t error
)
911 blk_status_t io_error
;
913 struct mapped_device
*md
= io
->md
;
915 /* Push-back supersedes any I/O errors */
916 if (unlikely(error
)) {
917 spin_lock_irqsave(&io
->endio_lock
, flags
);
918 if (!(io
->status
== BLK_STS_DM_REQUEUE
&& __noflush_suspending(md
)))
920 spin_unlock_irqrestore(&io
->endio_lock
, flags
);
923 if (atomic_dec_and_test(&io
->io_count
)) {
924 if (io
->status
== BLK_STS_DM_REQUEUE
) {
926 * Target requested pushing back the I/O.
928 spin_lock_irqsave(&md
->deferred_lock
, flags
);
929 if (__noflush_suspending(md
))
930 /* NOTE early return due to BLK_STS_DM_REQUEUE below */
931 bio_list_add_head(&md
->deferred
, io
->orig_bio
);
933 /* noflush suspend was interrupted. */
934 io
->status
= BLK_STS_IOERR
;
935 spin_unlock_irqrestore(&md
->deferred_lock
, flags
);
938 io_error
= io
->status
;
943 if (io_error
== BLK_STS_DM_REQUEUE
)
946 if ((bio
->bi_opf
& REQ_PREFLUSH
) && bio
->bi_iter
.bi_size
) {
948 * Preflush done for flush with data, reissue
949 * without REQ_PREFLUSH.
951 bio
->bi_opf
&= ~REQ_PREFLUSH
;
954 /* done with normal IO or empty flush */
956 bio
->bi_status
= io_error
;
962 void disable_discard(struct mapped_device
*md
)
964 struct queue_limits
*limits
= dm_get_queue_limits(md
);
966 /* device doesn't really support DISCARD, disable it */
967 limits
->max_discard_sectors
= 0;
968 blk_queue_flag_clear(QUEUE_FLAG_DISCARD
, md
->queue
);
971 void disable_write_same(struct mapped_device
*md
)
973 struct queue_limits
*limits
= dm_get_queue_limits(md
);
975 /* device doesn't really support WRITE SAME, disable it */
976 limits
->max_write_same_sectors
= 0;
979 void disable_write_zeroes(struct mapped_device
*md
)
981 struct queue_limits
*limits
= dm_get_queue_limits(md
);
983 /* device doesn't really support WRITE ZEROES, disable it */
984 limits
->max_write_zeroes_sectors
= 0;
987 static bool swap_bios_limit(struct dm_target
*ti
, struct bio
*bio
)
989 return unlikely((bio
->bi_opf
& REQ_SWAP
) != 0) && unlikely(ti
->limit_swap_bios
);
992 static void clone_endio(struct bio
*bio
)
994 blk_status_t error
= bio
->bi_status
;
995 struct dm_target_io
*tio
= container_of(bio
, struct dm_target_io
, clone
);
996 struct dm_io
*io
= tio
->io
;
997 struct mapped_device
*md
= tio
->io
->md
;
998 dm_endio_fn endio
= tio
->ti
->type
->end_io
;
999 struct bio
*orig_bio
= io
->orig_bio
;
1000 struct request_queue
*q
= bio
->bi_bdev
->bd_disk
->queue
;
1002 if (unlikely(error
== BLK_STS_TARGET
)) {
1003 if (bio_op(bio
) == REQ_OP_DISCARD
&&
1004 !q
->limits
.max_discard_sectors
)
1005 disable_discard(md
);
1006 else if (bio_op(bio
) == REQ_OP_WRITE_SAME
&&
1007 !q
->limits
.max_write_same_sectors
)
1008 disable_write_same(md
);
1009 else if (bio_op(bio
) == REQ_OP_WRITE_ZEROES
&&
1010 !q
->limits
.max_write_zeroes_sectors
)
1011 disable_write_zeroes(md
);
1015 * For zone-append bios get offset in zone of the written
1016 * sector and add that to the original bio sector pos.
1018 if (bio_op(orig_bio
) == REQ_OP_ZONE_APPEND
) {
1019 sector_t written_sector
= bio
->bi_iter
.bi_sector
;
1020 struct request_queue
*q
= orig_bio
->bi_bdev
->bd_disk
->queue
;
1021 u64 mask
= (u64
)blk_queue_zone_sectors(q
) - 1;
1023 orig_bio
->bi_iter
.bi_sector
+= written_sector
& mask
;
1027 int r
= endio(tio
->ti
, bio
, &error
);
1029 case DM_ENDIO_REQUEUE
:
1030 error
= BLK_STS_DM_REQUEUE
;
1034 case DM_ENDIO_INCOMPLETE
:
1035 /* The target will handle the io */
1038 DMWARN("unimplemented target endio return value: %d", r
);
1043 if (unlikely(swap_bios_limit(tio
->ti
, bio
))) {
1044 struct mapped_device
*md
= io
->md
;
1045 up(&md
->swap_bios_semaphore
);
1049 dec_pending(io
, error
);
1053 * Return maximum size of I/O possible at the supplied sector up to the current
1056 static inline sector_t
max_io_len_target_boundary(struct dm_target
*ti
,
1057 sector_t target_offset
)
1059 return ti
->len
- target_offset
;
1062 static sector_t
max_io_len(struct dm_target
*ti
, sector_t sector
)
1064 sector_t target_offset
= dm_target_offset(ti
, sector
);
1065 sector_t len
= max_io_len_target_boundary(ti
, target_offset
);
1069 * Does the target need to split IO even further?
1070 * - varied (per target) IO splitting is a tenet of DM; this
1071 * explains why stacked chunk_sectors based splitting via
1072 * blk_max_size_offset() isn't possible here. So pass in
1073 * ti->max_io_len to override stacked chunk_sectors.
1075 if (ti
->max_io_len
) {
1076 max_len
= blk_max_size_offset(ti
->table
->md
->queue
,
1077 target_offset
, ti
->max_io_len
);
1085 int dm_set_target_max_io_len(struct dm_target
*ti
, sector_t len
)
1087 if (len
> UINT_MAX
) {
1088 DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)",
1089 (unsigned long long)len
, UINT_MAX
);
1090 ti
->error
= "Maximum size of target IO is too large";
1094 ti
->max_io_len
= (uint32_t) len
;
1098 EXPORT_SYMBOL_GPL(dm_set_target_max_io_len
);
1100 static struct dm_target
*dm_dax_get_live_target(struct mapped_device
*md
,
1101 sector_t sector
, int *srcu_idx
)
1102 __acquires(md
->io_barrier
)
1104 struct dm_table
*map
;
1105 struct dm_target
*ti
;
1107 map
= dm_get_live_table(md
, srcu_idx
);
1111 ti
= dm_table_find_target(map
, sector
);
1118 static long dm_dax_direct_access(struct dax_device
*dax_dev
, pgoff_t pgoff
,
1119 long nr_pages
, void **kaddr
, pfn_t
*pfn
)
1121 struct mapped_device
*md
= dax_get_private(dax_dev
);
1122 sector_t sector
= pgoff
* PAGE_SECTORS
;
1123 struct dm_target
*ti
;
1124 long len
, ret
= -EIO
;
1127 ti
= dm_dax_get_live_target(md
, sector
, &srcu_idx
);
1131 if (!ti
->type
->direct_access
)
1133 len
= max_io_len(ti
, sector
) / PAGE_SECTORS
;
1136 nr_pages
= min(len
, nr_pages
);
1137 ret
= ti
->type
->direct_access(ti
, pgoff
, nr_pages
, kaddr
, pfn
);
1140 dm_put_live_table(md
, srcu_idx
);
1145 static bool dm_dax_supported(struct dax_device
*dax_dev
, struct block_device
*bdev
,
1146 int blocksize
, sector_t start
, sector_t len
)
1148 struct mapped_device
*md
= dax_get_private(dax_dev
);
1149 struct dm_table
*map
;
1153 map
= dm_get_live_table(md
, &srcu_idx
);
1157 ret
= dm_table_supports_dax(map
, device_not_dax_capable
, &blocksize
);
1160 dm_put_live_table(md
, srcu_idx
);
1165 static size_t dm_dax_copy_from_iter(struct dax_device
*dax_dev
, pgoff_t pgoff
,
1166 void *addr
, size_t bytes
, struct iov_iter
*i
)
1168 struct mapped_device
*md
= dax_get_private(dax_dev
);
1169 sector_t sector
= pgoff
* PAGE_SECTORS
;
1170 struct dm_target
*ti
;
1174 ti
= dm_dax_get_live_target(md
, sector
, &srcu_idx
);
1178 if (!ti
->type
->dax_copy_from_iter
) {
1179 ret
= copy_from_iter(addr
, bytes
, i
);
1182 ret
= ti
->type
->dax_copy_from_iter(ti
, pgoff
, addr
, bytes
, i
);
1184 dm_put_live_table(md
, srcu_idx
);
1189 static size_t dm_dax_copy_to_iter(struct dax_device
*dax_dev
, pgoff_t pgoff
,
1190 void *addr
, size_t bytes
, struct iov_iter
*i
)
1192 struct mapped_device
*md
= dax_get_private(dax_dev
);
1193 sector_t sector
= pgoff
* PAGE_SECTORS
;
1194 struct dm_target
*ti
;
1198 ti
= dm_dax_get_live_target(md
, sector
, &srcu_idx
);
1202 if (!ti
->type
->dax_copy_to_iter
) {
1203 ret
= copy_to_iter(addr
, bytes
, i
);
1206 ret
= ti
->type
->dax_copy_to_iter(ti
, pgoff
, addr
, bytes
, i
);
1208 dm_put_live_table(md
, srcu_idx
);
1213 static int dm_dax_zero_page_range(struct dax_device
*dax_dev
, pgoff_t pgoff
,
1216 struct mapped_device
*md
= dax_get_private(dax_dev
);
1217 sector_t sector
= pgoff
* PAGE_SECTORS
;
1218 struct dm_target
*ti
;
1222 ti
= dm_dax_get_live_target(md
, sector
, &srcu_idx
);
1226 if (WARN_ON(!ti
->type
->dax_zero_page_range
)) {
1228 * ->zero_page_range() is mandatory dax operation. If we are
1229 * here, something is wrong.
1233 ret
= ti
->type
->dax_zero_page_range(ti
, pgoff
, nr_pages
);
1235 dm_put_live_table(md
, srcu_idx
);
1241 * A target may call dm_accept_partial_bio only from the map routine. It is
1242 * allowed for all bio types except REQ_PREFLUSH, REQ_OP_ZONE_RESET,
1243 * REQ_OP_ZONE_OPEN, REQ_OP_ZONE_CLOSE and REQ_OP_ZONE_FINISH.
1245 * dm_accept_partial_bio informs the dm that the target only wants to process
1246 * additional n_sectors sectors of the bio and the rest of the data should be
1247 * sent in a next bio.
1249 * A diagram that explains the arithmetics:
1250 * +--------------------+---------------+-------+
1252 * +--------------------+---------------+-------+
1254 * <-------------- *tio->len_ptr --------------->
1255 * <------- bi_size ------->
1258 * Region 1 was already iterated over with bio_advance or similar function.
1259 * (it may be empty if the target doesn't use bio_advance)
1260 * Region 2 is the remaining bio size that the target wants to process.
1261 * (it may be empty if region 1 is non-empty, although there is no reason
1263 * The target requires that region 3 is to be sent in the next bio.
1265 * If the target wants to receive multiple copies of the bio (via num_*bios, etc),
1266 * the partially processed part (the sum of regions 1+2) must be the same for all
1267 * copies of the bio.
1269 void dm_accept_partial_bio(struct bio
*bio
, unsigned n_sectors
)
1271 struct dm_target_io
*tio
= container_of(bio
, struct dm_target_io
, clone
);
1272 unsigned bi_size
= bio
->bi_iter
.bi_size
>> SECTOR_SHIFT
;
1273 BUG_ON(bio
->bi_opf
& REQ_PREFLUSH
);
1274 BUG_ON(bi_size
> *tio
->len_ptr
);
1275 BUG_ON(n_sectors
> bi_size
);
1276 *tio
->len_ptr
-= bi_size
- n_sectors
;
1277 bio
->bi_iter
.bi_size
= n_sectors
<< SECTOR_SHIFT
;
1279 EXPORT_SYMBOL_GPL(dm_accept_partial_bio
);
1281 static noinline
void __set_swap_bios_limit(struct mapped_device
*md
, int latch
)
1283 mutex_lock(&md
->swap_bios_lock
);
1284 while (latch
< md
->swap_bios
) {
1286 down(&md
->swap_bios_semaphore
);
1289 while (latch
> md
->swap_bios
) {
1291 up(&md
->swap_bios_semaphore
);
1294 mutex_unlock(&md
->swap_bios_lock
);
1297 static blk_qc_t
__map_bio(struct dm_target_io
*tio
)
1301 struct bio
*clone
= &tio
->clone
;
1302 struct dm_io
*io
= tio
->io
;
1303 struct dm_target
*ti
= tio
->ti
;
1304 blk_qc_t ret
= BLK_QC_T_NONE
;
1306 clone
->bi_end_io
= clone_endio
;
1309 * Map the clone. If r == 0 we don't need to do
1310 * anything, the target has assumed ownership of
1313 atomic_inc(&io
->io_count
);
1314 sector
= clone
->bi_iter
.bi_sector
;
1316 if (unlikely(swap_bios_limit(ti
, clone
))) {
1317 struct mapped_device
*md
= io
->md
;
1318 int latch
= get_swap_bios();
1319 if (unlikely(latch
!= md
->swap_bios
))
1320 __set_swap_bios_limit(md
, latch
);
1321 down(&md
->swap_bios_semaphore
);
1324 r
= ti
->type
->map(ti
, clone
);
1326 case DM_MAPIO_SUBMITTED
:
1328 case DM_MAPIO_REMAPPED
:
1329 /* the bio has been remapped so dispatch it */
1330 trace_block_bio_remap(clone
, bio_dev(io
->orig_bio
), sector
);
1331 ret
= submit_bio_noacct(clone
);
1334 if (unlikely(swap_bios_limit(ti
, clone
))) {
1335 struct mapped_device
*md
= io
->md
;
1336 up(&md
->swap_bios_semaphore
);
1339 dec_pending(io
, BLK_STS_IOERR
);
1341 case DM_MAPIO_REQUEUE
:
1342 if (unlikely(swap_bios_limit(ti
, clone
))) {
1343 struct mapped_device
*md
= io
->md
;
1344 up(&md
->swap_bios_semaphore
);
1347 dec_pending(io
, BLK_STS_DM_REQUEUE
);
1350 DMWARN("unimplemented target map return value: %d", r
);
1357 static void bio_setup_sector(struct bio
*bio
, sector_t sector
, unsigned len
)
1359 bio
->bi_iter
.bi_sector
= sector
;
1360 bio
->bi_iter
.bi_size
= to_bytes(len
);
1364 * Creates a bio that consists of range of complete bvecs.
1366 static int clone_bio(struct dm_target_io
*tio
, struct bio
*bio
,
1367 sector_t sector
, unsigned len
)
1369 struct bio
*clone
= &tio
->clone
;
1372 __bio_clone_fast(clone
, bio
);
1374 r
= bio_crypt_clone(clone
, bio
, GFP_NOIO
);
1378 if (bio_integrity(bio
)) {
1379 if (unlikely(!dm_target_has_integrity(tio
->ti
->type
) &&
1380 !dm_target_passes_integrity(tio
->ti
->type
))) {
1381 DMWARN("%s: the target %s doesn't support integrity data.",
1382 dm_device_name(tio
->io
->md
),
1383 tio
->ti
->type
->name
);
1387 r
= bio_integrity_clone(clone
, bio
, GFP_NOIO
);
1392 bio_advance(clone
, to_bytes(sector
- clone
->bi_iter
.bi_sector
));
1393 clone
->bi_iter
.bi_size
= to_bytes(len
);
1395 if (bio_integrity(bio
))
1396 bio_integrity_trim(clone
);
1401 static void alloc_multiple_bios(struct bio_list
*blist
, struct clone_info
*ci
,
1402 struct dm_target
*ti
, unsigned num_bios
)
1404 struct dm_target_io
*tio
;
1410 if (num_bios
== 1) {
1411 tio
= alloc_tio(ci
, ti
, 0, GFP_NOIO
);
1412 bio_list_add(blist
, &tio
->clone
);
1416 for (try = 0; try < 2; try++) {
1421 mutex_lock(&ci
->io
->md
->table_devices_lock
);
1422 for (bio_nr
= 0; bio_nr
< num_bios
; bio_nr
++) {
1423 tio
= alloc_tio(ci
, ti
, bio_nr
, try ? GFP_NOIO
: GFP_NOWAIT
);
1427 bio_list_add(blist
, &tio
->clone
);
1430 mutex_unlock(&ci
->io
->md
->table_devices_lock
);
1431 if (bio_nr
== num_bios
)
1434 while ((bio
= bio_list_pop(blist
))) {
1435 tio
= container_of(bio
, struct dm_target_io
, clone
);
1441 static blk_qc_t
__clone_and_map_simple_bio(struct clone_info
*ci
,
1442 struct dm_target_io
*tio
, unsigned *len
)
1444 struct bio
*clone
= &tio
->clone
;
1448 __bio_clone_fast(clone
, ci
->bio
);
1450 bio_setup_sector(clone
, ci
->sector
, *len
);
1452 return __map_bio(tio
);
1455 static void __send_duplicate_bios(struct clone_info
*ci
, struct dm_target
*ti
,
1456 unsigned num_bios
, unsigned *len
)
1458 struct bio_list blist
= BIO_EMPTY_LIST
;
1460 struct dm_target_io
*tio
;
1462 alloc_multiple_bios(&blist
, ci
, ti
, num_bios
);
1464 while ((bio
= bio_list_pop(&blist
))) {
1465 tio
= container_of(bio
, struct dm_target_io
, clone
);
1466 (void) __clone_and_map_simple_bio(ci
, tio
, len
);
1470 static int __send_empty_flush(struct clone_info
*ci
)
1472 unsigned target_nr
= 0;
1473 struct dm_target
*ti
;
1474 struct bio flush_bio
;
1477 * Use an on-stack bio for this, it's safe since we don't
1478 * need to reference it after submit. It's just used as
1479 * the basis for the clone(s).
1481 bio_init(&flush_bio
, NULL
, 0);
1482 flush_bio
.bi_opf
= REQ_OP_WRITE
| REQ_PREFLUSH
| REQ_SYNC
;
1483 bio_set_dev(&flush_bio
, ci
->io
->md
->disk
->part0
);
1485 ci
->bio
= &flush_bio
;
1486 ci
->sector_count
= 0;
1488 BUG_ON(bio_has_data(ci
->bio
));
1489 while ((ti
= dm_table_get_target(ci
->map
, target_nr
++)))
1490 __send_duplicate_bios(ci
, ti
, ti
->num_flush_bios
, NULL
);
1492 bio_uninit(ci
->bio
);
1496 static int __clone_and_map_data_bio(struct clone_info
*ci
, struct dm_target
*ti
,
1497 sector_t sector
, unsigned *len
)
1499 struct bio
*bio
= ci
->bio
;
1500 struct dm_target_io
*tio
;
1503 tio
= alloc_tio(ci
, ti
, 0, GFP_NOIO
);
1505 r
= clone_bio(tio
, bio
, sector
, *len
);
1510 (void) __map_bio(tio
);
1515 static int __send_changing_extent_only(struct clone_info
*ci
, struct dm_target
*ti
,
1521 * Even though the device advertised support for this type of
1522 * request, that does not mean every target supports it, and
1523 * reconfiguration might also have changed that since the
1524 * check was performed.
1529 len
= min_t(sector_t
, ci
->sector_count
,
1530 max_io_len_target_boundary(ti
, dm_target_offset(ti
, ci
->sector
)));
1532 __send_duplicate_bios(ci
, ti
, num_bios
, &len
);
1535 ci
->sector_count
-= len
;
1540 static bool is_abnormal_io(struct bio
*bio
)
1544 switch (bio_op(bio
)) {
1545 case REQ_OP_DISCARD
:
1546 case REQ_OP_SECURE_ERASE
:
1547 case REQ_OP_WRITE_SAME
:
1548 case REQ_OP_WRITE_ZEROES
:
1556 static bool __process_abnormal_io(struct clone_info
*ci
, struct dm_target
*ti
,
1559 struct bio
*bio
= ci
->bio
;
1560 unsigned num_bios
= 0;
1562 switch (bio_op(bio
)) {
1563 case REQ_OP_DISCARD
:
1564 num_bios
= ti
->num_discard_bios
;
1566 case REQ_OP_SECURE_ERASE
:
1567 num_bios
= ti
->num_secure_erase_bios
;
1569 case REQ_OP_WRITE_SAME
:
1570 num_bios
= ti
->num_write_same_bios
;
1572 case REQ_OP_WRITE_ZEROES
:
1573 num_bios
= ti
->num_write_zeroes_bios
;
1579 *result
= __send_changing_extent_only(ci
, ti
, num_bios
);
1584 * Select the correct strategy for processing a non-flush bio.
1586 static int __split_and_process_non_flush(struct clone_info
*ci
)
1588 struct dm_target
*ti
;
1592 ti
= dm_table_find_target(ci
->map
, ci
->sector
);
1596 if (__process_abnormal_io(ci
, ti
, &r
))
1599 len
= min_t(sector_t
, max_io_len(ti
, ci
->sector
), ci
->sector_count
);
1601 r
= __clone_and_map_data_bio(ci
, ti
, ci
->sector
, &len
);
1606 ci
->sector_count
-= len
;
1611 static void init_clone_info(struct clone_info
*ci
, struct mapped_device
*md
,
1612 struct dm_table
*map
, struct bio
*bio
)
1615 ci
->io
= alloc_io(md
, bio
);
1616 ci
->sector
= bio
->bi_iter
.bi_sector
;
1619 #define __dm_part_stat_sub(part, field, subnd) \
1620 (part_stat_get(part, field) -= (subnd))
1623 * Entry point to split a bio into clones and submit them to the targets.
1625 static blk_qc_t
__split_and_process_bio(struct mapped_device
*md
,
1626 struct dm_table
*map
, struct bio
*bio
)
1628 struct clone_info ci
;
1629 blk_qc_t ret
= BLK_QC_T_NONE
;
1632 init_clone_info(&ci
, md
, map
, bio
);
1634 if (bio
->bi_opf
& REQ_PREFLUSH
) {
1635 error
= __send_empty_flush(&ci
);
1636 /* dec_pending submits any data associated with flush */
1637 } else if (op_is_zone_mgmt(bio_op(bio
))) {
1639 ci
.sector_count
= 0;
1640 error
= __split_and_process_non_flush(&ci
);
1643 ci
.sector_count
= bio_sectors(bio
);
1644 while (ci
.sector_count
&& !error
) {
1645 error
= __split_and_process_non_flush(&ci
);
1646 if (ci
.sector_count
&& !error
) {
1648 * Remainder must be passed to submit_bio_noacct()
1649 * so that it gets handled *after* bios already submitted
1650 * have been completely processed.
1651 * We take a clone of the original to store in
1652 * ci.io->orig_bio to be used by end_io_acct() and
1653 * for dec_pending to use for completion handling.
1655 struct bio
*b
= bio_split(bio
, bio_sectors(bio
) - ci
.sector_count
,
1656 GFP_NOIO
, &md
->queue
->bio_split
);
1657 ci
.io
->orig_bio
= b
;
1660 * Adjust IO stats for each split, otherwise upon queue
1661 * reentry there will be redundant IO accounting.
1662 * NOTE: this is a stop-gap fix, a proper fix involves
1663 * significant refactoring of DM core's bio splitting
1664 * (by eliminating DM's splitting and just using bio_split)
1667 __dm_part_stat_sub(dm_disk(md
)->part0
,
1668 sectors
[op_stat_group(bio_op(bio
))], ci
.sector_count
);
1672 trace_block_split(b
, bio
->bi_iter
.bi_sector
);
1673 ret
= submit_bio_noacct(bio
);
1679 /* drop the extra reference count */
1680 dec_pending(ci
.io
, errno_to_blk_status(error
));
1684 static blk_qc_t
dm_submit_bio(struct bio
*bio
)
1686 struct mapped_device
*md
= bio
->bi_bdev
->bd_disk
->private_data
;
1687 blk_qc_t ret
= BLK_QC_T_NONE
;
1689 struct dm_table
*map
;
1691 map
= dm_get_live_table(md
, &srcu_idx
);
1692 if (unlikely(!map
)) {
1693 DMERR_LIMIT("%s: mapping table unavailable, erroring io",
1694 dm_device_name(md
));
1699 /* If suspended, queue this IO for later */
1700 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND
, &md
->flags
))) {
1701 if (bio
->bi_opf
& REQ_NOWAIT
)
1702 bio_wouldblock_error(bio
);
1703 else if (bio
->bi_opf
& REQ_RAHEAD
)
1711 * Use blk_queue_split() for abnormal IO (e.g. discard, writesame, etc)
1712 * otherwise associated queue_limits won't be imposed.
1714 if (is_abnormal_io(bio
))
1715 blk_queue_split(&bio
);
1717 ret
= __split_and_process_bio(md
, map
, bio
);
1719 dm_put_live_table(md
, srcu_idx
);
1723 /*-----------------------------------------------------------------
1724 * An IDR is used to keep track of allocated minor numbers.
1725 *---------------------------------------------------------------*/
1726 static void free_minor(int minor
)
1728 spin_lock(&_minor_lock
);
1729 idr_remove(&_minor_idr
, minor
);
1730 spin_unlock(&_minor_lock
);
1734 * See if the device with a specific minor # is free.
1736 static int specific_minor(int minor
)
1740 if (minor
>= (1 << MINORBITS
))
1743 idr_preload(GFP_KERNEL
);
1744 spin_lock(&_minor_lock
);
1746 r
= idr_alloc(&_minor_idr
, MINOR_ALLOCED
, minor
, minor
+ 1, GFP_NOWAIT
);
1748 spin_unlock(&_minor_lock
);
1751 return r
== -ENOSPC
? -EBUSY
: r
;
1755 static int next_free_minor(int *minor
)
1759 idr_preload(GFP_KERNEL
);
1760 spin_lock(&_minor_lock
);
1762 r
= idr_alloc(&_minor_idr
, MINOR_ALLOCED
, 0, 1 << MINORBITS
, GFP_NOWAIT
);
1764 spin_unlock(&_minor_lock
);
1772 static const struct block_device_operations dm_blk_dops
;
1773 static const struct block_device_operations dm_rq_blk_dops
;
1774 static const struct dax_operations dm_dax_ops
;
1776 static void dm_wq_work(struct work_struct
*work
);
1778 #ifdef CONFIG_BLK_INLINE_ENCRYPTION
1779 static void dm_queue_destroy_keyslot_manager(struct request_queue
*q
)
1781 dm_destroy_keyslot_manager(q
->ksm
);
1784 #else /* CONFIG_BLK_INLINE_ENCRYPTION */
1786 static inline void dm_queue_destroy_keyslot_manager(struct request_queue
*q
)
1789 #endif /* !CONFIG_BLK_INLINE_ENCRYPTION */
1791 static void cleanup_mapped_device(struct mapped_device
*md
)
1794 destroy_workqueue(md
->wq
);
1795 bioset_exit(&md
->bs
);
1796 bioset_exit(&md
->io_bs
);
1799 kill_dax(md
->dax_dev
);
1800 put_dax(md
->dax_dev
);
1805 spin_lock(&_minor_lock
);
1806 md
->disk
->private_data
= NULL
;
1807 spin_unlock(&_minor_lock
);
1808 del_gendisk(md
->disk
);
1813 dm_queue_destroy_keyslot_manager(md
->queue
);
1814 blk_cleanup_queue(md
->queue
);
1817 cleanup_srcu_struct(&md
->io_barrier
);
1819 mutex_destroy(&md
->suspend_lock
);
1820 mutex_destroy(&md
->type_lock
);
1821 mutex_destroy(&md
->table_devices_lock
);
1822 mutex_destroy(&md
->swap_bios_lock
);
1824 dm_mq_cleanup_mapped_device(md
);
1828 * Allocate and initialise a blank device with a given minor.
1830 static struct mapped_device
*alloc_dev(int minor
)
1832 int r
, numa_node_id
= dm_get_numa_node();
1833 struct mapped_device
*md
;
1836 md
= kvzalloc_node(sizeof(*md
), GFP_KERNEL
, numa_node_id
);
1838 DMWARN("unable to allocate device, out of memory.");
1842 if (!try_module_get(THIS_MODULE
))
1843 goto bad_module_get
;
1845 /* get a minor number for the dev */
1846 if (minor
== DM_ANY_MINOR
)
1847 r
= next_free_minor(&minor
);
1849 r
= specific_minor(minor
);
1853 r
= init_srcu_struct(&md
->io_barrier
);
1855 goto bad_io_barrier
;
1857 md
->numa_node_id
= numa_node_id
;
1858 md
->init_tio_pdu
= false;
1859 md
->type
= DM_TYPE_NONE
;
1860 mutex_init(&md
->suspend_lock
);
1861 mutex_init(&md
->type_lock
);
1862 mutex_init(&md
->table_devices_lock
);
1863 spin_lock_init(&md
->deferred_lock
);
1864 atomic_set(&md
->holders
, 1);
1865 atomic_set(&md
->open_count
, 0);
1866 atomic_set(&md
->event_nr
, 0);
1867 atomic_set(&md
->uevent_seq
, 0);
1868 INIT_LIST_HEAD(&md
->uevent_list
);
1869 INIT_LIST_HEAD(&md
->table_devices
);
1870 spin_lock_init(&md
->uevent_lock
);
1873 * default to bio-based until DM table is loaded and md->type
1874 * established. If request-based table is loaded: blk-mq will
1875 * override accordingly.
1877 md
->queue
= blk_alloc_queue(numa_node_id
);
1881 md
->disk
= alloc_disk_node(1, md
->numa_node_id
);
1885 init_waitqueue_head(&md
->wait
);
1886 INIT_WORK(&md
->work
, dm_wq_work
);
1887 init_waitqueue_head(&md
->eventq
);
1888 init_completion(&md
->kobj_holder
.completion
);
1890 md
->swap_bios
= get_swap_bios();
1891 sema_init(&md
->swap_bios_semaphore
, md
->swap_bios
);
1892 mutex_init(&md
->swap_bios_lock
);
1894 md
->disk
->major
= _major
;
1895 md
->disk
->first_minor
= minor
;
1896 md
->disk
->fops
= &dm_blk_dops
;
1897 md
->disk
->queue
= md
->queue
;
1898 md
->disk
->private_data
= md
;
1899 sprintf(md
->disk
->disk_name
, "dm-%d", minor
);
1901 if (IS_ENABLED(CONFIG_DAX_DRIVER
)) {
1902 md
->dax_dev
= alloc_dax(md
, md
->disk
->disk_name
,
1904 if (IS_ERR(md
->dax_dev
))
1908 add_disk_no_queue_reg(md
->disk
);
1909 format_dev_t(md
->name
, MKDEV(_major
, minor
));
1911 md
->wq
= alloc_workqueue("kdmflush", WQ_MEM_RECLAIM
, 0);
1915 dm_stats_init(&md
->stats
);
1917 /* Populate the mapping, nobody knows we exist yet */
1918 spin_lock(&_minor_lock
);
1919 old_md
= idr_replace(&_minor_idr
, md
, minor
);
1920 spin_unlock(&_minor_lock
);
1922 BUG_ON(old_md
!= MINOR_ALLOCED
);
1927 cleanup_mapped_device(md
);
1931 module_put(THIS_MODULE
);
1937 static void unlock_fs(struct mapped_device
*md
);
1939 static void free_dev(struct mapped_device
*md
)
1941 int minor
= MINOR(disk_devt(md
->disk
));
1945 cleanup_mapped_device(md
);
1947 free_table_devices(&md
->table_devices
);
1948 dm_stats_cleanup(&md
->stats
);
1951 module_put(THIS_MODULE
);
1955 static int __bind_mempools(struct mapped_device
*md
, struct dm_table
*t
)
1957 struct dm_md_mempools
*p
= dm_table_get_md_mempools(t
);
1960 if (dm_table_bio_based(t
)) {
1962 * The md may already have mempools that need changing.
1963 * If so, reload bioset because front_pad may have changed
1964 * because a different table was loaded.
1966 bioset_exit(&md
->bs
);
1967 bioset_exit(&md
->io_bs
);
1969 } else if (bioset_initialized(&md
->bs
)) {
1971 * There's no need to reload with request-based dm
1972 * because the size of front_pad doesn't change.
1973 * Note for future: If you are to reload bioset,
1974 * prep-ed requests in the queue may refer
1975 * to bio from the old bioset, so you must walk
1976 * through the queue to unprep.
1982 bioset_initialized(&md
->bs
) ||
1983 bioset_initialized(&md
->io_bs
));
1985 ret
= bioset_init_from_src(&md
->bs
, &p
->bs
);
1988 ret
= bioset_init_from_src(&md
->io_bs
, &p
->io_bs
);
1990 bioset_exit(&md
->bs
);
1992 /* mempool bind completed, no longer need any mempools in the table */
1993 dm_table_free_md_mempools(t
);
1998 * Bind a table to the device.
2000 static void event_callback(void *context
)
2002 unsigned long flags
;
2004 struct mapped_device
*md
= (struct mapped_device
*) context
;
2006 spin_lock_irqsave(&md
->uevent_lock
, flags
);
2007 list_splice_init(&md
->uevent_list
, &uevents
);
2008 spin_unlock_irqrestore(&md
->uevent_lock
, flags
);
2010 dm_send_uevents(&uevents
, &disk_to_dev(md
->disk
)->kobj
);
2012 atomic_inc(&md
->event_nr
);
2013 wake_up(&md
->eventq
);
2014 dm_issue_global_event();
2018 * Returns old map, which caller must destroy.
2020 static struct dm_table
*__bind(struct mapped_device
*md
, struct dm_table
*t
,
2021 struct queue_limits
*limits
)
2023 struct dm_table
*old_map
;
2024 struct request_queue
*q
= md
->queue
;
2025 bool request_based
= dm_table_request_based(t
);
2029 lockdep_assert_held(&md
->suspend_lock
);
2031 size
= dm_table_get_size(t
);
2034 * Wipe any geometry if the size of the table changed.
2036 if (size
!= dm_get_size(md
))
2037 memset(&md
->geometry
, 0, sizeof(md
->geometry
));
2039 if (!get_capacity(md
->disk
))
2040 set_capacity(md
->disk
, size
);
2042 set_capacity_and_notify(md
->disk
, size
);
2044 dm_table_event_callback(t
, event_callback
, md
);
2047 * The queue hasn't been stopped yet, if the old table type wasn't
2048 * for request-based during suspension. So stop it to prevent
2049 * I/O mapping before resume.
2050 * This must be done before setting the queue restrictions,
2051 * because request-based dm may be run just after the setting.
2056 if (request_based
) {
2058 * Leverage the fact that request-based DM targets are
2059 * immutable singletons - used to optimize dm_mq_queue_rq.
2061 md
->immutable_target
= dm_table_get_immutable_target(t
);
2064 ret
= __bind_mempools(md
, t
);
2066 old_map
= ERR_PTR(ret
);
2070 old_map
= rcu_dereference_protected(md
->map
, lockdep_is_held(&md
->suspend_lock
));
2071 rcu_assign_pointer(md
->map
, (void *)t
);
2072 md
->immutable_target_type
= dm_table_get_immutable_target_type(t
);
2074 dm_table_set_restrictions(t
, q
, limits
);
2083 * Returns unbound table for the caller to free.
2085 static struct dm_table
*__unbind(struct mapped_device
*md
)
2087 struct dm_table
*map
= rcu_dereference_protected(md
->map
, 1);
2092 dm_table_event_callback(map
, NULL
, NULL
);
2093 RCU_INIT_POINTER(md
->map
, NULL
);
2100 * Constructor for a new device.
2102 int dm_create(int minor
, struct mapped_device
**result
)
2105 struct mapped_device
*md
;
2107 md
= alloc_dev(minor
);
2111 r
= dm_sysfs_init(md
);
2122 * Functions to manage md->type.
2123 * All are required to hold md->type_lock.
2125 void dm_lock_md_type(struct mapped_device
*md
)
2127 mutex_lock(&md
->type_lock
);
2130 void dm_unlock_md_type(struct mapped_device
*md
)
2132 mutex_unlock(&md
->type_lock
);
2135 void dm_set_md_type(struct mapped_device
*md
, enum dm_queue_mode type
)
2137 BUG_ON(!mutex_is_locked(&md
->type_lock
));
2141 enum dm_queue_mode
dm_get_md_type(struct mapped_device
*md
)
2146 struct target_type
*dm_get_immutable_target_type(struct mapped_device
*md
)
2148 return md
->immutable_target_type
;
2152 * The queue_limits are only valid as long as you have a reference
2155 struct queue_limits
*dm_get_queue_limits(struct mapped_device
*md
)
2157 BUG_ON(!atomic_read(&md
->holders
));
2158 return &md
->queue
->limits
;
2160 EXPORT_SYMBOL_GPL(dm_get_queue_limits
);
2163 * Setup the DM device's queue based on md's type
2165 int dm_setup_md_queue(struct mapped_device
*md
, struct dm_table
*t
)
2168 struct queue_limits limits
;
2169 enum dm_queue_mode type
= dm_get_md_type(md
);
2172 case DM_TYPE_REQUEST_BASED
:
2173 md
->disk
->fops
= &dm_rq_blk_dops
;
2174 r
= dm_mq_init_request_queue(md
, t
);
2176 DMERR("Cannot initialize queue for request-based dm mapped device");
2180 case DM_TYPE_BIO_BASED
:
2181 case DM_TYPE_DAX_BIO_BASED
:
2188 r
= dm_calculate_queue_limits(t
, &limits
);
2190 DMERR("Cannot calculate initial queue limits");
2193 dm_table_set_restrictions(t
, md
->queue
, &limits
);
2194 blk_register_queue(md
->disk
);
2199 struct mapped_device
*dm_get_md(dev_t dev
)
2201 struct mapped_device
*md
;
2202 unsigned minor
= MINOR(dev
);
2204 if (MAJOR(dev
) != _major
|| minor
>= (1 << MINORBITS
))
2207 spin_lock(&_minor_lock
);
2209 md
= idr_find(&_minor_idr
, minor
);
2210 if (!md
|| md
== MINOR_ALLOCED
|| (MINOR(disk_devt(dm_disk(md
))) != minor
) ||
2211 test_bit(DMF_FREEING
, &md
->flags
) || dm_deleting_md(md
)) {
2217 spin_unlock(&_minor_lock
);
2221 EXPORT_SYMBOL_GPL(dm_get_md
);
2223 void *dm_get_mdptr(struct mapped_device
*md
)
2225 return md
->interface_ptr
;
2228 void dm_set_mdptr(struct mapped_device
*md
, void *ptr
)
2230 md
->interface_ptr
= ptr
;
2233 void dm_get(struct mapped_device
*md
)
2235 atomic_inc(&md
->holders
);
2236 BUG_ON(test_bit(DMF_FREEING
, &md
->flags
));
2239 int dm_hold(struct mapped_device
*md
)
2241 spin_lock(&_minor_lock
);
2242 if (test_bit(DMF_FREEING
, &md
->flags
)) {
2243 spin_unlock(&_minor_lock
);
2247 spin_unlock(&_minor_lock
);
2250 EXPORT_SYMBOL_GPL(dm_hold
);
2252 const char *dm_device_name(struct mapped_device
*md
)
2256 EXPORT_SYMBOL_GPL(dm_device_name
);
2258 static void __dm_destroy(struct mapped_device
*md
, bool wait
)
2260 struct dm_table
*map
;
2265 spin_lock(&_minor_lock
);
2266 idr_replace(&_minor_idr
, MINOR_ALLOCED
, MINOR(disk_devt(dm_disk(md
))));
2267 set_bit(DMF_FREEING
, &md
->flags
);
2268 spin_unlock(&_minor_lock
);
2270 blk_set_queue_dying(md
->queue
);
2273 * Take suspend_lock so that presuspend and postsuspend methods
2274 * do not race with internal suspend.
2276 mutex_lock(&md
->suspend_lock
);
2277 map
= dm_get_live_table(md
, &srcu_idx
);
2278 if (!dm_suspended_md(md
)) {
2279 dm_table_presuspend_targets(map
);
2280 set_bit(DMF_SUSPENDED
, &md
->flags
);
2281 set_bit(DMF_POST_SUSPENDING
, &md
->flags
);
2282 dm_table_postsuspend_targets(map
);
2284 /* dm_put_live_table must be before msleep, otherwise deadlock is possible */
2285 dm_put_live_table(md
, srcu_idx
);
2286 mutex_unlock(&md
->suspend_lock
);
2289 * Rare, but there may be I/O requests still going to complete,
2290 * for example. Wait for all references to disappear.
2291 * No one should increment the reference count of the mapped_device,
2292 * after the mapped_device state becomes DMF_FREEING.
2295 while (atomic_read(&md
->holders
))
2297 else if (atomic_read(&md
->holders
))
2298 DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)",
2299 dm_device_name(md
), atomic_read(&md
->holders
));
2302 dm_table_destroy(__unbind(md
));
2306 void dm_destroy(struct mapped_device
*md
)
2308 __dm_destroy(md
, true);
2311 void dm_destroy_immediate(struct mapped_device
*md
)
2313 __dm_destroy(md
, false);
2316 void dm_put(struct mapped_device
*md
)
2318 atomic_dec(&md
->holders
);
2320 EXPORT_SYMBOL_GPL(dm_put
);
2322 static bool md_in_flight_bios(struct mapped_device
*md
)
2325 struct block_device
*part
= dm_disk(md
)->part0
;
2328 for_each_possible_cpu(cpu
) {
2329 sum
+= part_stat_local_read_cpu(part
, in_flight
[0], cpu
);
2330 sum
+= part_stat_local_read_cpu(part
, in_flight
[1], cpu
);
2336 static int dm_wait_for_bios_completion(struct mapped_device
*md
, long task_state
)
2342 prepare_to_wait(&md
->wait
, &wait
, task_state
);
2344 if (!md_in_flight_bios(md
))
2347 if (signal_pending_state(task_state
, current
)) {
2354 finish_wait(&md
->wait
, &wait
);
2359 static int dm_wait_for_completion(struct mapped_device
*md
, long task_state
)
2363 if (!queue_is_mq(md
->queue
))
2364 return dm_wait_for_bios_completion(md
, task_state
);
2367 if (!blk_mq_queue_inflight(md
->queue
))
2370 if (signal_pending_state(task_state
, current
)) {
2382 * Process the deferred bios
2384 static void dm_wq_work(struct work_struct
*work
)
2386 struct mapped_device
*md
= container_of(work
, struct mapped_device
, work
);
2389 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND
, &md
->flags
)) {
2390 spin_lock_irq(&md
->deferred_lock
);
2391 bio
= bio_list_pop(&md
->deferred
);
2392 spin_unlock_irq(&md
->deferred_lock
);
2397 submit_bio_noacct(bio
);
2401 static void dm_queue_flush(struct mapped_device
*md
)
2403 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND
, &md
->flags
);
2404 smp_mb__after_atomic();
2405 queue_work(md
->wq
, &md
->work
);
2409 * Swap in a new table, returning the old one for the caller to destroy.
2411 struct dm_table
*dm_swap_table(struct mapped_device
*md
, struct dm_table
*table
)
2413 struct dm_table
*live_map
= NULL
, *map
= ERR_PTR(-EINVAL
);
2414 struct queue_limits limits
;
2417 mutex_lock(&md
->suspend_lock
);
2419 /* device must be suspended */
2420 if (!dm_suspended_md(md
))
2424 * If the new table has no data devices, retain the existing limits.
2425 * This helps multipath with queue_if_no_path if all paths disappear,
2426 * then new I/O is queued based on these limits, and then some paths
2429 if (dm_table_has_no_data_devices(table
)) {
2430 live_map
= dm_get_live_table_fast(md
);
2432 limits
= md
->queue
->limits
;
2433 dm_put_live_table_fast(md
);
2437 r
= dm_calculate_queue_limits(table
, &limits
);
2444 map
= __bind(md
, table
, &limits
);
2445 dm_issue_global_event();
2448 mutex_unlock(&md
->suspend_lock
);
2453 * Functions to lock and unlock any filesystem running on the
2456 static int lock_fs(struct mapped_device
*md
)
2460 WARN_ON(test_bit(DMF_FROZEN
, &md
->flags
));
2462 r
= freeze_bdev(md
->disk
->part0
);
2464 set_bit(DMF_FROZEN
, &md
->flags
);
2468 static void unlock_fs(struct mapped_device
*md
)
2470 if (!test_bit(DMF_FROZEN
, &md
->flags
))
2472 thaw_bdev(md
->disk
->part0
);
2473 clear_bit(DMF_FROZEN
, &md
->flags
);
2477 * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG
2478 * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE
2479 * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY
2481 * If __dm_suspend returns 0, the device is completely quiescent
2482 * now. There is no request-processing activity. All new requests
2483 * are being added to md->deferred list.
2485 static int __dm_suspend(struct mapped_device
*md
, struct dm_table
*map
,
2486 unsigned suspend_flags
, long task_state
,
2487 int dmf_suspended_flag
)
2489 bool do_lockfs
= suspend_flags
& DM_SUSPEND_LOCKFS_FLAG
;
2490 bool noflush
= suspend_flags
& DM_SUSPEND_NOFLUSH_FLAG
;
2493 lockdep_assert_held(&md
->suspend_lock
);
2496 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
2497 * This flag is cleared before dm_suspend returns.
2500 set_bit(DMF_NOFLUSH_SUSPENDING
, &md
->flags
);
2502 DMDEBUG("%s: suspending with flush", dm_device_name(md
));
2505 * This gets reverted if there's an error later and the targets
2506 * provide the .presuspend_undo hook.
2508 dm_table_presuspend_targets(map
);
2511 * Flush I/O to the device.
2512 * Any I/O submitted after lock_fs() may not be flushed.
2513 * noflush takes precedence over do_lockfs.
2514 * (lock_fs() flushes I/Os and waits for them to complete.)
2516 if (!noflush
&& do_lockfs
) {
2519 dm_table_presuspend_undo_targets(map
);
2525 * Here we must make sure that no processes are submitting requests
2526 * to target drivers i.e. no one may be executing
2527 * __split_and_process_bio from dm_submit_bio.
2529 * To get all processes out of __split_and_process_bio in dm_submit_bio,
2530 * we take the write lock. To prevent any process from reentering
2531 * __split_and_process_bio from dm_submit_bio and quiesce the thread
2532 * (dm_wq_work), we set DMF_BLOCK_IO_FOR_SUSPEND and call
2533 * flush_workqueue(md->wq).
2535 set_bit(DMF_BLOCK_IO_FOR_SUSPEND
, &md
->flags
);
2537 synchronize_srcu(&md
->io_barrier
);
2540 * Stop md->queue before flushing md->wq in case request-based
2541 * dm defers requests to md->wq from md->queue.
2543 if (dm_request_based(md
))
2544 dm_stop_queue(md
->queue
);
2546 flush_workqueue(md
->wq
);
2549 * At this point no more requests are entering target request routines.
2550 * We call dm_wait_for_completion to wait for all existing requests
2553 r
= dm_wait_for_completion(md
, task_state
);
2555 set_bit(dmf_suspended_flag
, &md
->flags
);
2558 clear_bit(DMF_NOFLUSH_SUSPENDING
, &md
->flags
);
2560 synchronize_srcu(&md
->io_barrier
);
2562 /* were we interrupted ? */
2566 if (dm_request_based(md
))
2567 dm_start_queue(md
->queue
);
2570 dm_table_presuspend_undo_targets(map
);
2571 /* pushback list is already flushed, so skip flush */
2578 * We need to be able to change a mapping table under a mounted
2579 * filesystem. For example we might want to move some data in
2580 * the background. Before the table can be swapped with
2581 * dm_bind_table, dm_suspend must be called to flush any in
2582 * flight bios and ensure that any further io gets deferred.
2585 * Suspend mechanism in request-based dm.
2587 * 1. Flush all I/Os by lock_fs() if needed.
2588 * 2. Stop dispatching any I/O by stopping the request_queue.
2589 * 3. Wait for all in-flight I/Os to be completed or requeued.
2591 * To abort suspend, start the request_queue.
2593 int dm_suspend(struct mapped_device
*md
, unsigned suspend_flags
)
2595 struct dm_table
*map
= NULL
;
2599 mutex_lock_nested(&md
->suspend_lock
, SINGLE_DEPTH_NESTING
);
2601 if (dm_suspended_md(md
)) {
2606 if (dm_suspended_internally_md(md
)) {
2607 /* already internally suspended, wait for internal resume */
2608 mutex_unlock(&md
->suspend_lock
);
2609 r
= wait_on_bit(&md
->flags
, DMF_SUSPENDED_INTERNALLY
, TASK_INTERRUPTIBLE
);
2615 map
= rcu_dereference_protected(md
->map
, lockdep_is_held(&md
->suspend_lock
));
2617 r
= __dm_suspend(md
, map
, suspend_flags
, TASK_INTERRUPTIBLE
, DMF_SUSPENDED
);
2621 set_bit(DMF_POST_SUSPENDING
, &md
->flags
);
2622 dm_table_postsuspend_targets(map
);
2623 clear_bit(DMF_POST_SUSPENDING
, &md
->flags
);
2626 mutex_unlock(&md
->suspend_lock
);
2630 static int __dm_resume(struct mapped_device
*md
, struct dm_table
*map
)
2633 int r
= dm_table_resume_targets(map
);
2641 * Flushing deferred I/Os must be done after targets are resumed
2642 * so that mapping of targets can work correctly.
2643 * Request-based dm is queueing the deferred I/Os in its request_queue.
2645 if (dm_request_based(md
))
2646 dm_start_queue(md
->queue
);
2653 int dm_resume(struct mapped_device
*md
)
2656 struct dm_table
*map
= NULL
;
2660 mutex_lock_nested(&md
->suspend_lock
, SINGLE_DEPTH_NESTING
);
2662 if (!dm_suspended_md(md
))
2665 if (dm_suspended_internally_md(md
)) {
2666 /* already internally suspended, wait for internal resume */
2667 mutex_unlock(&md
->suspend_lock
);
2668 r
= wait_on_bit(&md
->flags
, DMF_SUSPENDED_INTERNALLY
, TASK_INTERRUPTIBLE
);
2674 map
= rcu_dereference_protected(md
->map
, lockdep_is_held(&md
->suspend_lock
));
2675 if (!map
|| !dm_table_get_size(map
))
2678 r
= __dm_resume(md
, map
);
2682 clear_bit(DMF_SUSPENDED
, &md
->flags
);
2684 mutex_unlock(&md
->suspend_lock
);
2690 * Internal suspend/resume works like userspace-driven suspend. It waits
2691 * until all bios finish and prevents issuing new bios to the target drivers.
2692 * It may be used only from the kernel.
2695 static void __dm_internal_suspend(struct mapped_device
*md
, unsigned suspend_flags
)
2697 struct dm_table
*map
= NULL
;
2699 lockdep_assert_held(&md
->suspend_lock
);
2701 if (md
->internal_suspend_count
++)
2702 return; /* nested internal suspend */
2704 if (dm_suspended_md(md
)) {
2705 set_bit(DMF_SUSPENDED_INTERNALLY
, &md
->flags
);
2706 return; /* nest suspend */
2709 map
= rcu_dereference_protected(md
->map
, lockdep_is_held(&md
->suspend_lock
));
2712 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is
2713 * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend
2714 * would require changing .presuspend to return an error -- avoid this
2715 * until there is a need for more elaborate variants of internal suspend.
2717 (void) __dm_suspend(md
, map
, suspend_flags
, TASK_UNINTERRUPTIBLE
,
2718 DMF_SUSPENDED_INTERNALLY
);
2720 set_bit(DMF_POST_SUSPENDING
, &md
->flags
);
2721 dm_table_postsuspend_targets(map
);
2722 clear_bit(DMF_POST_SUSPENDING
, &md
->flags
);
2725 static void __dm_internal_resume(struct mapped_device
*md
)
2727 BUG_ON(!md
->internal_suspend_count
);
2729 if (--md
->internal_suspend_count
)
2730 return; /* resume from nested internal suspend */
2732 if (dm_suspended_md(md
))
2733 goto done
; /* resume from nested suspend */
2736 * NOTE: existing callers don't need to call dm_table_resume_targets
2737 * (which may fail -- so best to avoid it for now by passing NULL map)
2739 (void) __dm_resume(md
, NULL
);
2742 clear_bit(DMF_SUSPENDED_INTERNALLY
, &md
->flags
);
2743 smp_mb__after_atomic();
2744 wake_up_bit(&md
->flags
, DMF_SUSPENDED_INTERNALLY
);
2747 void dm_internal_suspend_noflush(struct mapped_device
*md
)
2749 mutex_lock(&md
->suspend_lock
);
2750 __dm_internal_suspend(md
, DM_SUSPEND_NOFLUSH_FLAG
);
2751 mutex_unlock(&md
->suspend_lock
);
2753 EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush
);
2755 void dm_internal_resume(struct mapped_device
*md
)
2757 mutex_lock(&md
->suspend_lock
);
2758 __dm_internal_resume(md
);
2759 mutex_unlock(&md
->suspend_lock
);
2761 EXPORT_SYMBOL_GPL(dm_internal_resume
);
2764 * Fast variants of internal suspend/resume hold md->suspend_lock,
2765 * which prevents interaction with userspace-driven suspend.
2768 void dm_internal_suspend_fast(struct mapped_device
*md
)
2770 mutex_lock(&md
->suspend_lock
);
2771 if (dm_suspended_md(md
) || dm_suspended_internally_md(md
))
2774 set_bit(DMF_BLOCK_IO_FOR_SUSPEND
, &md
->flags
);
2775 synchronize_srcu(&md
->io_barrier
);
2776 flush_workqueue(md
->wq
);
2777 dm_wait_for_completion(md
, TASK_UNINTERRUPTIBLE
);
2779 EXPORT_SYMBOL_GPL(dm_internal_suspend_fast
);
2781 void dm_internal_resume_fast(struct mapped_device
*md
)
2783 if (dm_suspended_md(md
) || dm_suspended_internally_md(md
))
2789 mutex_unlock(&md
->suspend_lock
);
2791 EXPORT_SYMBOL_GPL(dm_internal_resume_fast
);
2793 /*-----------------------------------------------------------------
2794 * Event notification.
2795 *---------------------------------------------------------------*/
2796 int dm_kobject_uevent(struct mapped_device
*md
, enum kobject_action action
,
2801 char udev_cookie
[DM_COOKIE_LENGTH
];
2802 char *envp
[] = { udev_cookie
, NULL
};
2804 noio_flag
= memalloc_noio_save();
2807 r
= kobject_uevent(&disk_to_dev(md
->disk
)->kobj
, action
);
2809 snprintf(udev_cookie
, DM_COOKIE_LENGTH
, "%s=%u",
2810 DM_COOKIE_ENV_VAR_NAME
, cookie
);
2811 r
= kobject_uevent_env(&disk_to_dev(md
->disk
)->kobj
,
2815 memalloc_noio_restore(noio_flag
);
2820 uint32_t dm_next_uevent_seq(struct mapped_device
*md
)
2822 return atomic_add_return(1, &md
->uevent_seq
);
2825 uint32_t dm_get_event_nr(struct mapped_device
*md
)
2827 return atomic_read(&md
->event_nr
);
2830 int dm_wait_event(struct mapped_device
*md
, int event_nr
)
2832 return wait_event_interruptible(md
->eventq
,
2833 (event_nr
!= atomic_read(&md
->event_nr
)));
2836 void dm_uevent_add(struct mapped_device
*md
, struct list_head
*elist
)
2838 unsigned long flags
;
2840 spin_lock_irqsave(&md
->uevent_lock
, flags
);
2841 list_add(elist
, &md
->uevent_list
);
2842 spin_unlock_irqrestore(&md
->uevent_lock
, flags
);
2846 * The gendisk is only valid as long as you have a reference
2849 struct gendisk
*dm_disk(struct mapped_device
*md
)
2853 EXPORT_SYMBOL_GPL(dm_disk
);
2855 struct kobject
*dm_kobject(struct mapped_device
*md
)
2857 return &md
->kobj_holder
.kobj
;
2860 struct mapped_device
*dm_get_from_kobject(struct kobject
*kobj
)
2862 struct mapped_device
*md
;
2864 md
= container_of(kobj
, struct mapped_device
, kobj_holder
.kobj
);
2866 spin_lock(&_minor_lock
);
2867 if (test_bit(DMF_FREEING
, &md
->flags
) || dm_deleting_md(md
)) {
2873 spin_unlock(&_minor_lock
);
2878 int dm_suspended_md(struct mapped_device
*md
)
2880 return test_bit(DMF_SUSPENDED
, &md
->flags
);
2883 static int dm_post_suspending_md(struct mapped_device
*md
)
2885 return test_bit(DMF_POST_SUSPENDING
, &md
->flags
);
2888 int dm_suspended_internally_md(struct mapped_device
*md
)
2890 return test_bit(DMF_SUSPENDED_INTERNALLY
, &md
->flags
);
2893 int dm_test_deferred_remove_flag(struct mapped_device
*md
)
2895 return test_bit(DMF_DEFERRED_REMOVE
, &md
->flags
);
2898 int dm_suspended(struct dm_target
*ti
)
2900 return dm_suspended_md(ti
->table
->md
);
2902 EXPORT_SYMBOL_GPL(dm_suspended
);
2904 int dm_post_suspending(struct dm_target
*ti
)
2906 return dm_post_suspending_md(ti
->table
->md
);
2908 EXPORT_SYMBOL_GPL(dm_post_suspending
);
2910 int dm_noflush_suspending(struct dm_target
*ti
)
2912 return __noflush_suspending(ti
->table
->md
);
2914 EXPORT_SYMBOL_GPL(dm_noflush_suspending
);
2916 struct dm_md_mempools
*dm_alloc_md_mempools(struct mapped_device
*md
, enum dm_queue_mode type
,
2917 unsigned integrity
, unsigned per_io_data_size
,
2918 unsigned min_pool_size
)
2920 struct dm_md_mempools
*pools
= kzalloc_node(sizeof(*pools
), GFP_KERNEL
, md
->numa_node_id
);
2921 unsigned int pool_size
= 0;
2922 unsigned int front_pad
, io_front_pad
;
2929 case DM_TYPE_BIO_BASED
:
2930 case DM_TYPE_DAX_BIO_BASED
:
2931 pool_size
= max(dm_get_reserved_bio_based_ios(), min_pool_size
);
2932 front_pad
= roundup(per_io_data_size
, __alignof__(struct dm_target_io
)) + DM_TARGET_IO_BIO_OFFSET
;
2933 io_front_pad
= roundup(per_io_data_size
, __alignof__(struct dm_io
)) + DM_IO_BIO_OFFSET
;
2934 ret
= bioset_init(&pools
->io_bs
, pool_size
, io_front_pad
, 0);
2937 if (integrity
&& bioset_integrity_create(&pools
->io_bs
, pool_size
))
2940 case DM_TYPE_REQUEST_BASED
:
2941 pool_size
= max(dm_get_reserved_rq_based_ios(), min_pool_size
);
2942 front_pad
= offsetof(struct dm_rq_clone_bio_info
, clone
);
2943 /* per_io_data_size is used for blk-mq pdu at queue allocation */
2949 ret
= bioset_init(&pools
->bs
, pool_size
, front_pad
, 0);
2953 if (integrity
&& bioset_integrity_create(&pools
->bs
, pool_size
))
2959 dm_free_md_mempools(pools
);
2964 void dm_free_md_mempools(struct dm_md_mempools
*pools
)
2969 bioset_exit(&pools
->bs
);
2970 bioset_exit(&pools
->io_bs
);
2982 static int dm_call_pr(struct block_device
*bdev
, iterate_devices_callout_fn fn
,
2985 struct mapped_device
*md
= bdev
->bd_disk
->private_data
;
2986 struct dm_table
*table
;
2987 struct dm_target
*ti
;
2988 int ret
= -ENOTTY
, srcu_idx
;
2990 table
= dm_get_live_table(md
, &srcu_idx
);
2991 if (!table
|| !dm_table_get_size(table
))
2994 /* We only support devices that have a single target */
2995 if (dm_table_get_num_targets(table
) != 1)
2997 ti
= dm_table_get_target(table
, 0);
3000 if (!ti
->type
->iterate_devices
)
3003 ret
= ti
->type
->iterate_devices(ti
, fn
, data
);
3005 dm_put_live_table(md
, srcu_idx
);
3010 * For register / unregister we need to manually call out to every path.
3012 static int __dm_pr_register(struct dm_target
*ti
, struct dm_dev
*dev
,
3013 sector_t start
, sector_t len
, void *data
)
3015 struct dm_pr
*pr
= data
;
3016 const struct pr_ops
*ops
= dev
->bdev
->bd_disk
->fops
->pr_ops
;
3018 if (!ops
|| !ops
->pr_register
)
3020 return ops
->pr_register(dev
->bdev
, pr
->old_key
, pr
->new_key
, pr
->flags
);
3023 static int dm_pr_register(struct block_device
*bdev
, u64 old_key
, u64 new_key
,
3034 ret
= dm_call_pr(bdev
, __dm_pr_register
, &pr
);
3035 if (ret
&& new_key
) {
3036 /* unregister all paths if we failed to register any path */
3037 pr
.old_key
= new_key
;
3040 pr
.fail_early
= false;
3041 dm_call_pr(bdev
, __dm_pr_register
, &pr
);
3047 static int dm_pr_reserve(struct block_device
*bdev
, u64 key
, enum pr_type type
,
3050 struct mapped_device
*md
= bdev
->bd_disk
->private_data
;
3051 const struct pr_ops
*ops
;
3054 r
= dm_prepare_ioctl(md
, &srcu_idx
, &bdev
);
3058 ops
= bdev
->bd_disk
->fops
->pr_ops
;
3059 if (ops
&& ops
->pr_reserve
)
3060 r
= ops
->pr_reserve(bdev
, key
, type
, flags
);
3064 dm_unprepare_ioctl(md
, srcu_idx
);
3068 static int dm_pr_release(struct block_device
*bdev
, u64 key
, enum pr_type type
)
3070 struct mapped_device
*md
= bdev
->bd_disk
->private_data
;
3071 const struct pr_ops
*ops
;
3074 r
= dm_prepare_ioctl(md
, &srcu_idx
, &bdev
);
3078 ops
= bdev
->bd_disk
->fops
->pr_ops
;
3079 if (ops
&& ops
->pr_release
)
3080 r
= ops
->pr_release(bdev
, key
, type
);
3084 dm_unprepare_ioctl(md
, srcu_idx
);
3088 static int dm_pr_preempt(struct block_device
*bdev
, u64 old_key
, u64 new_key
,
3089 enum pr_type type
, bool abort
)
3091 struct mapped_device
*md
= bdev
->bd_disk
->private_data
;
3092 const struct pr_ops
*ops
;
3095 r
= dm_prepare_ioctl(md
, &srcu_idx
, &bdev
);
3099 ops
= bdev
->bd_disk
->fops
->pr_ops
;
3100 if (ops
&& ops
->pr_preempt
)
3101 r
= ops
->pr_preempt(bdev
, old_key
, new_key
, type
, abort
);
3105 dm_unprepare_ioctl(md
, srcu_idx
);
3109 static int dm_pr_clear(struct block_device
*bdev
, u64 key
)
3111 struct mapped_device
*md
= bdev
->bd_disk
->private_data
;
3112 const struct pr_ops
*ops
;
3115 r
= dm_prepare_ioctl(md
, &srcu_idx
, &bdev
);
3119 ops
= bdev
->bd_disk
->fops
->pr_ops
;
3120 if (ops
&& ops
->pr_clear
)
3121 r
= ops
->pr_clear(bdev
, key
);
3125 dm_unprepare_ioctl(md
, srcu_idx
);
3129 static const struct pr_ops dm_pr_ops
= {
3130 .pr_register
= dm_pr_register
,
3131 .pr_reserve
= dm_pr_reserve
,
3132 .pr_release
= dm_pr_release
,
3133 .pr_preempt
= dm_pr_preempt
,
3134 .pr_clear
= dm_pr_clear
,
3137 static const struct block_device_operations dm_blk_dops
= {
3138 .submit_bio
= dm_submit_bio
,
3139 .open
= dm_blk_open
,
3140 .release
= dm_blk_close
,
3141 .ioctl
= dm_blk_ioctl
,
3142 .getgeo
= dm_blk_getgeo
,
3143 .report_zones
= dm_blk_report_zones
,
3144 .pr_ops
= &dm_pr_ops
,
3145 .owner
= THIS_MODULE
3148 static const struct block_device_operations dm_rq_blk_dops
= {
3149 .open
= dm_blk_open
,
3150 .release
= dm_blk_close
,
3151 .ioctl
= dm_blk_ioctl
,
3152 .getgeo
= dm_blk_getgeo
,
3153 .pr_ops
= &dm_pr_ops
,
3154 .owner
= THIS_MODULE
3157 static const struct dax_operations dm_dax_ops
= {
3158 .direct_access
= dm_dax_direct_access
,
3159 .dax_supported
= dm_dax_supported
,
3160 .copy_from_iter
= dm_dax_copy_from_iter
,
3161 .copy_to_iter
= dm_dax_copy_to_iter
,
3162 .zero_page_range
= dm_dax_zero_page_range
,
3168 module_init(dm_init
);
3169 module_exit(dm_exit
);
3171 module_param(major
, uint
, 0);
3172 MODULE_PARM_DESC(major
, "The major number of the device mapper");
3174 module_param(reserved_bio_based_ios
, uint
, S_IRUGO
| S_IWUSR
);
3175 MODULE_PARM_DESC(reserved_bio_based_ios
, "Reserved IOs in bio-based mempools");
3177 module_param(dm_numa_node
, int, S_IRUGO
| S_IWUSR
);
3178 MODULE_PARM_DESC(dm_numa_node
, "NUMA node for DM device memory allocations");
3180 module_param(swap_bios
, int, S_IRUGO
| S_IWUSR
);
3181 MODULE_PARM_DESC(swap_bios
, "Maximum allowed inflight swap IOs");
3183 MODULE_DESCRIPTION(DM_NAME
" driver");
3184 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
3185 MODULE_LICENSE("GPL");