]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blame - drivers/md/dm.c
md/r5cache: r5cache recovery: part 2
[mirror_ubuntu-bionic-kernel.git] / drivers / md / dm.c
CommitLineData
1da177e4
LT
1/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
784aae73 3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
1da177e4
LT
4 *
5 * This file is released under the GPL.
6 */
7
4cc96131
MS
8#include "dm-core.h"
9#include "dm-rq.h"
51e5b2bd 10#include "dm-uevent.h"
1da177e4
LT
11
12#include <linux/init.h>
13#include <linux/module.h>
48c9c27b 14#include <linux/mutex.h>
1da177e4
LT
15#include <linux/blkpg.h>
16#include <linux/bio.h>
1da177e4
LT
17#include <linux/mempool.h>
18#include <linux/slab.h>
19#include <linux/idr.h>
3ac51e74 20#include <linux/hdreg.h>
3f77316d 21#include <linux/delay.h>
ffcc3936 22#include <linux/wait.h>
71cdb697 23#include <linux/pr.h>
55782138 24
72d94861
AK
25#define DM_MSG_PREFIX "core"
26
71a16736
NK
27#ifdef CONFIG_PRINTK
28/*
29 * ratelimit state to be used in DMXXX_LIMIT().
30 */
31DEFINE_RATELIMIT_STATE(dm_ratelimit_state,
32 DEFAULT_RATELIMIT_INTERVAL,
33 DEFAULT_RATELIMIT_BURST);
34EXPORT_SYMBOL(dm_ratelimit_state);
35#endif
36
60935eb2
MB
37/*
38 * Cookies are numeric values sent with CHANGE and REMOVE
39 * uevents while resuming, removing or renaming the device.
40 */
41#define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE"
42#define DM_COOKIE_LENGTH 24
43
1da177e4
LT
44static const char *_name = DM_NAME;
45
46static unsigned int major = 0;
47static unsigned int _major = 0;
48
d15b774c
AK
49static DEFINE_IDR(_minor_idr);
50
f32c10b0 51static DEFINE_SPINLOCK(_minor_lock);
2c140a24
MP
52
53static void do_deferred_remove(struct work_struct *w);
54
55static DECLARE_WORK(deferred_remove_work, do_deferred_remove);
56
acfe0ad7
MP
57static struct workqueue_struct *deferred_remove_workqueue;
58
1da177e4
LT
59/*
60 * One of these is allocated per bio.
61 */
62struct dm_io {
63 struct mapped_device *md;
64 int error;
1da177e4 65 atomic_t io_count;
6ae2fa67 66 struct bio *bio;
3eaf840e 67 unsigned long start_time;
f88fb981 68 spinlock_t endio_lock;
fd2ed4d2 69 struct dm_stats_aux stats_aux;
1da177e4
LT
70};
71
ba61fdd1
JM
72#define MINOR_ALLOCED ((void *)-1)
73
1da177e4
LT
74/*
75 * Bits for the md->flags field.
76 */
1eb787ec 77#define DMF_BLOCK_IO_FOR_SUSPEND 0
1da177e4 78#define DMF_SUSPENDED 1
aa8d7c2f 79#define DMF_FROZEN 2
fba9f90e 80#define DMF_FREEING 3
5c6bd75d 81#define DMF_DELETING 4
2e93ccc1 82#define DMF_NOFLUSH_SUSPENDING 5
8ae12666
KO
83#define DMF_DEFERRED_REMOVE 6
84#define DMF_SUSPENDED_INTERNALLY 7
1da177e4 85
115485e8 86#define DM_NUMA_NODE NUMA_NO_NODE
115485e8 87static int dm_numa_node = DM_NUMA_NODE;
faad87df 88
e6ee8c0b
KU
89/*
90 * For mempools pre-allocation at the table loading time.
91 */
92struct dm_md_mempools {
93 mempool_t *io_pool;
1ae49ea2 94 mempool_t *rq_pool;
e6ee8c0b
KU
95 struct bio_set *bs;
96};
97
86f1152b
BM
98struct table_device {
99 struct list_head list;
100 atomic_t count;
101 struct dm_dev dm_dev;
102};
103
e18b890b 104static struct kmem_cache *_io_cache;
8fbf26ad 105static struct kmem_cache *_rq_tio_cache;
1ae49ea2 106static struct kmem_cache *_rq_cache;
94818742 107
e8603136
MS
108/*
109 * Bio-based DM's mempools' reserved IOs set by the user.
110 */
4cc96131 111#define RESERVED_BIO_BASED_IOS 16
e8603136
MS
112static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS;
113
115485e8
MS
114static int __dm_get_module_param_int(int *module_param, int min, int max)
115{
116 int param = ACCESS_ONCE(*module_param);
117 int modified_param = 0;
118 bool modified = true;
119
120 if (param < min)
121 modified_param = min;
122 else if (param > max)
123 modified_param = max;
124 else
125 modified = false;
126
127 if (modified) {
128 (void)cmpxchg(module_param, param, modified_param);
129 param = modified_param;
130 }
131
132 return param;
133}
134
4cc96131
MS
135unsigned __dm_get_module_param(unsigned *module_param,
136 unsigned def, unsigned max)
f4790826 137{
09c2d531
MS
138 unsigned param = ACCESS_ONCE(*module_param);
139 unsigned modified_param = 0;
f4790826 140
09c2d531
MS
141 if (!param)
142 modified_param = def;
143 else if (param > max)
144 modified_param = max;
f4790826 145
09c2d531
MS
146 if (modified_param) {
147 (void)cmpxchg(module_param, param, modified_param);
148 param = modified_param;
f4790826
MS
149 }
150
09c2d531 151 return param;
f4790826
MS
152}
153
e8603136
MS
154unsigned dm_get_reserved_bio_based_ios(void)
155{
09c2d531 156 return __dm_get_module_param(&reserved_bio_based_ios,
4cc96131 157 RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS);
e8603136
MS
158}
159EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios);
160
115485e8
MS
161static unsigned dm_get_numa_node(void)
162{
163 return __dm_get_module_param_int(&dm_numa_node,
164 DM_NUMA_NODE, num_online_nodes() - 1);
165}
166
1da177e4
LT
167static int __init local_init(void)
168{
51157b4a 169 int r = -ENOMEM;
1da177e4 170
1da177e4 171 /* allocate a slab for the dm_ios */
028867ac 172 _io_cache = KMEM_CACHE(dm_io, 0);
1da177e4 173 if (!_io_cache)
51157b4a 174 return r;
1da177e4 175
8fbf26ad
KU
176 _rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0);
177 if (!_rq_tio_cache)
dba14160 178 goto out_free_io_cache;
8fbf26ad 179
eca7ee6d 180 _rq_cache = kmem_cache_create("dm_old_clone_request", sizeof(struct request),
1ae49ea2
MS
181 __alignof__(struct request), 0, NULL);
182 if (!_rq_cache)
183 goto out_free_rq_tio_cache;
184
51e5b2bd 185 r = dm_uevent_init();
51157b4a 186 if (r)
1ae49ea2 187 goto out_free_rq_cache;
51e5b2bd 188
acfe0ad7
MP
189 deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1);
190 if (!deferred_remove_workqueue) {
191 r = -ENOMEM;
192 goto out_uevent_exit;
193 }
194
1da177e4
LT
195 _major = major;
196 r = register_blkdev(_major, _name);
51157b4a 197 if (r < 0)
acfe0ad7 198 goto out_free_workqueue;
1da177e4
LT
199
200 if (!_major)
201 _major = r;
202
203 return 0;
51157b4a 204
acfe0ad7
MP
205out_free_workqueue:
206 destroy_workqueue(deferred_remove_workqueue);
51157b4a
KU
207out_uevent_exit:
208 dm_uevent_exit();
1ae49ea2
MS
209out_free_rq_cache:
210 kmem_cache_destroy(_rq_cache);
8fbf26ad
KU
211out_free_rq_tio_cache:
212 kmem_cache_destroy(_rq_tio_cache);
51157b4a
KU
213out_free_io_cache:
214 kmem_cache_destroy(_io_cache);
215
216 return r;
1da177e4
LT
217}
218
219static void local_exit(void)
220{
2c140a24 221 flush_scheduled_work();
acfe0ad7 222 destroy_workqueue(deferred_remove_workqueue);
2c140a24 223
1ae49ea2 224 kmem_cache_destroy(_rq_cache);
8fbf26ad 225 kmem_cache_destroy(_rq_tio_cache);
1da177e4 226 kmem_cache_destroy(_io_cache);
00d59405 227 unregister_blkdev(_major, _name);
51e5b2bd 228 dm_uevent_exit();
1da177e4
LT
229
230 _major = 0;
231
232 DMINFO("cleaned up");
233}
234
b9249e55 235static int (*_inits[])(void) __initdata = {
1da177e4
LT
236 local_init,
237 dm_target_init,
238 dm_linear_init,
239 dm_stripe_init,
952b3557 240 dm_io_init,
945fa4d2 241 dm_kcopyd_init,
1da177e4 242 dm_interface_init,
fd2ed4d2 243 dm_statistics_init,
1da177e4
LT
244};
245
b9249e55 246static void (*_exits[])(void) = {
1da177e4
LT
247 local_exit,
248 dm_target_exit,
249 dm_linear_exit,
250 dm_stripe_exit,
952b3557 251 dm_io_exit,
945fa4d2 252 dm_kcopyd_exit,
1da177e4 253 dm_interface_exit,
fd2ed4d2 254 dm_statistics_exit,
1da177e4
LT
255};
256
257static int __init dm_init(void)
258{
259 const int count = ARRAY_SIZE(_inits);
260
261 int r, i;
262
263 for (i = 0; i < count; i++) {
264 r = _inits[i]();
265 if (r)
266 goto bad;
267 }
268
269 return 0;
270
271 bad:
272 while (i--)
273 _exits[i]();
274
275 return r;
276}
277
278static void __exit dm_exit(void)
279{
280 int i = ARRAY_SIZE(_exits);
281
282 while (i--)
283 _exits[i]();
d15b774c
AK
284
285 /*
286 * Should be empty by this point.
287 */
d15b774c 288 idr_destroy(&_minor_idr);
1da177e4
LT
289}
290
291/*
292 * Block device functions
293 */
432a212c
MA
294int dm_deleting_md(struct mapped_device *md)
295{
296 return test_bit(DMF_DELETING, &md->flags);
297}
298
fe5f9f2c 299static int dm_blk_open(struct block_device *bdev, fmode_t mode)
1da177e4
LT
300{
301 struct mapped_device *md;
302
fba9f90e
JM
303 spin_lock(&_minor_lock);
304
fe5f9f2c 305 md = bdev->bd_disk->private_data;
fba9f90e
JM
306 if (!md)
307 goto out;
308
5c6bd75d 309 if (test_bit(DMF_FREEING, &md->flags) ||
432a212c 310 dm_deleting_md(md)) {
fba9f90e
JM
311 md = NULL;
312 goto out;
313 }
314
1da177e4 315 dm_get(md);
5c6bd75d 316 atomic_inc(&md->open_count);
fba9f90e
JM
317out:
318 spin_unlock(&_minor_lock);
319
320 return md ? 0 : -ENXIO;
1da177e4
LT
321}
322
db2a144b 323static void dm_blk_close(struct gendisk *disk, fmode_t mode)
1da177e4 324{
63a4f065 325 struct mapped_device *md;
6e9624b8 326
4a1aeb98
MB
327 spin_lock(&_minor_lock);
328
63a4f065
MS
329 md = disk->private_data;
330 if (WARN_ON(!md))
331 goto out;
332
2c140a24
MP
333 if (atomic_dec_and_test(&md->open_count) &&
334 (test_bit(DMF_DEFERRED_REMOVE, &md->flags)))
acfe0ad7 335 queue_work(deferred_remove_workqueue, &deferred_remove_work);
2c140a24 336
1da177e4 337 dm_put(md);
63a4f065 338out:
4a1aeb98 339 spin_unlock(&_minor_lock);
1da177e4
LT
340}
341
5c6bd75d
AK
342int dm_open_count(struct mapped_device *md)
343{
344 return atomic_read(&md->open_count);
345}
346
347/*
348 * Guarantees nothing is using the device before it's deleted.
349 */
2c140a24 350int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred)
5c6bd75d
AK
351{
352 int r = 0;
353
354 spin_lock(&_minor_lock);
355
2c140a24 356 if (dm_open_count(md)) {
5c6bd75d 357 r = -EBUSY;
2c140a24
MP
358 if (mark_deferred)
359 set_bit(DMF_DEFERRED_REMOVE, &md->flags);
360 } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags))
361 r = -EEXIST;
5c6bd75d
AK
362 else
363 set_bit(DMF_DELETING, &md->flags);
364
365 spin_unlock(&_minor_lock);
366
367 return r;
368}
369
2c140a24
MP
370int dm_cancel_deferred_remove(struct mapped_device *md)
371{
372 int r = 0;
373
374 spin_lock(&_minor_lock);
375
376 if (test_bit(DMF_DELETING, &md->flags))
377 r = -EBUSY;
378 else
379 clear_bit(DMF_DEFERRED_REMOVE, &md->flags);
380
381 spin_unlock(&_minor_lock);
382
383 return r;
384}
385
386static void do_deferred_remove(struct work_struct *w)
387{
388 dm_deferred_remove();
389}
390
fd2ed4d2
MP
391sector_t dm_get_size(struct mapped_device *md)
392{
393 return get_capacity(md->disk);
394}
395
9974fa2c
MS
396struct request_queue *dm_get_md_queue(struct mapped_device *md)
397{
398 return md->queue;
399}
400
fd2ed4d2
MP
401struct dm_stats *dm_get_stats(struct mapped_device *md)
402{
403 return &md->stats;
404}
405
3ac51e74
DW
406static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
407{
408 struct mapped_device *md = bdev->bd_disk->private_data;
409
410 return dm_get_geometry(md, geo);
411}
412
956a4025
MS
413static int dm_grab_bdev_for_ioctl(struct mapped_device *md,
414 struct block_device **bdev,
415 fmode_t *mode)
aa129a22 416{
66482026 417 struct dm_target *tgt;
6c182cd8 418 struct dm_table *map;
956a4025 419 int srcu_idx, r;
aa129a22 420
6c182cd8 421retry:
e56f81e0 422 r = -ENOTTY;
956a4025 423 map = dm_get_live_table(md, &srcu_idx);
aa129a22
MB
424 if (!map || !dm_table_get_size(map))
425 goto out;
426
427 /* We only support devices that have a single target */
428 if (dm_table_get_num_targets(map) != 1)
429 goto out;
430
66482026
MS
431 tgt = dm_table_get_target(map, 0);
432 if (!tgt->type->prepare_ioctl)
4d341d82 433 goto out;
aa129a22 434
4f186f8b 435 if (dm_suspended_md(md)) {
aa129a22
MB
436 r = -EAGAIN;
437 goto out;
438 }
439
66482026 440 r = tgt->type->prepare_ioctl(tgt, bdev, mode);
e56f81e0
CH
441 if (r < 0)
442 goto out;
aa129a22 443
956a4025
MS
444 bdgrab(*bdev);
445 dm_put_live_table(md, srcu_idx);
e56f81e0 446 return r;
aa129a22 447
aa129a22 448out:
956a4025 449 dm_put_live_table(md, srcu_idx);
5bbbfdf6 450 if (r == -ENOTCONN && !fatal_signal_pending(current)) {
6c182cd8
HR
451 msleep(10);
452 goto retry;
453 }
e56f81e0
CH
454 return r;
455}
456
457static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
458 unsigned int cmd, unsigned long arg)
459{
460 struct mapped_device *md = bdev->bd_disk->private_data;
956a4025 461 int r;
e56f81e0 462
956a4025 463 r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
e56f81e0
CH
464 if (r < 0)
465 return r;
6c182cd8 466
e56f81e0
CH
467 if (r > 0) {
468 /*
469 * Target determined this ioctl is being issued against
470 * a logical partition of the parent bdev; so extra
471 * validation is needed.
472 */
473 r = scsi_verify_blk_ioctl(NULL, cmd);
474 if (r)
475 goto out;
476 }
6c182cd8 477
66482026 478 r = __blkdev_driver_ioctl(bdev, mode, cmd, arg);
e56f81e0 479out:
956a4025 480 bdput(bdev);
aa129a22
MB
481 return r;
482}
483
028867ac 484static struct dm_io *alloc_io(struct mapped_device *md)
1da177e4
LT
485{
486 return mempool_alloc(md->io_pool, GFP_NOIO);
487}
488
028867ac 489static void free_io(struct mapped_device *md, struct dm_io *io)
1da177e4
LT
490{
491 mempool_free(io, md->io_pool);
492}
493
cfae7529 494static void free_tio(struct dm_target_io *tio)
1da177e4 495{
dba14160 496 bio_put(&tio->clone);
1da177e4
LT
497}
498
4cc96131 499int md_in_flight(struct mapped_device *md)
90abb8c4
KU
500{
501 return atomic_read(&md->pending[READ]) +
502 atomic_read(&md->pending[WRITE]);
503}
504
3eaf840e
JNN
505static void start_io_acct(struct dm_io *io)
506{
507 struct mapped_device *md = io->md;
fd2ed4d2 508 struct bio *bio = io->bio;
c9959059 509 int cpu;
fd2ed4d2 510 int rw = bio_data_dir(bio);
3eaf840e
JNN
511
512 io->start_time = jiffies;
513
074a7aca
TH
514 cpu = part_stat_lock();
515 part_round_stats(cpu, &dm_disk(md)->part0);
516 part_stat_unlock();
1e9bb880
SL
517 atomic_set(&dm_disk(md)->part0.in_flight[rw],
518 atomic_inc_return(&md->pending[rw]));
fd2ed4d2
MP
519
520 if (unlikely(dm_stats_used(&md->stats)))
528ec5ab
MC
521 dm_stats_account_io(&md->stats, bio_data_dir(bio),
522 bio->bi_iter.bi_sector, bio_sectors(bio),
523 false, 0, &io->stats_aux);
3eaf840e
JNN
524}
525
d221d2e7 526static void end_io_acct(struct dm_io *io)
3eaf840e
JNN
527{
528 struct mapped_device *md = io->md;
529 struct bio *bio = io->bio;
530 unsigned long duration = jiffies - io->start_time;
18c0b223 531 int pending;
3eaf840e
JNN
532 int rw = bio_data_dir(bio);
533
18c0b223 534 generic_end_io_acct(rw, &dm_disk(md)->part0, io->start_time);
3eaf840e 535
fd2ed4d2 536 if (unlikely(dm_stats_used(&md->stats)))
528ec5ab
MC
537 dm_stats_account_io(&md->stats, bio_data_dir(bio),
538 bio->bi_iter.bi_sector, bio_sectors(bio),
539 true, duration, &io->stats_aux);
fd2ed4d2 540
af7e466a
MP
541 /*
542 * After this is decremented the bio must not be touched if it is
d87f4c14 543 * a flush.
af7e466a 544 */
1e9bb880
SL
545 pending = atomic_dec_return(&md->pending[rw]);
546 atomic_set(&dm_disk(md)->part0.in_flight[rw], pending);
316d315b 547 pending += atomic_read(&md->pending[rw^0x1]);
3eaf840e 548
d221d2e7
MP
549 /* nudge anyone waiting on suspend queue */
550 if (!pending)
551 wake_up(&md->wait);
3eaf840e
JNN
552}
553
1da177e4
LT
554/*
555 * Add the bio to the list of deferred io.
556 */
92c63902 557static void queue_io(struct mapped_device *md, struct bio *bio)
1da177e4 558{
05447420 559 unsigned long flags;
1da177e4 560
05447420 561 spin_lock_irqsave(&md->deferred_lock, flags);
1da177e4 562 bio_list_add(&md->deferred, bio);
05447420 563 spin_unlock_irqrestore(&md->deferred_lock, flags);
6a8736d1 564 queue_work(md->wq, &md->work);
1da177e4
LT
565}
566
567/*
568 * Everyone (including functions in this file), should use this
569 * function to access the md->map field, and make sure they call
83d5e5b0 570 * dm_put_live_table() when finished.
1da177e4 571 */
83d5e5b0 572struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier)
1da177e4 573{
83d5e5b0
MP
574 *srcu_idx = srcu_read_lock(&md->io_barrier);
575
576 return srcu_dereference(md->map, &md->io_barrier);
577}
1da177e4 578
83d5e5b0
MP
579void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier)
580{
581 srcu_read_unlock(&md->io_barrier, srcu_idx);
582}
583
584void dm_sync_table(struct mapped_device *md)
585{
586 synchronize_srcu(&md->io_barrier);
587 synchronize_rcu_expedited();
588}
589
590/*
591 * A fast alternative to dm_get_live_table/dm_put_live_table.
592 * The caller must not block between these two functions.
593 */
594static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU)
595{
596 rcu_read_lock();
597 return rcu_dereference(md->map);
598}
1da177e4 599
83d5e5b0
MP
600static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)
601{
602 rcu_read_unlock();
1da177e4
LT
603}
604
86f1152b
BM
605/*
606 * Open a table device so we can use it as a map destination.
607 */
608static int open_table_device(struct table_device *td, dev_t dev,
609 struct mapped_device *md)
610{
611 static char *_claim_ptr = "I belong to device-mapper";
612 struct block_device *bdev;
613
614 int r;
615
616 BUG_ON(td->dm_dev.bdev);
617
618 bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _claim_ptr);
619 if (IS_ERR(bdev))
620 return PTR_ERR(bdev);
621
622 r = bd_link_disk_holder(bdev, dm_disk(md));
623 if (r) {
624 blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL);
625 return r;
626 }
627
628 td->dm_dev.bdev = bdev;
629 return 0;
630}
631
632/*
633 * Close a table device that we've been using.
634 */
635static void close_table_device(struct table_device *td, struct mapped_device *md)
636{
637 if (!td->dm_dev.bdev)
638 return;
639
640 bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md));
641 blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL);
642 td->dm_dev.bdev = NULL;
643}
644
645static struct table_device *find_table_device(struct list_head *l, dev_t dev,
646 fmode_t mode) {
647 struct table_device *td;
648
649 list_for_each_entry(td, l, list)
650 if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode)
651 return td;
652
653 return NULL;
654}
655
656int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode,
657 struct dm_dev **result) {
658 int r;
659 struct table_device *td;
660
661 mutex_lock(&md->table_devices_lock);
662 td = find_table_device(&md->table_devices, dev, mode);
663 if (!td) {
115485e8 664 td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id);
86f1152b
BM
665 if (!td) {
666 mutex_unlock(&md->table_devices_lock);
667 return -ENOMEM;
668 }
669
670 td->dm_dev.mode = mode;
671 td->dm_dev.bdev = NULL;
672
673 if ((r = open_table_device(td, dev, md))) {
674 mutex_unlock(&md->table_devices_lock);
675 kfree(td);
676 return r;
677 }
678
679 format_dev_t(td->dm_dev.name, dev);
680
681 atomic_set(&td->count, 0);
682 list_add(&td->list, &md->table_devices);
683 }
684 atomic_inc(&td->count);
685 mutex_unlock(&md->table_devices_lock);
686
687 *result = &td->dm_dev;
688 return 0;
689}
690EXPORT_SYMBOL_GPL(dm_get_table_device);
691
692void dm_put_table_device(struct mapped_device *md, struct dm_dev *d)
693{
694 struct table_device *td = container_of(d, struct table_device, dm_dev);
695
696 mutex_lock(&md->table_devices_lock);
697 if (atomic_dec_and_test(&td->count)) {
698 close_table_device(td, md);
699 list_del(&td->list);
700 kfree(td);
701 }
702 mutex_unlock(&md->table_devices_lock);
703}
704EXPORT_SYMBOL(dm_put_table_device);
705
706static void free_table_devices(struct list_head *devices)
707{
708 struct list_head *tmp, *next;
709
710 list_for_each_safe(tmp, next, devices) {
711 struct table_device *td = list_entry(tmp, struct table_device, list);
712
713 DMWARN("dm_destroy: %s still exists with %d references",
714 td->dm_dev.name, atomic_read(&td->count));
715 kfree(td);
716 }
717}
718
3ac51e74
DW
719/*
720 * Get the geometry associated with a dm device
721 */
722int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
723{
724 *geo = md->geometry;
725
726 return 0;
727}
728
729/*
730 * Set the geometry of a device.
731 */
732int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
733{
734 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
735
736 if (geo->start > sz) {
737 DMWARN("Start sector is beyond the geometry limits.");
738 return -EINVAL;
739 }
740
741 md->geometry = *geo;
742
743 return 0;
744}
745
1da177e4
LT
746/*-----------------------------------------------------------------
747 * CRUD START:
748 * A more elegant soln is in the works that uses the queue
749 * merge fn, unfortunately there are a couple of changes to
750 * the block layer that I want to make for this. So in the
751 * interests of getting something for people to use I give
752 * you this clearly demarcated crap.
753 *---------------------------------------------------------------*/
754
2e93ccc1
KU
755static int __noflush_suspending(struct mapped_device *md)
756{
757 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
758}
759
1da177e4
LT
760/*
761 * Decrements the number of outstanding ios that a bio has been
762 * cloned into, completing the original io if necc.
763 */
858119e1 764static void dec_pending(struct dm_io *io, int error)
1da177e4 765{
2e93ccc1 766 unsigned long flags;
b35f8caa
MB
767 int io_error;
768 struct bio *bio;
769 struct mapped_device *md = io->md;
2e93ccc1
KU
770
771 /* Push-back supersedes any I/O errors */
f88fb981
KU
772 if (unlikely(error)) {
773 spin_lock_irqsave(&io->endio_lock, flags);
774 if (!(io->error > 0 && __noflush_suspending(md)))
775 io->error = error;
776 spin_unlock_irqrestore(&io->endio_lock, flags);
777 }
1da177e4
LT
778
779 if (atomic_dec_and_test(&io->io_count)) {
2e93ccc1
KU
780 if (io->error == DM_ENDIO_REQUEUE) {
781 /*
782 * Target requested pushing back the I/O.
2e93ccc1 783 */
022c2611 784 spin_lock_irqsave(&md->deferred_lock, flags);
6a8736d1
TH
785 if (__noflush_suspending(md))
786 bio_list_add_head(&md->deferred, io->bio);
787 else
2e93ccc1
KU
788 /* noflush suspend was interrupted. */
789 io->error = -EIO;
022c2611 790 spin_unlock_irqrestore(&md->deferred_lock, flags);
2e93ccc1
KU
791 }
792
b35f8caa
MB
793 io_error = io->error;
794 bio = io->bio;
6a8736d1
TH
795 end_io_acct(io);
796 free_io(md, io);
797
798 if (io_error == DM_ENDIO_REQUEUE)
799 return;
2e93ccc1 800
1eff9d32 801 if ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size) {
af7e466a 802 /*
6a8736d1 803 * Preflush done for flush with data, reissue
28a8f0d3 804 * without REQ_PREFLUSH.
af7e466a 805 */
1eff9d32 806 bio->bi_opf &= ~REQ_PREFLUSH;
6a8736d1 807 queue_io(md, bio);
af7e466a 808 } else {
b372d360 809 /* done with normal IO or empty flush */
0a82a8d1 810 trace_block_bio_complete(md->queue, bio, io_error);
4246a0b6
CH
811 bio->bi_error = io_error;
812 bio_endio(bio);
b35f8caa 813 }
1da177e4
LT
814 }
815}
816
4cc96131 817void disable_write_same(struct mapped_device *md)
7eee4ae2
MS
818{
819 struct queue_limits *limits = dm_get_queue_limits(md);
820
821 /* device doesn't really support WRITE SAME, disable it */
822 limits->max_write_same_sectors = 0;
823}
824
4246a0b6 825static void clone_endio(struct bio *bio)
1da177e4 826{
4246a0b6 827 int error = bio->bi_error;
5164bece 828 int r = error;
bfc6d41c 829 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
b35f8caa 830 struct dm_io *io = tio->io;
9faf400f 831 struct mapped_device *md = tio->io->md;
1da177e4
LT
832 dm_endio_fn endio = tio->ti->type->end_io;
833
1da177e4 834 if (endio) {
7de3ee57 835 r = endio(tio->ti, bio, error);
2e93ccc1
KU
836 if (r < 0 || r == DM_ENDIO_REQUEUE)
837 /*
838 * error and requeue request are handled
839 * in dec_pending().
840 */
1da177e4 841 error = r;
45cbcd79
KU
842 else if (r == DM_ENDIO_INCOMPLETE)
843 /* The target will handle the io */
6712ecf8 844 return;
45cbcd79
KU
845 else if (r) {
846 DMWARN("unimplemented target endio return value: %d", r);
847 BUG();
848 }
1da177e4
LT
849 }
850
e6047149 851 if (unlikely(r == -EREMOTEIO && (bio_op(bio) == REQ_OP_WRITE_SAME) &&
7eee4ae2
MS
852 !bdev_get_queue(bio->bi_bdev)->limits.max_write_same_sectors))
853 disable_write_same(md);
854
cfae7529 855 free_tio(tio);
b35f8caa 856 dec_pending(io, error);
1da177e4
LT
857}
858
56a67df7
MS
859/*
860 * Return maximum size of I/O possible at the supplied sector up to the current
861 * target boundary.
862 */
863static sector_t max_io_len_target_boundary(sector_t sector, struct dm_target *ti)
864{
865 sector_t target_offset = dm_target_offset(ti, sector);
866
867 return ti->len - target_offset;
868}
869
870static sector_t max_io_len(sector_t sector, struct dm_target *ti)
1da177e4 871{
56a67df7 872 sector_t len = max_io_len_target_boundary(sector, ti);
542f9038 873 sector_t offset, max_len;
1da177e4
LT
874
875 /*
542f9038 876 * Does the target need to split even further?
1da177e4 877 */
542f9038
MS
878 if (ti->max_io_len) {
879 offset = dm_target_offset(ti, sector);
880 if (unlikely(ti->max_io_len & (ti->max_io_len - 1)))
881 max_len = sector_div(offset, ti->max_io_len);
882 else
883 max_len = offset & (ti->max_io_len - 1);
884 max_len = ti->max_io_len - max_len;
885
886 if (len > max_len)
887 len = max_len;
1da177e4
LT
888 }
889
890 return len;
891}
892
542f9038
MS
893int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
894{
895 if (len > UINT_MAX) {
896 DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)",
897 (unsigned long long)len, UINT_MAX);
898 ti->error = "Maximum size of target IO is too large";
899 return -EINVAL;
900 }
901
902 ti->max_io_len = (uint32_t) len;
903
904 return 0;
905}
906EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);
907
545ed20e 908static long dm_blk_direct_access(struct block_device *bdev, sector_t sector,
f0c98ebc 909 void **kaddr, pfn_t *pfn, long size)
545ed20e
TK
910{
911 struct mapped_device *md = bdev->bd_disk->private_data;
912 struct dm_table *map;
913 struct dm_target *ti;
914 int srcu_idx;
915 long len, ret = -EIO;
916
917 map = dm_get_live_table(md, &srcu_idx);
918 if (!map)
919 goto out;
920
921 ti = dm_table_find_target(map, sector);
922 if (!dm_target_is_valid(ti))
923 goto out;
924
925 len = max_io_len(sector, ti) << SECTOR_SHIFT;
926 size = min(len, size);
927
928 if (ti->type->direct_access)
929 ret = ti->type->direct_access(ti, sector, kaddr, pfn, size);
930out:
931 dm_put_live_table(md, srcu_idx);
932 return min(ret, size);
933}
934
1dd40c3e
MP
935/*
936 * A target may call dm_accept_partial_bio only from the map routine. It is
28a8f0d3 937 * allowed for all bio types except REQ_PREFLUSH.
1dd40c3e
MP
938 *
939 * dm_accept_partial_bio informs the dm that the target only wants to process
940 * additional n_sectors sectors of the bio and the rest of the data should be
941 * sent in a next bio.
942 *
943 * A diagram that explains the arithmetics:
944 * +--------------------+---------------+-------+
945 * | 1 | 2 | 3 |
946 * +--------------------+---------------+-------+
947 *
948 * <-------------- *tio->len_ptr --------------->
949 * <------- bi_size ------->
950 * <-- n_sectors -->
951 *
952 * Region 1 was already iterated over with bio_advance or similar function.
953 * (it may be empty if the target doesn't use bio_advance)
954 * Region 2 is the remaining bio size that the target wants to process.
955 * (it may be empty if region 1 is non-empty, although there is no reason
956 * to make it empty)
957 * The target requires that region 3 is to be sent in the next bio.
958 *
959 * If the target wants to receive multiple copies of the bio (via num_*bios, etc),
960 * the partially processed part (the sum of regions 1+2) must be the same for all
961 * copies of the bio.
962 */
963void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors)
964{
965 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
966 unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT;
1eff9d32 967 BUG_ON(bio->bi_opf & REQ_PREFLUSH);
1dd40c3e
MP
968 BUG_ON(bi_size > *tio->len_ptr);
969 BUG_ON(n_sectors > bi_size);
970 *tio->len_ptr -= bi_size - n_sectors;
971 bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
972}
973EXPORT_SYMBOL_GPL(dm_accept_partial_bio);
974
bd2a49b8 975static void __map_bio(struct dm_target_io *tio)
1da177e4
LT
976{
977 int r;
2056a782 978 sector_t sector;
dba14160 979 struct bio *clone = &tio->clone;
bd2a49b8 980 struct dm_target *ti = tio->ti;
1da177e4 981
1da177e4 982 clone->bi_end_io = clone_endio;
1da177e4
LT
983
984 /*
985 * Map the clone. If r == 0 we don't need to do
986 * anything, the target has assumed ownership of
987 * this io.
988 */
989 atomic_inc(&tio->io->io_count);
4f024f37 990 sector = clone->bi_iter.bi_sector;
7de3ee57 991 r = ti->type->map(ti, clone);
45cbcd79 992 if (r == DM_MAPIO_REMAPPED) {
1da177e4 993 /* the bio has been remapped so dispatch it */
2056a782 994
d07335e5
MS
995 trace_block_bio_remap(bdev_get_queue(clone->bi_bdev), clone,
996 tio->io->bio->bi_bdev->bd_dev, sector);
2056a782 997
1da177e4 998 generic_make_request(clone);
2e93ccc1
KU
999 } else if (r < 0 || r == DM_MAPIO_REQUEUE) {
1000 /* error the io and bail out, or requeue it if needed */
9faf400f 1001 dec_pending(tio->io, r);
cfae7529 1002 free_tio(tio);
ab37844d 1003 } else if (r != DM_MAPIO_SUBMITTED) {
45cbcd79
KU
1004 DMWARN("unimplemented target map return value: %d", r);
1005 BUG();
1da177e4
LT
1006 }
1007}
1008
1009struct clone_info {
1010 struct mapped_device *md;
1011 struct dm_table *map;
1012 struct bio *bio;
1013 struct dm_io *io;
1014 sector_t sector;
e0d6609a 1015 unsigned sector_count;
1da177e4
LT
1016};
1017
e0d6609a 1018static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len)
bd2a49b8 1019{
4f024f37
KO
1020 bio->bi_iter.bi_sector = sector;
1021 bio->bi_iter.bi_size = to_bytes(len);
1da177e4
LT
1022}
1023
1024/*
1025 * Creates a bio that consists of range of complete bvecs.
1026 */
c80914e8
MS
1027static int clone_bio(struct dm_target_io *tio, struct bio *bio,
1028 sector_t sector, unsigned len)
1da177e4 1029{
dba14160 1030 struct bio *clone = &tio->clone;
1da177e4 1031
1c3b13e6
KO
1032 __bio_clone_fast(clone, bio);
1033
c80914e8
MS
1034 if (bio_integrity(bio)) {
1035 int r = bio_integrity_clone(clone, bio, GFP_NOIO);
1036 if (r < 0)
1037 return r;
1038 }
bd2a49b8 1039
1c3b13e6
KO
1040 bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector));
1041 clone->bi_iter.bi_size = to_bytes(len);
1042
1043 if (bio_integrity(bio))
1044 bio_integrity_trim(clone, 0, len);
c80914e8
MS
1045
1046 return 0;
1da177e4
LT
1047}
1048
9015df24 1049static struct dm_target_io *alloc_tio(struct clone_info *ci,
99778273 1050 struct dm_target *ti,
55a62eef 1051 unsigned target_bio_nr)
f9ab94ce 1052{
dba14160
MP
1053 struct dm_target_io *tio;
1054 struct bio *clone;
1055
99778273 1056 clone = bio_alloc_bioset(GFP_NOIO, 0, ci->md->bs);
dba14160 1057 tio = container_of(clone, struct dm_target_io, clone);
f9ab94ce
MP
1058
1059 tio->io = ci->io;
1060 tio->ti = ti;
55a62eef 1061 tio->target_bio_nr = target_bio_nr;
9015df24
AK
1062
1063 return tio;
1064}
1065
14fe594d
AK
1066static void __clone_and_map_simple_bio(struct clone_info *ci,
1067 struct dm_target *ti,
1dd40c3e 1068 unsigned target_bio_nr, unsigned *len)
9015df24 1069{
99778273 1070 struct dm_target_io *tio = alloc_tio(ci, ti, target_bio_nr);
dba14160 1071 struct bio *clone = &tio->clone;
9015df24 1072
1dd40c3e
MP
1073 tio->len_ptr = len;
1074
99778273 1075 __bio_clone_fast(clone, ci->bio);
bd2a49b8 1076 if (len)
1dd40c3e 1077 bio_setup_sector(clone, ci->sector, *len);
f9ab94ce 1078
bd2a49b8 1079 __map_bio(tio);
f9ab94ce
MP
1080}
1081
14fe594d 1082static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,
1dd40c3e 1083 unsigned num_bios, unsigned *len)
06a426ce 1084{
55a62eef 1085 unsigned target_bio_nr;
06a426ce 1086
55a62eef 1087 for (target_bio_nr = 0; target_bio_nr < num_bios; target_bio_nr++)
14fe594d 1088 __clone_and_map_simple_bio(ci, ti, target_bio_nr, len);
06a426ce
MS
1089}
1090
14fe594d 1091static int __send_empty_flush(struct clone_info *ci)
f9ab94ce 1092{
06a426ce 1093 unsigned target_nr = 0;
f9ab94ce
MP
1094 struct dm_target *ti;
1095
b372d360 1096 BUG_ON(bio_has_data(ci->bio));
f9ab94ce 1097 while ((ti = dm_table_get_target(ci->map, target_nr++)))
1dd40c3e 1098 __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL);
f9ab94ce 1099
f9ab94ce
MP
1100 return 0;
1101}
1102
c80914e8 1103static int __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti,
1dd40c3e 1104 sector_t sector, unsigned *len)
5ae89a87 1105{
dba14160 1106 struct bio *bio = ci->bio;
5ae89a87 1107 struct dm_target_io *tio;
b0d8ed4d
AK
1108 unsigned target_bio_nr;
1109 unsigned num_target_bios = 1;
c80914e8 1110 int r = 0;
5ae89a87 1111
b0d8ed4d
AK
1112 /*
1113 * Does the target want to receive duplicate copies of the bio?
1114 */
1115 if (bio_data_dir(bio) == WRITE && ti->num_write_bios)
1116 num_target_bios = ti->num_write_bios(ti, bio);
e4c93811 1117
b0d8ed4d 1118 for (target_bio_nr = 0; target_bio_nr < num_target_bios; target_bio_nr++) {
99778273 1119 tio = alloc_tio(ci, ti, target_bio_nr);
1dd40c3e 1120 tio->len_ptr = len;
c80914e8 1121 r = clone_bio(tio, bio, sector, *len);
072623de 1122 if (r < 0) {
cfae7529 1123 free_tio(tio);
c80914e8 1124 break;
072623de 1125 }
b0d8ed4d
AK
1126 __map_bio(tio);
1127 }
c80914e8
MS
1128
1129 return r;
5ae89a87
MS
1130}
1131
55a62eef 1132typedef unsigned (*get_num_bios_fn)(struct dm_target *ti);
23508a96 1133
55a62eef 1134static unsigned get_num_discard_bios(struct dm_target *ti)
23508a96 1135{
55a62eef 1136 return ti->num_discard_bios;
23508a96
MS
1137}
1138
55a62eef 1139static unsigned get_num_write_same_bios(struct dm_target *ti)
23508a96 1140{
55a62eef 1141 return ti->num_write_same_bios;
23508a96
MS
1142}
1143
1144typedef bool (*is_split_required_fn)(struct dm_target *ti);
9eef87da 1145
23508a96
MS
1146static bool is_split_required_for_discard(struct dm_target *ti)
1147{
55a62eef 1148 return ti->split_discard_bios;
cec47e3d
KU
1149}
1150
14fe594d
AK
1151static int __send_changing_extent_only(struct clone_info *ci,
1152 get_num_bios_fn get_num_bios,
1153 is_split_required_fn is_split_required)
ba1cbad9 1154{
5ae89a87 1155 struct dm_target *ti;
e0d6609a 1156 unsigned len;
55a62eef 1157 unsigned num_bios;
ba1cbad9 1158
a79245b3
MS
1159 do {
1160 ti = dm_table_find_target(ci->map, ci->sector);
1161 if (!dm_target_is_valid(ti))
1162 return -EIO;
2eb6e1e3 1163
5ae89a87 1164 /*
23508a96
MS
1165 * Even though the device advertised support for this type of
1166 * request, that does not mean every target supports it, and
936688d7 1167 * reconfiguration might also have changed that since the
a79245b3 1168 * check was performed.
5ae89a87 1169 */
55a62eef
AK
1170 num_bios = get_num_bios ? get_num_bios(ti) : 0;
1171 if (!num_bios)
a79245b3 1172 return -EOPNOTSUPP;
ba1cbad9 1173
23508a96 1174 if (is_split_required && !is_split_required(ti))
e0d6609a 1175 len = min((sector_t)ci->sector_count, max_io_len_target_boundary(ci->sector, ti));
7acf0277 1176 else
e0d6609a 1177 len = min((sector_t)ci->sector_count, max_io_len(ci->sector, ti));
de3ec86d 1178
1dd40c3e 1179 __send_duplicate_bios(ci, ti, num_bios, &len);
e262f347 1180
a79245b3
MS
1181 ci->sector += len;
1182 } while (ci->sector_count -= len);
5ae89a87
MS
1183
1184 return 0;
ba1cbad9
MS
1185}
1186
14fe594d 1187static int __send_discard(struct clone_info *ci)
23508a96 1188{
14fe594d
AK
1189 return __send_changing_extent_only(ci, get_num_discard_bios,
1190 is_split_required_for_discard);
23508a96 1191}
0ce65797 1192
14fe594d 1193static int __send_write_same(struct clone_info *ci)
0ce65797 1194{
14fe594d 1195 return __send_changing_extent_only(ci, get_num_write_same_bios, NULL);
0ce65797
MS
1196}
1197
e4c93811
AK
1198/*
1199 * Select the correct strategy for processing a non-flush bio.
1200 */
14fe594d 1201static int __split_and_process_non_flush(struct clone_info *ci)
0ce65797 1202{
dba14160 1203 struct bio *bio = ci->bio;
512875bd 1204 struct dm_target *ti;
1c3b13e6 1205 unsigned len;
c80914e8 1206 int r;
0ce65797 1207
e6047149 1208 if (unlikely(bio_op(bio) == REQ_OP_DISCARD))
14fe594d 1209 return __send_discard(ci);
e6047149 1210 else if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME))
14fe594d 1211 return __send_write_same(ci);
0ce65797 1212
512875bd
JN
1213 ti = dm_table_find_target(ci->map, ci->sector);
1214 if (!dm_target_is_valid(ti))
1215 return -EIO;
1216
1c3b13e6 1217 len = min_t(sector_t, max_io_len(ci->sector, ti), ci->sector_count);
0ce65797 1218
c80914e8
MS
1219 r = __clone_and_map_data_bio(ci, ti, ci->sector, &len);
1220 if (r < 0)
1221 return r;
0ce65797 1222
1c3b13e6
KO
1223 ci->sector += len;
1224 ci->sector_count -= len;
0ce65797 1225
1c3b13e6 1226 return 0;
0ce65797
MS
1227}
1228
1da177e4 1229/*
14fe594d 1230 * Entry point to split a bio into clones and submit them to the targets.
1da177e4 1231 */
83d5e5b0
MP
1232static void __split_and_process_bio(struct mapped_device *md,
1233 struct dm_table *map, struct bio *bio)
0ce65797 1234{
1da177e4 1235 struct clone_info ci;
512875bd 1236 int error = 0;
1da177e4 1237
83d5e5b0 1238 if (unlikely(!map)) {
6a8736d1 1239 bio_io_error(bio);
f0b9a450
MP
1240 return;
1241 }
692d0eb9 1242
83d5e5b0 1243 ci.map = map;
1da177e4 1244 ci.md = md;
1da177e4
LT
1245 ci.io = alloc_io(md);
1246 ci.io->error = 0;
1247 atomic_set(&ci.io->io_count, 1);
1248 ci.io->bio = bio;
1249 ci.io->md = md;
f88fb981 1250 spin_lock_init(&ci.io->endio_lock);
4f024f37 1251 ci.sector = bio->bi_iter.bi_sector;
0ce65797 1252
3eaf840e 1253 start_io_acct(ci.io);
0ce65797 1254
1eff9d32 1255 if (bio->bi_opf & REQ_PREFLUSH) {
b372d360
MS
1256 ci.bio = &ci.md->flush_bio;
1257 ci.sector_count = 0;
14fe594d 1258 error = __send_empty_flush(&ci);
b372d360
MS
1259 /* dec_pending submits any data associated with flush */
1260 } else {
6a8736d1 1261 ci.bio = bio;
d87f4c14 1262 ci.sector_count = bio_sectors(bio);
b372d360 1263 while (ci.sector_count && !error)
14fe594d 1264 error = __split_and_process_non_flush(&ci);
d87f4c14 1265 }
0ce65797 1266
1da177e4 1267 /* drop the extra reference count */
512875bd 1268 dec_pending(ci.io, error);
0ce65797 1269}
1da177e4
LT
1270/*-----------------------------------------------------------------
1271 * CRUD END
1272 *---------------------------------------------------------------*/
0ce65797 1273
cec47e3d 1274/*
1da177e4
LT
1275 * The request function that just remaps the bio built up by
1276 * dm_merge_bvec.
cec47e3d 1277 */
dece1635 1278static blk_qc_t dm_make_request(struct request_queue *q, struct bio *bio)
cec47e3d 1279{
12f03a49 1280 int rw = bio_data_dir(bio);
cec47e3d 1281 struct mapped_device *md = q->queuedata;
83d5e5b0
MP
1282 int srcu_idx;
1283 struct dm_table *map;
cec47e3d 1284
83d5e5b0 1285 map = dm_get_live_table(md, &srcu_idx);
29e4013d 1286
18c0b223 1287 generic_start_io_acct(rw, bio_sectors(bio), &dm_disk(md)->part0);
d0bcb878 1288
6a8736d1
TH
1289 /* if we're suspended, we have to queue this io for later */
1290 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
83d5e5b0 1291 dm_put_live_table(md, srcu_idx);
9eef87da 1292
1eff9d32 1293 if (!(bio->bi_opf & REQ_RAHEAD))
6a8736d1
TH
1294 queue_io(md, bio);
1295 else
54d9a1b4 1296 bio_io_error(bio);
dece1635 1297 return BLK_QC_T_NONE;
cec47e3d 1298 }
1da177e4 1299
83d5e5b0
MP
1300 __split_and_process_bio(md, map, bio);
1301 dm_put_live_table(md, srcu_idx);
dece1635 1302 return BLK_QC_T_NONE;
cec47e3d
KU
1303}
1304
1da177e4
LT
1305static int dm_any_congested(void *congested_data, int bdi_bits)
1306{
8a57dfc6
CS
1307 int r = bdi_bits;
1308 struct mapped_device *md = congested_data;
1309 struct dm_table *map;
1da177e4 1310
1eb787ec 1311 if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
e522c039 1312 if (dm_request_based(md)) {
cec47e3d 1313 /*
e522c039
MS
1314 * With request-based DM we only need to check the
1315 * top-level queue for congestion.
cec47e3d 1316 */
e522c039
MS
1317 r = md->queue->backing_dev_info.wb.state & bdi_bits;
1318 } else {
1319 map = dm_get_live_table_fast(md);
1320 if (map)
cec47e3d 1321 r = dm_table_any_congested(map, bdi_bits);
e522c039 1322 dm_put_live_table_fast(md);
8a57dfc6
CS
1323 }
1324 }
1325
1da177e4
LT
1326 return r;
1327}
1328
1329/*-----------------------------------------------------------------
1330 * An IDR is used to keep track of allocated minor numbers.
1331 *---------------------------------------------------------------*/
2b06cfff 1332static void free_minor(int minor)
1da177e4 1333{
f32c10b0 1334 spin_lock(&_minor_lock);
1da177e4 1335 idr_remove(&_minor_idr, minor);
f32c10b0 1336 spin_unlock(&_minor_lock);
1da177e4
LT
1337}
1338
1339/*
1340 * See if the device with a specific minor # is free.
1341 */
cf13ab8e 1342static int specific_minor(int minor)
1da177e4 1343{
c9d76be6 1344 int r;
1da177e4
LT
1345
1346 if (minor >= (1 << MINORBITS))
1347 return -EINVAL;
1348
c9d76be6 1349 idr_preload(GFP_KERNEL);
f32c10b0 1350 spin_lock(&_minor_lock);
1da177e4 1351
c9d76be6 1352 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT);
1da177e4 1353
f32c10b0 1354 spin_unlock(&_minor_lock);
c9d76be6
TH
1355 idr_preload_end();
1356 if (r < 0)
1357 return r == -ENOSPC ? -EBUSY : r;
1358 return 0;
1da177e4
LT
1359}
1360
cf13ab8e 1361static int next_free_minor(int *minor)
1da177e4 1362{
c9d76be6 1363 int r;
62f75c2f 1364
c9d76be6 1365 idr_preload(GFP_KERNEL);
f32c10b0 1366 spin_lock(&_minor_lock);
1da177e4 1367
c9d76be6 1368 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT);
1da177e4 1369
f32c10b0 1370 spin_unlock(&_minor_lock);
c9d76be6
TH
1371 idr_preload_end();
1372 if (r < 0)
1373 return r;
1374 *minor = r;
1375 return 0;
1da177e4
LT
1376}
1377
83d5cde4 1378static const struct block_device_operations dm_blk_dops;
1da177e4 1379
53d5914f
MP
1380static void dm_wq_work(struct work_struct *work);
1381
4cc96131 1382void dm_init_md_queue(struct mapped_device *md)
4a0b4ddf
MS
1383{
1384 /*
1385 * Request-based dm devices cannot be stacked on top of bio-based dm
bfebd1cd 1386 * devices. The type of this dm device may not have been decided yet.
4a0b4ddf
MS
1387 * The type is decided at the first table loading time.
1388 * To prevent problematic device stacking, clear the queue flag
1389 * for request stacking support until then.
1390 *
1391 * This queue is new, so no concurrency on the queue_flags.
1392 */
1393 queue_flag_clear_unlocked(QUEUE_FLAG_STACKABLE, md->queue);
ad5f498f
MP
1394
1395 /*
1396 * Initialize data that will only be used by a non-blk-mq DM queue
1397 * - must do so here (in alloc_dev callchain) before queue is used
1398 */
1399 md->queue->queuedata = md;
1400 md->queue->backing_dev_info.congested_data = md;
bfebd1cd 1401}
4a0b4ddf 1402
4cc96131 1403void dm_init_normal_md_queue(struct mapped_device *md)
bfebd1cd 1404{
17e149b8 1405 md->use_blk_mq = false;
bfebd1cd
MS
1406 dm_init_md_queue(md);
1407
1408 /*
1409 * Initialize aspects of queue that aren't relevant for blk-mq
1410 */
4a0b4ddf 1411 md->queue->backing_dev_info.congested_fn = dm_any_congested;
4a0b4ddf 1412 blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
4a0b4ddf
MS
1413}
1414
0f20972f
MS
1415static void cleanup_mapped_device(struct mapped_device *md)
1416{
0f20972f
MS
1417 if (md->wq)
1418 destroy_workqueue(md->wq);
1419 if (md->kworker_task)
1420 kthread_stop(md->kworker_task);
6f65985e
JL
1421 mempool_destroy(md->io_pool);
1422 mempool_destroy(md->rq_pool);
0f20972f
MS
1423 if (md->bs)
1424 bioset_free(md->bs);
1425
1426 if (md->disk) {
1427 spin_lock(&_minor_lock);
1428 md->disk->private_data = NULL;
1429 spin_unlock(&_minor_lock);
0f20972f
MS
1430 del_gendisk(md->disk);
1431 put_disk(md->disk);
1432 }
1433
1434 if (md->queue)
1435 blk_cleanup_queue(md->queue);
1436
d09960b0
TE
1437 cleanup_srcu_struct(&md->io_barrier);
1438
0f20972f
MS
1439 if (md->bdev) {
1440 bdput(md->bdev);
1441 md->bdev = NULL;
1442 }
4cc96131
MS
1443
1444 dm_mq_cleanup_mapped_device(md);
0f20972f
MS
1445}
1446
1da177e4
LT
1447/*
1448 * Allocate and initialise a blank device with a given minor.
1449 */
2b06cfff 1450static struct mapped_device *alloc_dev(int minor)
1da177e4 1451{
115485e8
MS
1452 int r, numa_node_id = dm_get_numa_node();
1453 struct mapped_device *md;
ba61fdd1 1454 void *old_md;
1da177e4 1455
115485e8 1456 md = kzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id);
1da177e4
LT
1457 if (!md) {
1458 DMWARN("unable to allocate device, out of memory.");
1459 return NULL;
1460 }
1461
10da4f79 1462 if (!try_module_get(THIS_MODULE))
6ed7ade8 1463 goto bad_module_get;
10da4f79 1464
1da177e4 1465 /* get a minor number for the dev */
2b06cfff 1466 if (minor == DM_ANY_MINOR)
cf13ab8e 1467 r = next_free_minor(&minor);
2b06cfff 1468 else
cf13ab8e 1469 r = specific_minor(minor);
1da177e4 1470 if (r < 0)
6ed7ade8 1471 goto bad_minor;
1da177e4 1472
83d5e5b0
MP
1473 r = init_srcu_struct(&md->io_barrier);
1474 if (r < 0)
1475 goto bad_io_barrier;
1476
115485e8 1477 md->numa_node_id = numa_node_id;
4cc96131 1478 md->use_blk_mq = dm_use_blk_mq_default();
591ddcfc 1479 md->init_tio_pdu = false;
a5664dad 1480 md->type = DM_TYPE_NONE;
e61290a4 1481 mutex_init(&md->suspend_lock);
a5664dad 1482 mutex_init(&md->type_lock);
86f1152b 1483 mutex_init(&md->table_devices_lock);
022c2611 1484 spin_lock_init(&md->deferred_lock);
1da177e4 1485 atomic_set(&md->holders, 1);
5c6bd75d 1486 atomic_set(&md->open_count, 0);
1da177e4 1487 atomic_set(&md->event_nr, 0);
7a8c3d3b
MA
1488 atomic_set(&md->uevent_seq, 0);
1489 INIT_LIST_HEAD(&md->uevent_list);
86f1152b 1490 INIT_LIST_HEAD(&md->table_devices);
7a8c3d3b 1491 spin_lock_init(&md->uevent_lock);
1da177e4 1492
115485e8 1493 md->queue = blk_alloc_queue_node(GFP_KERNEL, numa_node_id);
1da177e4 1494 if (!md->queue)
0f20972f 1495 goto bad;
1da177e4 1496
4a0b4ddf 1497 dm_init_md_queue(md);
9faf400f 1498
115485e8 1499 md->disk = alloc_disk_node(1, numa_node_id);
1da177e4 1500 if (!md->disk)
0f20972f 1501 goto bad;
1da177e4 1502
316d315b
NK
1503 atomic_set(&md->pending[0], 0);
1504 atomic_set(&md->pending[1], 0);
f0b04115 1505 init_waitqueue_head(&md->wait);
53d5914f 1506 INIT_WORK(&md->work, dm_wq_work);
f0b04115 1507 init_waitqueue_head(&md->eventq);
2995fa78 1508 init_completion(&md->kobj_holder.completion);
2eb6e1e3 1509 md->kworker_task = NULL;
f0b04115 1510
1da177e4
LT
1511 md->disk->major = _major;
1512 md->disk->first_minor = minor;
1513 md->disk->fops = &dm_blk_dops;
1514 md->disk->queue = md->queue;
1515 md->disk->private_data = md;
1516 sprintf(md->disk->disk_name, "dm-%d", minor);
1517 add_disk(md->disk);
7e51f257 1518 format_dev_t(md->name, MKDEV(_major, minor));
1da177e4 1519
670368a8 1520 md->wq = alloc_workqueue("kdmflush", WQ_MEM_RECLAIM, 0);
304f3f6a 1521 if (!md->wq)
0f20972f 1522 goto bad;
304f3f6a 1523
32a926da
MP
1524 md->bdev = bdget_disk(md->disk, 0);
1525 if (!md->bdev)
0f20972f 1526 goto bad;
32a926da 1527
6a8736d1
TH
1528 bio_init(&md->flush_bio);
1529 md->flush_bio.bi_bdev = md->bdev;
e6047149 1530 bio_set_op_attrs(&md->flush_bio, REQ_OP_WRITE, WRITE_FLUSH);
6a8736d1 1531
fd2ed4d2
MP
1532 dm_stats_init(&md->stats);
1533
ba61fdd1 1534 /* Populate the mapping, nobody knows we exist yet */
f32c10b0 1535 spin_lock(&_minor_lock);
ba61fdd1 1536 old_md = idr_replace(&_minor_idr, md, minor);
f32c10b0 1537 spin_unlock(&_minor_lock);
ba61fdd1
JM
1538
1539 BUG_ON(old_md != MINOR_ALLOCED);
1540
1da177e4
LT
1541 return md;
1542
0f20972f
MS
1543bad:
1544 cleanup_mapped_device(md);
83d5e5b0 1545bad_io_barrier:
1da177e4 1546 free_minor(minor);
6ed7ade8 1547bad_minor:
10da4f79 1548 module_put(THIS_MODULE);
6ed7ade8 1549bad_module_get:
1da177e4
LT
1550 kfree(md);
1551 return NULL;
1552}
1553
ae9da83f
JN
1554static void unlock_fs(struct mapped_device *md);
1555
1da177e4
LT
1556static void free_dev(struct mapped_device *md)
1557{
f331c029 1558 int minor = MINOR(disk_devt(md->disk));
63d94e48 1559
32a926da 1560 unlock_fs(md);
2eb6e1e3 1561
0f20972f 1562 cleanup_mapped_device(md);
63a4f065 1563
86f1152b 1564 free_table_devices(&md->table_devices);
63a4f065 1565 dm_stats_cleanup(&md->stats);
63a4f065
MS
1566 free_minor(minor);
1567
10da4f79 1568 module_put(THIS_MODULE);
1da177e4
LT
1569 kfree(md);
1570}
1571
e6ee8c0b
KU
1572static void __bind_mempools(struct mapped_device *md, struct dm_table *t)
1573{
c0820cf5 1574 struct dm_md_mempools *p = dm_table_get_md_mempools(t);
e6ee8c0b 1575
4e6e36c3
MS
1576 if (md->bs) {
1577 /* The md already has necessary mempools. */
545ed20e 1578 if (dm_table_bio_based(t)) {
16245bdc
JN
1579 /*
1580 * Reload bioset because front_pad may have changed
1581 * because a different table was loaded.
1582 */
1583 bioset_free(md->bs);
1584 md->bs = p->bs;
1585 p->bs = NULL;
16245bdc 1586 }
4e6e36c3
MS
1587 /*
1588 * There's no need to reload with request-based dm
1589 * because the size of front_pad doesn't change.
1590 * Note for future: If you are to reload bioset,
1591 * prep-ed requests in the queue may refer
1592 * to bio from the old bioset, so you must walk
1593 * through the queue to unprep.
1594 */
1595 goto out;
c0820cf5 1596 }
e6ee8c0b 1597
cbc4e3c1
MS
1598 BUG_ON(!p || md->io_pool || md->rq_pool || md->bs);
1599
e6ee8c0b
KU
1600 md->io_pool = p->io_pool;
1601 p->io_pool = NULL;
1ae49ea2
MS
1602 md->rq_pool = p->rq_pool;
1603 p->rq_pool = NULL;
e6ee8c0b
KU
1604 md->bs = p->bs;
1605 p->bs = NULL;
4e6e36c3 1606
e6ee8c0b 1607out:
02233342 1608 /* mempool bind completed, no longer need any mempools in the table */
e6ee8c0b
KU
1609 dm_table_free_md_mempools(t);
1610}
1611
1da177e4
LT
1612/*
1613 * Bind a table to the device.
1614 */
1615static void event_callback(void *context)
1616{
7a8c3d3b
MA
1617 unsigned long flags;
1618 LIST_HEAD(uevents);
1da177e4
LT
1619 struct mapped_device *md = (struct mapped_device *) context;
1620
7a8c3d3b
MA
1621 spin_lock_irqsave(&md->uevent_lock, flags);
1622 list_splice_init(&md->uevent_list, &uevents);
1623 spin_unlock_irqrestore(&md->uevent_lock, flags);
1624
ed9e1982 1625 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
7a8c3d3b 1626
1da177e4
LT
1627 atomic_inc(&md->event_nr);
1628 wake_up(&md->eventq);
1629}
1630
c217649b
MS
1631/*
1632 * Protected by md->suspend_lock obtained by dm_swap_table().
1633 */
4e90188b 1634static void __set_size(struct mapped_device *md, sector_t size)
1da177e4 1635{
4e90188b 1636 set_capacity(md->disk, size);
1da177e4 1637
db8fef4f 1638 i_size_write(md->bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
1da177e4
LT
1639}
1640
042d2a9b
AK
1641/*
1642 * Returns old map, which caller must destroy.
1643 */
1644static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
1645 struct queue_limits *limits)
1da177e4 1646{
042d2a9b 1647 struct dm_table *old_map;
165125e1 1648 struct request_queue *q = md->queue;
1da177e4
LT
1649 sector_t size;
1650
5a8f1f80
BVA
1651 lockdep_assert_held(&md->suspend_lock);
1652
1da177e4 1653 size = dm_table_get_size(t);
3ac51e74
DW
1654
1655 /*
1656 * Wipe any geometry if the size of the table changed.
1657 */
fd2ed4d2 1658 if (size != dm_get_size(md))
3ac51e74
DW
1659 memset(&md->geometry, 0, sizeof(md->geometry));
1660
32a926da 1661 __set_size(md, size);
d5816876 1662
2ca3310e
AK
1663 dm_table_event_callback(t, event_callback, md);
1664
e6ee8c0b
KU
1665 /*
1666 * The queue hasn't been stopped yet, if the old table type wasn't
1667 * for request-based during suspension. So stop it to prevent
1668 * I/O mapping before resume.
1669 * This must be done before setting the queue restrictions,
1670 * because request-based dm may be run just after the setting.
1671 */
16f12266 1672 if (dm_table_request_based(t)) {
eca7ee6d 1673 dm_stop_queue(q);
16f12266
MS
1674 /*
1675 * Leverage the fact that request-based DM targets are
1676 * immutable singletons and establish md->immutable_target
1677 * - used to optimize both dm_request_fn and dm_mq_queue_rq
1678 */
1679 md->immutable_target = dm_table_get_immutable_target(t);
1680 }
e6ee8c0b
KU
1681
1682 __bind_mempools(md, t);
1683
a12f5d48 1684 old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
1d3aa6f6 1685 rcu_assign_pointer(md->map, (void *)t);
36a0456f
AK
1686 md->immutable_target_type = dm_table_get_immutable_target_type(t);
1687
754c5fc7 1688 dm_table_set_restrictions(t, q, limits);
41abc4e1
HR
1689 if (old_map)
1690 dm_sync_table(md);
1da177e4 1691
042d2a9b 1692 return old_map;
1da177e4
LT
1693}
1694
a7940155
AK
1695/*
1696 * Returns unbound table for the caller to free.
1697 */
1698static struct dm_table *__unbind(struct mapped_device *md)
1da177e4 1699{
a12f5d48 1700 struct dm_table *map = rcu_dereference_protected(md->map, 1);
1da177e4
LT
1701
1702 if (!map)
a7940155 1703 return NULL;
1da177e4
LT
1704
1705 dm_table_event_callback(map, NULL, NULL);
9cdb8520 1706 RCU_INIT_POINTER(md->map, NULL);
83d5e5b0 1707 dm_sync_table(md);
a7940155
AK
1708
1709 return map;
1da177e4
LT
1710}
1711
1712/*
1713 * Constructor for a new device.
1714 */
2b06cfff 1715int dm_create(int minor, struct mapped_device **result)
1da177e4
LT
1716{
1717 struct mapped_device *md;
1718
2b06cfff 1719 md = alloc_dev(minor);
1da177e4
LT
1720 if (!md)
1721 return -ENXIO;
1722
784aae73
MB
1723 dm_sysfs_init(md);
1724
1da177e4
LT
1725 *result = md;
1726 return 0;
1727}
1728
a5664dad
MS
1729/*
1730 * Functions to manage md->type.
1731 * All are required to hold md->type_lock.
1732 */
1733void dm_lock_md_type(struct mapped_device *md)
1734{
1735 mutex_lock(&md->type_lock);
1736}
1737
1738void dm_unlock_md_type(struct mapped_device *md)
1739{
1740 mutex_unlock(&md->type_lock);
1741}
1742
1743void dm_set_md_type(struct mapped_device *md, unsigned type)
1744{
00c4fc3b 1745 BUG_ON(!mutex_is_locked(&md->type_lock));
a5664dad
MS
1746 md->type = type;
1747}
1748
1749unsigned dm_get_md_type(struct mapped_device *md)
1750{
1751 return md->type;
1752}
1753
36a0456f
AK
1754struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
1755{
1756 return md->immutable_target_type;
1757}
1758
f84cb8a4
MS
1759/*
1760 * The queue_limits are only valid as long as you have a reference
1761 * count on 'md'.
1762 */
1763struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
1764{
1765 BUG_ON(!atomic_read(&md->holders));
1766 return &md->queue->limits;
1767}
1768EXPORT_SYMBOL_GPL(dm_get_queue_limits);
1769
4a0b4ddf
MS
1770/*
1771 * Setup the DM device's queue based on md's type
1772 */
591ddcfc 1773int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
4a0b4ddf 1774{
bfebd1cd 1775 int r;
545ed20e 1776 unsigned type = dm_get_md_type(md);
bfebd1cd 1777
545ed20e 1778 switch (type) {
bfebd1cd 1779 case DM_TYPE_REQUEST_BASED:
eca7ee6d 1780 r = dm_old_init_request_queue(md);
bfebd1cd 1781 if (r) {
eca7ee6d 1782 DMERR("Cannot initialize queue for request-based mapped device");
bfebd1cd 1783 return r;
ff36ab34 1784 }
bfebd1cd
MS
1785 break;
1786 case DM_TYPE_MQ_REQUEST_BASED:
e83068a5 1787 r = dm_mq_init_request_queue(md, t);
bfebd1cd 1788 if (r) {
eca7ee6d 1789 DMERR("Cannot initialize queue for request-based dm-mq mapped device");
bfebd1cd
MS
1790 return r;
1791 }
1792 break;
1793 case DM_TYPE_BIO_BASED:
545ed20e 1794 case DM_TYPE_DAX_BIO_BASED:
eca7ee6d 1795 dm_init_normal_md_queue(md);
ff36ab34 1796 blk_queue_make_request(md->queue, dm_make_request);
dbba42d8
MP
1797 /*
1798 * DM handles splitting bios as needed. Free the bio_split bioset
1799 * since it won't be used (saves 1 process per bio-based DM device).
1800 */
1801 bioset_free(md->queue->bio_split);
1802 md->queue->bio_split = NULL;
545ed20e
TK
1803
1804 if (type == DM_TYPE_DAX_BIO_BASED)
1805 queue_flag_set_unlocked(QUEUE_FLAG_DAX, md->queue);
bfebd1cd 1806 break;
4a0b4ddf
MS
1807 }
1808
1809 return 0;
1810}
1811
2bec1f4a 1812struct mapped_device *dm_get_md(dev_t dev)
1da177e4
LT
1813{
1814 struct mapped_device *md;
1da177e4
LT
1815 unsigned minor = MINOR(dev);
1816
1817 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
1818 return NULL;
1819
f32c10b0 1820 spin_lock(&_minor_lock);
1da177e4
LT
1821
1822 md = idr_find(&_minor_idr, minor);
2bec1f4a
MP
1823 if (md) {
1824 if ((md == MINOR_ALLOCED ||
1825 (MINOR(disk_devt(dm_disk(md))) != minor) ||
1826 dm_deleting_md(md) ||
1827 test_bit(DMF_FREEING, &md->flags))) {
1828 md = NULL;
1829 goto out;
1830 }
1831 dm_get(md);
fba9f90e 1832 }
1da177e4 1833
fba9f90e 1834out:
f32c10b0 1835 spin_unlock(&_minor_lock);
1da177e4 1836
637842cf
DT
1837 return md;
1838}
3cf2e4ba 1839EXPORT_SYMBOL_GPL(dm_get_md);
d229a958 1840
9ade92a9 1841void *dm_get_mdptr(struct mapped_device *md)
637842cf 1842{
9ade92a9 1843 return md->interface_ptr;
1da177e4
LT
1844}
1845
1846void dm_set_mdptr(struct mapped_device *md, void *ptr)
1847{
1848 md->interface_ptr = ptr;
1849}
1850
1851void dm_get(struct mapped_device *md)
1852{
1853 atomic_inc(&md->holders);
3f77316d 1854 BUG_ON(test_bit(DMF_FREEING, &md->flags));
1da177e4
LT
1855}
1856
09ee96b2
MP
1857int dm_hold(struct mapped_device *md)
1858{
1859 spin_lock(&_minor_lock);
1860 if (test_bit(DMF_FREEING, &md->flags)) {
1861 spin_unlock(&_minor_lock);
1862 return -EBUSY;
1863 }
1864 dm_get(md);
1865 spin_unlock(&_minor_lock);
1866 return 0;
1867}
1868EXPORT_SYMBOL_GPL(dm_hold);
1869
72d94861
AK
1870const char *dm_device_name(struct mapped_device *md)
1871{
1872 return md->name;
1873}
1874EXPORT_SYMBOL_GPL(dm_device_name);
1875
3f77316d 1876static void __dm_destroy(struct mapped_device *md, bool wait)
1da177e4 1877{
3b785fbc 1878 struct request_queue *q = dm_get_md_queue(md);
1134e5ae 1879 struct dm_table *map;
83d5e5b0 1880 int srcu_idx;
1da177e4 1881
3f77316d 1882 might_sleep();
fba9f90e 1883
63a4f065 1884 spin_lock(&_minor_lock);
3f77316d
KU
1885 idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
1886 set_bit(DMF_FREEING, &md->flags);
1887 spin_unlock(&_minor_lock);
3b785fbc
BVA
1888
1889 spin_lock_irq(q->queue_lock);
1890 queue_flag_set(QUEUE_FLAG_DYING, q);
1891 spin_unlock_irq(q->queue_lock);
3f77316d 1892
02233342 1893 if (dm_request_based(md) && md->kworker_task)
3989144f 1894 kthread_flush_worker(&md->kworker);
2eb6e1e3 1895
ab7c7bb6
MP
1896 /*
1897 * Take suspend_lock so that presuspend and postsuspend methods
1898 * do not race with internal suspend.
1899 */
1900 mutex_lock(&md->suspend_lock);
2a708cff 1901 map = dm_get_live_table(md, &srcu_idx);
3f77316d
KU
1902 if (!dm_suspended_md(md)) {
1903 dm_table_presuspend_targets(map);
1904 dm_table_postsuspend_targets(map);
1da177e4 1905 }
83d5e5b0
MP
1906 /* dm_put_live_table must be before msleep, otherwise deadlock is possible */
1907 dm_put_live_table(md, srcu_idx);
2a708cff 1908 mutex_unlock(&md->suspend_lock);
83d5e5b0 1909
3f77316d
KU
1910 /*
1911 * Rare, but there may be I/O requests still going to complete,
1912 * for example. Wait for all references to disappear.
1913 * No one should increment the reference count of the mapped_device,
1914 * after the mapped_device state becomes DMF_FREEING.
1915 */
1916 if (wait)
1917 while (atomic_read(&md->holders))
1918 msleep(1);
1919 else if (atomic_read(&md->holders))
1920 DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)",
1921 dm_device_name(md), atomic_read(&md->holders));
1922
1923 dm_sysfs_exit(md);
3f77316d
KU
1924 dm_table_destroy(__unbind(md));
1925 free_dev(md);
1926}
1927
1928void dm_destroy(struct mapped_device *md)
1929{
1930 __dm_destroy(md, true);
1931}
1932
1933void dm_destroy_immediate(struct mapped_device *md)
1934{
1935 __dm_destroy(md, false);
1936}
1937
1938void dm_put(struct mapped_device *md)
1939{
1940 atomic_dec(&md->holders);
1da177e4 1941}
79eb885c 1942EXPORT_SYMBOL_GPL(dm_put);
1da177e4 1943
b48633f8 1944static int dm_wait_for_completion(struct mapped_device *md, long task_state)
46125c1c
MB
1945{
1946 int r = 0;
9f4c3f87 1947 DEFINE_WAIT(wait);
46125c1c
MB
1948
1949 while (1) {
9f4c3f87 1950 prepare_to_wait(&md->wait, &wait, task_state);
46125c1c 1951
b4324fee 1952 if (!md_in_flight(md))
46125c1c
MB
1953 break;
1954
e3fabdfd 1955 if (signal_pending_state(task_state, current)) {
46125c1c
MB
1956 r = -EINTR;
1957 break;
1958 }
1959
1960 io_schedule();
1961 }
9f4c3f87 1962 finish_wait(&md->wait, &wait);
b44ebeb0 1963
46125c1c
MB
1964 return r;
1965}
1966
1da177e4
LT
1967/*
1968 * Process the deferred bios
1969 */
ef208587 1970static void dm_wq_work(struct work_struct *work)
1da177e4 1971{
ef208587
MP
1972 struct mapped_device *md = container_of(work, struct mapped_device,
1973 work);
6d6f10df 1974 struct bio *c;
83d5e5b0
MP
1975 int srcu_idx;
1976 struct dm_table *map;
1da177e4 1977
83d5e5b0 1978 map = dm_get_live_table(md, &srcu_idx);
ef208587 1979
3b00b203 1980 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
df12ee99
AK
1981 spin_lock_irq(&md->deferred_lock);
1982 c = bio_list_pop(&md->deferred);
1983 spin_unlock_irq(&md->deferred_lock);
1984
6a8736d1 1985 if (!c)
df12ee99 1986 break;
022c2611 1987
e6ee8c0b
KU
1988 if (dm_request_based(md))
1989 generic_make_request(c);
6a8736d1 1990 else
83d5e5b0 1991 __split_and_process_bio(md, map, c);
022c2611 1992 }
73d410c0 1993
83d5e5b0 1994 dm_put_live_table(md, srcu_idx);
1da177e4
LT
1995}
1996
9a1fb464 1997static void dm_queue_flush(struct mapped_device *md)
304f3f6a 1998{
3b00b203 1999 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
4e857c58 2000 smp_mb__after_atomic();
53d5914f 2001 queue_work(md->wq, &md->work);
304f3f6a
MB
2002}
2003
1da177e4 2004/*
042d2a9b 2005 * Swap in a new table, returning the old one for the caller to destroy.
1da177e4 2006 */
042d2a9b 2007struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table)
1da177e4 2008{
87eb5b21 2009 struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL);
754c5fc7 2010 struct queue_limits limits;
042d2a9b 2011 int r;
1da177e4 2012
e61290a4 2013 mutex_lock(&md->suspend_lock);
1da177e4
LT
2014
2015 /* device must be suspended */
4f186f8b 2016 if (!dm_suspended_md(md))
93c534ae 2017 goto out;
1da177e4 2018
3ae70656
MS
2019 /*
2020 * If the new table has no data devices, retain the existing limits.
2021 * This helps multipath with queue_if_no_path if all paths disappear,
2022 * then new I/O is queued based on these limits, and then some paths
2023 * reappear.
2024 */
2025 if (dm_table_has_no_data_devices(table)) {
83d5e5b0 2026 live_map = dm_get_live_table_fast(md);
3ae70656
MS
2027 if (live_map)
2028 limits = md->queue->limits;
83d5e5b0 2029 dm_put_live_table_fast(md);
3ae70656
MS
2030 }
2031
87eb5b21
MC
2032 if (!live_map) {
2033 r = dm_calculate_queue_limits(table, &limits);
2034 if (r) {
2035 map = ERR_PTR(r);
2036 goto out;
2037 }
042d2a9b 2038 }
754c5fc7 2039
042d2a9b 2040 map = __bind(md, table, &limits);
1da177e4 2041
93c534ae 2042out:
e61290a4 2043 mutex_unlock(&md->suspend_lock);
042d2a9b 2044 return map;
1da177e4
LT
2045}
2046
2047/*
2048 * Functions to lock and unlock any filesystem running on the
2049 * device.
2050 */
2ca3310e 2051static int lock_fs(struct mapped_device *md)
1da177e4 2052{
e39e2e95 2053 int r;
1da177e4
LT
2054
2055 WARN_ON(md->frozen_sb);
dfbe03f6 2056
db8fef4f 2057 md->frozen_sb = freeze_bdev(md->bdev);
dfbe03f6 2058 if (IS_ERR(md->frozen_sb)) {
cf222b37 2059 r = PTR_ERR(md->frozen_sb);
e39e2e95
AK
2060 md->frozen_sb = NULL;
2061 return r;
dfbe03f6
AK
2062 }
2063
aa8d7c2f
AK
2064 set_bit(DMF_FROZEN, &md->flags);
2065
1da177e4
LT
2066 return 0;
2067}
2068
2ca3310e 2069static void unlock_fs(struct mapped_device *md)
1da177e4 2070{
aa8d7c2f
AK
2071 if (!test_bit(DMF_FROZEN, &md->flags))
2072 return;
2073
db8fef4f 2074 thaw_bdev(md->bdev, md->frozen_sb);
1da177e4 2075 md->frozen_sb = NULL;
aa8d7c2f 2076 clear_bit(DMF_FROZEN, &md->flags);
1da177e4
LT
2077}
2078
2079/*
b48633f8
BVA
2080 * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG
2081 * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE
2082 * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY
2083 *
ffcc3936
MS
2084 * If __dm_suspend returns 0, the device is completely quiescent
2085 * now. There is no request-processing activity. All new requests
2086 * are being added to md->deferred list.
cec47e3d 2087 *
ffcc3936 2088 * Caller must hold md->suspend_lock
cec47e3d 2089 */
ffcc3936 2090static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
b48633f8 2091 unsigned suspend_flags, long task_state,
eaf9a736 2092 int dmf_suspended_flag)
1da177e4 2093{
ffcc3936
MS
2094 bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG;
2095 bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG;
2096 int r;
1da177e4 2097
5a8f1f80
BVA
2098 lockdep_assert_held(&md->suspend_lock);
2099
2e93ccc1
KU
2100 /*
2101 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
2102 * This flag is cleared before dm_suspend returns.
2103 */
2104 if (noflush)
2105 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2106
d67ee213
MS
2107 /*
2108 * This gets reverted if there's an error later and the targets
2109 * provide the .presuspend_undo hook.
2110 */
cf222b37
AK
2111 dm_table_presuspend_targets(map);
2112
32a926da 2113 /*
9f518b27
KU
2114 * Flush I/O to the device.
2115 * Any I/O submitted after lock_fs() may not be flushed.
2116 * noflush takes precedence over do_lockfs.
2117 * (lock_fs() flushes I/Os and waits for them to complete.)
32a926da
MP
2118 */
2119 if (!noflush && do_lockfs) {
2120 r = lock_fs(md);
d67ee213
MS
2121 if (r) {
2122 dm_table_presuspend_undo_targets(map);
ffcc3936 2123 return r;
d67ee213 2124 }
aa8d7c2f 2125 }
1da177e4
LT
2126
2127 /*
3b00b203
MP
2128 * Here we must make sure that no processes are submitting requests
2129 * to target drivers i.e. no one may be executing
2130 * __split_and_process_bio. This is called from dm_request and
2131 * dm_wq_work.
2132 *
2133 * To get all processes out of __split_and_process_bio in dm_request,
2134 * we take the write lock. To prevent any process from reentering
6a8736d1
TH
2135 * __split_and_process_bio from dm_request and quiesce the thread
2136 * (dm_wq_work), we set BMF_BLOCK_IO_FOR_SUSPEND and call
2137 * flush_workqueue(md->wq).
1da177e4 2138 */
1eb787ec 2139 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
41abc4e1
HR
2140 if (map)
2141 synchronize_srcu(&md->io_barrier);
1da177e4 2142
d0bcb878 2143 /*
29e4013d
TH
2144 * Stop md->queue before flushing md->wq in case request-based
2145 * dm defers requests to md->wq from md->queue.
d0bcb878 2146 */
2eb6e1e3 2147 if (dm_request_based(md)) {
eca7ee6d 2148 dm_stop_queue(md->queue);
02233342 2149 if (md->kworker_task)
3989144f 2150 kthread_flush_worker(&md->kworker);
2eb6e1e3 2151 }
cec47e3d 2152
d0bcb878
KU
2153 flush_workqueue(md->wq);
2154
1da177e4 2155 /*
3b00b203
MP
2156 * At this point no more requests are entering target request routines.
2157 * We call dm_wait_for_completion to wait for all existing requests
2158 * to finish.
1da177e4 2159 */
b48633f8 2160 r = dm_wait_for_completion(md, task_state);
eaf9a736
MS
2161 if (!r)
2162 set_bit(dmf_suspended_flag, &md->flags);
1da177e4 2163
6d6f10df 2164 if (noflush)
022c2611 2165 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
41abc4e1
HR
2166 if (map)
2167 synchronize_srcu(&md->io_barrier);
2e93ccc1 2168
1da177e4 2169 /* were we interrupted ? */
46125c1c 2170 if (r < 0) {
9a1fb464 2171 dm_queue_flush(md);
73d410c0 2172
cec47e3d 2173 if (dm_request_based(md))
eca7ee6d 2174 dm_start_queue(md->queue);
cec47e3d 2175
2ca3310e 2176 unlock_fs(md);
d67ee213 2177 dm_table_presuspend_undo_targets(map);
ffcc3936 2178 /* pushback list is already flushed, so skip flush */
2ca3310e 2179 }
1da177e4 2180
ffcc3936
MS
2181 return r;
2182}
2183
2184/*
2185 * We need to be able to change a mapping table under a mounted
2186 * filesystem. For example we might want to move some data in
2187 * the background. Before the table can be swapped with
2188 * dm_bind_table, dm_suspend must be called to flush any in
2189 * flight bios and ensure that any further io gets deferred.
2190 */
2191/*
2192 * Suspend mechanism in request-based dm.
2193 *
2194 * 1. Flush all I/Os by lock_fs() if needed.
2195 * 2. Stop dispatching any I/O by stopping the request_queue.
2196 * 3. Wait for all in-flight I/Os to be completed or requeued.
2197 *
2198 * To abort suspend, start the request_queue.
2199 */
2200int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
2201{
2202 struct dm_table *map = NULL;
2203 int r = 0;
2204
2205retry:
2206 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2207
2208 if (dm_suspended_md(md)) {
2209 r = -EINVAL;
2210 goto out_unlock;
2211 }
2212
2213 if (dm_suspended_internally_md(md)) {
2214 /* already internally suspended, wait for internal resume */
2215 mutex_unlock(&md->suspend_lock);
2216 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2217 if (r)
2218 return r;
2219 goto retry;
2220 }
2221
a12f5d48 2222 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
ffcc3936 2223
eaf9a736 2224 r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED);
ffcc3936
MS
2225 if (r)
2226 goto out_unlock;
3b00b203 2227
4d4471cb
KU
2228 dm_table_postsuspend_targets(map);
2229
d287483d 2230out_unlock:
e61290a4 2231 mutex_unlock(&md->suspend_lock);
cf222b37 2232 return r;
1da177e4
LT
2233}
2234
ffcc3936
MS
2235static int __dm_resume(struct mapped_device *md, struct dm_table *map)
2236{
2237 if (map) {
2238 int r = dm_table_resume_targets(map);
2239 if (r)
2240 return r;
2241 }
2242
2243 dm_queue_flush(md);
2244
2245 /*
2246 * Flushing deferred I/Os must be done after targets are resumed
2247 * so that mapping of targets can work correctly.
2248 * Request-based dm is queueing the deferred I/Os in its request_queue.
2249 */
2250 if (dm_request_based(md))
eca7ee6d 2251 dm_start_queue(md->queue);
ffcc3936
MS
2252
2253 unlock_fs(md);
2254
2255 return 0;
2256}
2257
1da177e4
LT
2258int dm_resume(struct mapped_device *md)
2259{
8dc23658 2260 int r;
cf222b37 2261 struct dm_table *map = NULL;
1da177e4 2262
ffcc3936 2263retry:
8dc23658 2264 r = -EINVAL;
ffcc3936
MS
2265 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2266
4f186f8b 2267 if (!dm_suspended_md(md))
cf222b37 2268 goto out;
cf222b37 2269
ffcc3936
MS
2270 if (dm_suspended_internally_md(md)) {
2271 /* already internally suspended, wait for internal resume */
2272 mutex_unlock(&md->suspend_lock);
2273 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2274 if (r)
2275 return r;
2276 goto retry;
2277 }
2278
a12f5d48 2279 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2ca3310e 2280 if (!map || !dm_table_get_size(map))
cf222b37 2281 goto out;
1da177e4 2282
ffcc3936 2283 r = __dm_resume(md, map);
8757b776
MB
2284 if (r)
2285 goto out;
2ca3310e 2286
2ca3310e 2287 clear_bit(DMF_SUSPENDED, &md->flags);
cf222b37 2288out:
e61290a4 2289 mutex_unlock(&md->suspend_lock);
2ca3310e 2290
cf222b37 2291 return r;
1da177e4
LT
2292}
2293
fd2ed4d2
MP
2294/*
2295 * Internal suspend/resume works like userspace-driven suspend. It waits
2296 * until all bios finish and prevents issuing new bios to the target drivers.
2297 * It may be used only from the kernel.
fd2ed4d2
MP
2298 */
2299
ffcc3936 2300static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags)
fd2ed4d2 2301{
ffcc3936
MS
2302 struct dm_table *map = NULL;
2303
96b26c8c 2304 if (md->internal_suspend_count++)
ffcc3936
MS
2305 return; /* nested internal suspend */
2306
2307 if (dm_suspended_md(md)) {
2308 set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2309 return; /* nest suspend */
2310 }
2311
a12f5d48 2312 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
ffcc3936
MS
2313
2314 /*
2315 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is
2316 * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend
2317 * would require changing .presuspend to return an error -- avoid this
2318 * until there is a need for more elaborate variants of internal suspend.
2319 */
eaf9a736
MS
2320 (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE,
2321 DMF_SUSPENDED_INTERNALLY);
ffcc3936
MS
2322
2323 dm_table_postsuspend_targets(map);
2324}
2325
2326static void __dm_internal_resume(struct mapped_device *md)
2327{
96b26c8c
MP
2328 BUG_ON(!md->internal_suspend_count);
2329
2330 if (--md->internal_suspend_count)
ffcc3936
MS
2331 return; /* resume from nested internal suspend */
2332
fd2ed4d2 2333 if (dm_suspended_md(md))
ffcc3936
MS
2334 goto done; /* resume from nested suspend */
2335
2336 /*
2337 * NOTE: existing callers don't need to call dm_table_resume_targets
2338 * (which may fail -- so best to avoid it for now by passing NULL map)
2339 */
2340 (void) __dm_resume(md, NULL);
2341
2342done:
2343 clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2344 smp_mb__after_atomic();
2345 wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY);
2346}
2347
2348void dm_internal_suspend_noflush(struct mapped_device *md)
2349{
2350 mutex_lock(&md->suspend_lock);
2351 __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG);
2352 mutex_unlock(&md->suspend_lock);
2353}
2354EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush);
2355
2356void dm_internal_resume(struct mapped_device *md)
2357{
2358 mutex_lock(&md->suspend_lock);
2359 __dm_internal_resume(md);
2360 mutex_unlock(&md->suspend_lock);
2361}
2362EXPORT_SYMBOL_GPL(dm_internal_resume);
2363
2364/*
2365 * Fast variants of internal suspend/resume hold md->suspend_lock,
2366 * which prevents interaction with userspace-driven suspend.
2367 */
2368
2369void dm_internal_suspend_fast(struct mapped_device *md)
2370{
2371 mutex_lock(&md->suspend_lock);
2372 if (dm_suspended_md(md) || dm_suspended_internally_md(md))
fd2ed4d2
MP
2373 return;
2374
2375 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2376 synchronize_srcu(&md->io_barrier);
2377 flush_workqueue(md->wq);
2378 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
2379}
b735fede 2380EXPORT_SYMBOL_GPL(dm_internal_suspend_fast);
fd2ed4d2 2381
ffcc3936 2382void dm_internal_resume_fast(struct mapped_device *md)
fd2ed4d2 2383{
ffcc3936 2384 if (dm_suspended_md(md) || dm_suspended_internally_md(md))
fd2ed4d2
MP
2385 goto done;
2386
2387 dm_queue_flush(md);
2388
2389done:
2390 mutex_unlock(&md->suspend_lock);
2391}
b735fede 2392EXPORT_SYMBOL_GPL(dm_internal_resume_fast);
fd2ed4d2 2393
1da177e4
LT
2394/*-----------------------------------------------------------------
2395 * Event notification.
2396 *---------------------------------------------------------------*/
3abf85b5 2397int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
60935eb2 2398 unsigned cookie)
69267a30 2399{
60935eb2
MB
2400 char udev_cookie[DM_COOKIE_LENGTH];
2401 char *envp[] = { udev_cookie, NULL };
2402
2403 if (!cookie)
3abf85b5 2404 return kobject_uevent(&disk_to_dev(md->disk)->kobj, action);
60935eb2
MB
2405 else {
2406 snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u",
2407 DM_COOKIE_ENV_VAR_NAME, cookie);
3abf85b5
PR
2408 return kobject_uevent_env(&disk_to_dev(md->disk)->kobj,
2409 action, envp);
60935eb2 2410 }
69267a30
AK
2411}
2412
7a8c3d3b
MA
2413uint32_t dm_next_uevent_seq(struct mapped_device *md)
2414{
2415 return atomic_add_return(1, &md->uevent_seq);
2416}
2417
1da177e4
LT
2418uint32_t dm_get_event_nr(struct mapped_device *md)
2419{
2420 return atomic_read(&md->event_nr);
2421}
2422
2423int dm_wait_event(struct mapped_device *md, int event_nr)
2424{
2425 return wait_event_interruptible(md->eventq,
2426 (event_nr != atomic_read(&md->event_nr)));
2427}
2428
7a8c3d3b
MA
2429void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
2430{
2431 unsigned long flags;
2432
2433 spin_lock_irqsave(&md->uevent_lock, flags);
2434 list_add(elist, &md->uevent_list);
2435 spin_unlock_irqrestore(&md->uevent_lock, flags);
2436}
2437
1da177e4
LT
2438/*
2439 * The gendisk is only valid as long as you have a reference
2440 * count on 'md'.
2441 */
2442struct gendisk *dm_disk(struct mapped_device *md)
2443{
2444 return md->disk;
2445}
65ff5b7d 2446EXPORT_SYMBOL_GPL(dm_disk);
1da177e4 2447
784aae73
MB
2448struct kobject *dm_kobject(struct mapped_device *md)
2449{
2995fa78 2450 return &md->kobj_holder.kobj;
784aae73
MB
2451}
2452
784aae73
MB
2453struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
2454{
2455 struct mapped_device *md;
2456
2995fa78 2457 md = container_of(kobj, struct mapped_device, kobj_holder.kobj);
784aae73 2458
4d89b7b4 2459 if (test_bit(DMF_FREEING, &md->flags) ||
432a212c 2460 dm_deleting_md(md))
4d89b7b4
MB
2461 return NULL;
2462
784aae73
MB
2463 dm_get(md);
2464 return md;
2465}
2466
4f186f8b 2467int dm_suspended_md(struct mapped_device *md)
1da177e4
LT
2468{
2469 return test_bit(DMF_SUSPENDED, &md->flags);
2470}
2471
ffcc3936
MS
2472int dm_suspended_internally_md(struct mapped_device *md)
2473{
2474 return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2475}
2476
2c140a24
MP
2477int dm_test_deferred_remove_flag(struct mapped_device *md)
2478{
2479 return test_bit(DMF_DEFERRED_REMOVE, &md->flags);
2480}
2481
64dbce58
KU
2482int dm_suspended(struct dm_target *ti)
2483{
ecdb2e25 2484 return dm_suspended_md(dm_table_get_md(ti->table));
64dbce58
KU
2485}
2486EXPORT_SYMBOL_GPL(dm_suspended);
2487
2e93ccc1
KU
2488int dm_noflush_suspending(struct dm_target *ti)
2489{
ecdb2e25 2490 return __noflush_suspending(dm_table_get_md(ti->table));
2e93ccc1
KU
2491}
2492EXPORT_SYMBOL_GPL(dm_noflush_suspending);
2493
78d8e58a 2494struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, unsigned type,
30187e1d 2495 unsigned integrity, unsigned per_io_data_size)
e6ee8c0b 2496{
115485e8 2497 struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);
78d8e58a
MS
2498 struct kmem_cache *cachep = NULL;
2499 unsigned int pool_size = 0;
5f015204 2500 unsigned int front_pad;
e6ee8c0b
KU
2501
2502 if (!pools)
4e6e36c3 2503 return NULL;
e6ee8c0b 2504
78d8e58a
MS
2505 switch (type) {
2506 case DM_TYPE_BIO_BASED:
545ed20e 2507 case DM_TYPE_DAX_BIO_BASED:
78d8e58a
MS
2508 cachep = _io_cache;
2509 pool_size = dm_get_reserved_bio_based_ios();
30187e1d 2510 front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
78d8e58a
MS
2511 break;
2512 case DM_TYPE_REQUEST_BASED:
2513 cachep = _rq_tio_cache;
2514 pool_size = dm_get_reserved_rq_based_ios();
2515 pools->rq_pool = mempool_create_slab_pool(pool_size, _rq_cache);
2516 if (!pools->rq_pool)
2517 goto out;
2518 /* fall through to setup remaining rq-based pools */
2519 case DM_TYPE_MQ_REQUEST_BASED:
2520 if (!pool_size)
2521 pool_size = dm_get_reserved_rq_based_ios();
2522 front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
591ddcfc 2523 /* per_io_data_size is used for blk-mq pdu at queue allocation */
78d8e58a
MS
2524 break;
2525 default:
2526 BUG();
2527 }
2528
2529 if (cachep) {
2530 pools->io_pool = mempool_create_slab_pool(pool_size, cachep);
2531 if (!pools->io_pool)
2532 goto out;
2533 }
e6ee8c0b 2534
3d8aab2d 2535 pools->bs = bioset_create_nobvec(pool_size, front_pad);
e6ee8c0b 2536 if (!pools->bs)
5f015204 2537 goto out;
e6ee8c0b 2538
a91a2785 2539 if (integrity && bioset_integrity_create(pools->bs, pool_size))
5f015204 2540 goto out;
a91a2785 2541
e6ee8c0b 2542 return pools;
5f1b670d 2543
5f1b670d
CH
2544out:
2545 dm_free_md_mempools(pools);
78d8e58a 2546
4e6e36c3 2547 return NULL;
e6ee8c0b
KU
2548}
2549
2550void dm_free_md_mempools(struct dm_md_mempools *pools)
2551{
2552 if (!pools)
2553 return;
2554
6f65985e
JL
2555 mempool_destroy(pools->io_pool);
2556 mempool_destroy(pools->rq_pool);
1ae49ea2 2557
e6ee8c0b
KU
2558 if (pools->bs)
2559 bioset_free(pools->bs);
2560
2561 kfree(pools);
2562}
2563
9c72bad1
CH
2564struct dm_pr {
2565 u64 old_key;
2566 u64 new_key;
2567 u32 flags;
2568 bool fail_early;
2569};
2570
2571static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn,
2572 void *data)
71cdb697
CH
2573{
2574 struct mapped_device *md = bdev->bd_disk->private_data;
9c72bad1
CH
2575 struct dm_table *table;
2576 struct dm_target *ti;
2577 int ret = -ENOTTY, srcu_idx;
71cdb697 2578
9c72bad1
CH
2579 table = dm_get_live_table(md, &srcu_idx);
2580 if (!table || !dm_table_get_size(table))
2581 goto out;
71cdb697 2582
9c72bad1
CH
2583 /* We only support devices that have a single target */
2584 if (dm_table_get_num_targets(table) != 1)
2585 goto out;
2586 ti = dm_table_get_target(table, 0);
71cdb697 2587
9c72bad1
CH
2588 ret = -EINVAL;
2589 if (!ti->type->iterate_devices)
2590 goto out;
2591
2592 ret = ti->type->iterate_devices(ti, fn, data);
2593out:
2594 dm_put_live_table(md, srcu_idx);
2595 return ret;
2596}
2597
2598/*
2599 * For register / unregister we need to manually call out to every path.
2600 */
2601static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev,
2602 sector_t start, sector_t len, void *data)
2603{
2604 struct dm_pr *pr = data;
2605 const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
2606
2607 if (!ops || !ops->pr_register)
2608 return -EOPNOTSUPP;
2609 return ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags);
2610}
2611
2612static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key,
2613 u32 flags)
2614{
2615 struct dm_pr pr = {
2616 .old_key = old_key,
2617 .new_key = new_key,
2618 .flags = flags,
2619 .fail_early = true,
2620 };
2621 int ret;
2622
2623 ret = dm_call_pr(bdev, __dm_pr_register, &pr);
2624 if (ret && new_key) {
2625 /* unregister all paths if we failed to register any path */
2626 pr.old_key = new_key;
2627 pr.new_key = 0;
2628 pr.flags = 0;
2629 pr.fail_early = false;
2630 dm_call_pr(bdev, __dm_pr_register, &pr);
2631 }
2632
2633 return ret;
71cdb697
CH
2634}
2635
2636static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type,
956a4025 2637 u32 flags)
71cdb697
CH
2638{
2639 struct mapped_device *md = bdev->bd_disk->private_data;
2640 const struct pr_ops *ops;
71cdb697 2641 fmode_t mode;
956a4025 2642 int r;
71cdb697 2643
956a4025 2644 r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
71cdb697
CH
2645 if (r < 0)
2646 return r;
2647
2648 ops = bdev->bd_disk->fops->pr_ops;
2649 if (ops && ops->pr_reserve)
2650 r = ops->pr_reserve(bdev, key, type, flags);
2651 else
2652 r = -EOPNOTSUPP;
2653
956a4025 2654 bdput(bdev);
71cdb697
CH
2655 return r;
2656}
2657
2658static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
2659{
2660 struct mapped_device *md = bdev->bd_disk->private_data;
2661 const struct pr_ops *ops;
71cdb697 2662 fmode_t mode;
956a4025 2663 int r;
71cdb697 2664
956a4025 2665 r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
71cdb697
CH
2666 if (r < 0)
2667 return r;
2668
2669 ops = bdev->bd_disk->fops->pr_ops;
2670 if (ops && ops->pr_release)
2671 r = ops->pr_release(bdev, key, type);
2672 else
2673 r = -EOPNOTSUPP;
2674
956a4025 2675 bdput(bdev);
71cdb697
CH
2676 return r;
2677}
2678
2679static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key,
956a4025 2680 enum pr_type type, bool abort)
71cdb697
CH
2681{
2682 struct mapped_device *md = bdev->bd_disk->private_data;
2683 const struct pr_ops *ops;
71cdb697 2684 fmode_t mode;
956a4025 2685 int r;
71cdb697 2686
956a4025 2687 r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
71cdb697
CH
2688 if (r < 0)
2689 return r;
2690
2691 ops = bdev->bd_disk->fops->pr_ops;
2692 if (ops && ops->pr_preempt)
2693 r = ops->pr_preempt(bdev, old_key, new_key, type, abort);
2694 else
2695 r = -EOPNOTSUPP;
2696
956a4025 2697 bdput(bdev);
71cdb697
CH
2698 return r;
2699}
2700
2701static int dm_pr_clear(struct block_device *bdev, u64 key)
2702{
2703 struct mapped_device *md = bdev->bd_disk->private_data;
2704 const struct pr_ops *ops;
71cdb697 2705 fmode_t mode;
956a4025 2706 int r;
71cdb697 2707
956a4025 2708 r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
71cdb697
CH
2709 if (r < 0)
2710 return r;
2711
2712 ops = bdev->bd_disk->fops->pr_ops;
2713 if (ops && ops->pr_clear)
2714 r = ops->pr_clear(bdev, key);
2715 else
2716 r = -EOPNOTSUPP;
2717
956a4025 2718 bdput(bdev);
71cdb697
CH
2719 return r;
2720}
2721
2722static const struct pr_ops dm_pr_ops = {
2723 .pr_register = dm_pr_register,
2724 .pr_reserve = dm_pr_reserve,
2725 .pr_release = dm_pr_release,
2726 .pr_preempt = dm_pr_preempt,
2727 .pr_clear = dm_pr_clear,
2728};
2729
83d5cde4 2730static const struct block_device_operations dm_blk_dops = {
1da177e4
LT
2731 .open = dm_blk_open,
2732 .release = dm_blk_close,
aa129a22 2733 .ioctl = dm_blk_ioctl,
545ed20e 2734 .direct_access = dm_blk_direct_access,
3ac51e74 2735 .getgeo = dm_blk_getgeo,
71cdb697 2736 .pr_ops = &dm_pr_ops,
1da177e4
LT
2737 .owner = THIS_MODULE
2738};
2739
1da177e4
LT
2740/*
2741 * module hooks
2742 */
2743module_init(dm_init);
2744module_exit(dm_exit);
2745
2746module_param(major, uint, 0);
2747MODULE_PARM_DESC(major, "The major number of the device mapper");
f4790826 2748
e8603136
MS
2749module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR);
2750MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools");
2751
115485e8
MS
2752module_param(dm_numa_node, int, S_IRUGO | S_IWUSR);
2753MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations");
2754
1da177e4
LT
2755MODULE_DESCRIPTION(DM_NAME " driver");
2756MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
2757MODULE_LICENSE("GPL");