]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/md/dm.c
dm: allow active and inactive tables to share dm_devs
[mirror_ubuntu-artful-kernel.git] / drivers / md / dm.c
CommitLineData
1da177e4
LT
1/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
784aae73 3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
1da177e4
LT
4 *
5 * This file is released under the GPL.
6 */
7
8#include "dm.h"
51e5b2bd 9#include "dm-uevent.h"
1da177e4
LT
10
11#include <linux/init.h>
12#include <linux/module.h>
48c9c27b 13#include <linux/mutex.h>
1da177e4
LT
14#include <linux/moduleparam.h>
15#include <linux/blkpg.h>
16#include <linux/bio.h>
1da177e4
LT
17#include <linux/mempool.h>
18#include <linux/slab.h>
19#include <linux/idr.h>
3ac51e74 20#include <linux/hdreg.h>
3f77316d 21#include <linux/delay.h>
55782138
LZ
22
23#include <trace/events/block.h>
1da177e4 24
72d94861
AK
25#define DM_MSG_PREFIX "core"
26
71a16736
NK
27#ifdef CONFIG_PRINTK
28/*
29 * ratelimit state to be used in DMXXX_LIMIT().
30 */
31DEFINE_RATELIMIT_STATE(dm_ratelimit_state,
32 DEFAULT_RATELIMIT_INTERVAL,
33 DEFAULT_RATELIMIT_BURST);
34EXPORT_SYMBOL(dm_ratelimit_state);
35#endif
36
60935eb2
MB
37/*
38 * Cookies are numeric values sent with CHANGE and REMOVE
39 * uevents while resuming, removing or renaming the device.
40 */
41#define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE"
42#define DM_COOKIE_LENGTH 24
43
1da177e4
LT
44static const char *_name = DM_NAME;
45
46static unsigned int major = 0;
47static unsigned int _major = 0;
48
d15b774c
AK
49static DEFINE_IDR(_minor_idr);
50
f32c10b0 51static DEFINE_SPINLOCK(_minor_lock);
2c140a24
MP
52
53static void do_deferred_remove(struct work_struct *w);
54
55static DECLARE_WORK(deferred_remove_work, do_deferred_remove);
56
acfe0ad7
MP
57static struct workqueue_struct *deferred_remove_workqueue;
58
1da177e4 59/*
8fbf26ad 60 * For bio-based dm.
1da177e4
LT
61 * One of these is allocated per bio.
62 */
63struct dm_io {
64 struct mapped_device *md;
65 int error;
1da177e4 66 atomic_t io_count;
6ae2fa67 67 struct bio *bio;
3eaf840e 68 unsigned long start_time;
f88fb981 69 spinlock_t endio_lock;
fd2ed4d2 70 struct dm_stats_aux stats_aux;
1da177e4
LT
71};
72
8fbf26ad
KU
73/*
74 * For request-based dm.
75 * One of these is allocated per request.
76 */
77struct dm_rq_target_io {
78 struct mapped_device *md;
79 struct dm_target *ti;
80 struct request *orig, clone;
81 int error;
82 union map_info info;
83};
84
85/*
94818742
KO
86 * For request-based dm - the bio clones we allocate are embedded in these
87 * structs.
88 *
89 * We allocate these with bio_alloc_bioset, using the front_pad parameter when
90 * the bioset is created - this means the bio has to come at the end of the
91 * struct.
8fbf26ad
KU
92 */
93struct dm_rq_clone_bio_info {
94 struct bio *orig;
cec47e3d 95 struct dm_rq_target_io *tio;
94818742 96 struct bio clone;
8fbf26ad
KU
97};
98
cec47e3d
KU
99union map_info *dm_get_rq_mapinfo(struct request *rq)
100{
101 if (rq && rq->end_io_data)
102 return &((struct dm_rq_target_io *)rq->end_io_data)->info;
103 return NULL;
104}
105EXPORT_SYMBOL_GPL(dm_get_rq_mapinfo);
106
ba61fdd1
JM
107#define MINOR_ALLOCED ((void *)-1)
108
1da177e4
LT
109/*
110 * Bits for the md->flags field.
111 */
1eb787ec 112#define DMF_BLOCK_IO_FOR_SUSPEND 0
1da177e4 113#define DMF_SUSPENDED 1
aa8d7c2f 114#define DMF_FROZEN 2
fba9f90e 115#define DMF_FREEING 3
5c6bd75d 116#define DMF_DELETING 4
2e93ccc1 117#define DMF_NOFLUSH_SUSPENDING 5
d5b9dd04 118#define DMF_MERGE_IS_OPTIONAL 6
2c140a24 119#define DMF_DEFERRED_REMOVE 7
1da177e4 120
83d5e5b0
MP
121/*
122 * A dummy definition to make RCU happy.
123 * struct dm_table should never be dereferenced in this file.
124 */
125struct dm_table {
126 int undefined__;
127};
128
304f3f6a
MB
129/*
130 * Work processed by per-device workqueue.
131 */
1da177e4 132struct mapped_device {
83d5e5b0 133 struct srcu_struct io_barrier;
e61290a4 134 struct mutex suspend_lock;
1da177e4 135 atomic_t holders;
5c6bd75d 136 atomic_t open_count;
1da177e4 137
2a7faeb1
MP
138 /*
139 * The current mapping.
140 * Use dm_get_live_table{_fast} or take suspend_lock for
141 * dereference.
142 */
143 struct dm_table *map;
144
86f1152b
BM
145 struct list_head table_devices;
146 struct mutex table_devices_lock;
147
1da177e4
LT
148 unsigned long flags;
149
165125e1 150 struct request_queue *queue;
a5664dad 151 unsigned type;
4a0b4ddf 152 /* Protect queue and type against concurrent access. */
a5664dad
MS
153 struct mutex type_lock;
154
36a0456f
AK
155 struct target_type *immutable_target_type;
156
1da177e4 157 struct gendisk *disk;
7e51f257 158 char name[16];
1da177e4
LT
159
160 void *interface_ptr;
161
162 /*
163 * A list of ios that arrived while we were suspended.
164 */
316d315b 165 atomic_t pending[2];
1da177e4 166 wait_queue_head_t wait;
53d5914f 167 struct work_struct work;
74859364 168 struct bio_list deferred;
022c2611 169 spinlock_t deferred_lock;
1da177e4 170
af7e466a 171 /*
29e4013d 172 * Processing queue (flush)
304f3f6a
MB
173 */
174 struct workqueue_struct *wq;
175
1da177e4
LT
176 /*
177 * io objects are allocated from here.
178 */
179 mempool_t *io_pool;
1da177e4 180
9faf400f
SB
181 struct bio_set *bs;
182
1da177e4
LT
183 /*
184 * Event handling.
185 */
186 atomic_t event_nr;
187 wait_queue_head_t eventq;
7a8c3d3b
MA
188 atomic_t uevent_seq;
189 struct list_head uevent_list;
190 spinlock_t uevent_lock; /* Protect access to uevent_list */
1da177e4
LT
191
192 /*
193 * freeze/thaw support require holding onto a super block
194 */
195 struct super_block *frozen_sb;
db8fef4f 196 struct block_device *bdev;
3ac51e74
DW
197
198 /* forced geometry settings */
199 struct hd_geometry geometry;
784aae73 200
2995fa78
MP
201 /* kobject and completion */
202 struct dm_kobject_holder kobj_holder;
be35f486 203
d87f4c14
TH
204 /* zero-length flush that will be cloned and submitted to targets */
205 struct bio flush_bio;
fd2ed4d2
MP
206
207 struct dm_stats stats;
1da177e4
LT
208};
209
e6ee8c0b
KU
210/*
211 * For mempools pre-allocation at the table loading time.
212 */
213struct dm_md_mempools {
214 mempool_t *io_pool;
e6ee8c0b
KU
215 struct bio_set *bs;
216};
217
86f1152b
BM
218struct table_device {
219 struct list_head list;
220 atomic_t count;
221 struct dm_dev dm_dev;
222};
223
6cfa5857
MS
224#define RESERVED_BIO_BASED_IOS 16
225#define RESERVED_REQUEST_BASED_IOS 256
f4790826 226#define RESERVED_MAX_IOS 1024
e18b890b 227static struct kmem_cache *_io_cache;
8fbf26ad 228static struct kmem_cache *_rq_tio_cache;
94818742 229
e8603136
MS
230/*
231 * Bio-based DM's mempools' reserved IOs set by the user.
232 */
233static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS;
234
f4790826
MS
235/*
236 * Request-based DM's mempools' reserved IOs set by the user.
237 */
238static unsigned reserved_rq_based_ios = RESERVED_REQUEST_BASED_IOS;
239
240static unsigned __dm_get_reserved_ios(unsigned *reserved_ios,
241 unsigned def, unsigned max)
242{
243 unsigned ios = ACCESS_ONCE(*reserved_ios);
244 unsigned modified_ios = 0;
245
246 if (!ios)
247 modified_ios = def;
248 else if (ios > max)
249 modified_ios = max;
250
251 if (modified_ios) {
252 (void)cmpxchg(reserved_ios, ios, modified_ios);
253 ios = modified_ios;
254 }
255
256 return ios;
257}
258
e8603136
MS
259unsigned dm_get_reserved_bio_based_ios(void)
260{
261 return __dm_get_reserved_ios(&reserved_bio_based_ios,
262 RESERVED_BIO_BASED_IOS, RESERVED_MAX_IOS);
263}
264EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios);
265
f4790826
MS
266unsigned dm_get_reserved_rq_based_ios(void)
267{
268 return __dm_get_reserved_ios(&reserved_rq_based_ios,
269 RESERVED_REQUEST_BASED_IOS, RESERVED_MAX_IOS);
270}
271EXPORT_SYMBOL_GPL(dm_get_reserved_rq_based_ios);
272
1da177e4
LT
273static int __init local_init(void)
274{
51157b4a 275 int r = -ENOMEM;
1da177e4 276
1da177e4 277 /* allocate a slab for the dm_ios */
028867ac 278 _io_cache = KMEM_CACHE(dm_io, 0);
1da177e4 279 if (!_io_cache)
51157b4a 280 return r;
1da177e4 281
8fbf26ad
KU
282 _rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0);
283 if (!_rq_tio_cache)
dba14160 284 goto out_free_io_cache;
8fbf26ad 285
51e5b2bd 286 r = dm_uevent_init();
51157b4a 287 if (r)
23e5083b 288 goto out_free_rq_tio_cache;
51e5b2bd 289
acfe0ad7
MP
290 deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1);
291 if (!deferred_remove_workqueue) {
292 r = -ENOMEM;
293 goto out_uevent_exit;
294 }
295
1da177e4
LT
296 _major = major;
297 r = register_blkdev(_major, _name);
51157b4a 298 if (r < 0)
acfe0ad7 299 goto out_free_workqueue;
1da177e4
LT
300
301 if (!_major)
302 _major = r;
303
304 return 0;
51157b4a 305
acfe0ad7
MP
306out_free_workqueue:
307 destroy_workqueue(deferred_remove_workqueue);
51157b4a
KU
308out_uevent_exit:
309 dm_uevent_exit();
8fbf26ad
KU
310out_free_rq_tio_cache:
311 kmem_cache_destroy(_rq_tio_cache);
51157b4a
KU
312out_free_io_cache:
313 kmem_cache_destroy(_io_cache);
314
315 return r;
1da177e4
LT
316}
317
318static void local_exit(void)
319{
2c140a24 320 flush_scheduled_work();
acfe0ad7 321 destroy_workqueue(deferred_remove_workqueue);
2c140a24 322
8fbf26ad 323 kmem_cache_destroy(_rq_tio_cache);
1da177e4 324 kmem_cache_destroy(_io_cache);
00d59405 325 unregister_blkdev(_major, _name);
51e5b2bd 326 dm_uevent_exit();
1da177e4
LT
327
328 _major = 0;
329
330 DMINFO("cleaned up");
331}
332
b9249e55 333static int (*_inits[])(void) __initdata = {
1da177e4
LT
334 local_init,
335 dm_target_init,
336 dm_linear_init,
337 dm_stripe_init,
952b3557 338 dm_io_init,
945fa4d2 339 dm_kcopyd_init,
1da177e4 340 dm_interface_init,
fd2ed4d2 341 dm_statistics_init,
1da177e4
LT
342};
343
b9249e55 344static void (*_exits[])(void) = {
1da177e4
LT
345 local_exit,
346 dm_target_exit,
347 dm_linear_exit,
348 dm_stripe_exit,
952b3557 349 dm_io_exit,
945fa4d2 350 dm_kcopyd_exit,
1da177e4 351 dm_interface_exit,
fd2ed4d2 352 dm_statistics_exit,
1da177e4
LT
353};
354
355static int __init dm_init(void)
356{
357 const int count = ARRAY_SIZE(_inits);
358
359 int r, i;
360
361 for (i = 0; i < count; i++) {
362 r = _inits[i]();
363 if (r)
364 goto bad;
365 }
366
367 return 0;
368
369 bad:
370 while (i--)
371 _exits[i]();
372
373 return r;
374}
375
376static void __exit dm_exit(void)
377{
378 int i = ARRAY_SIZE(_exits);
379
380 while (i--)
381 _exits[i]();
d15b774c
AK
382
383 /*
384 * Should be empty by this point.
385 */
d15b774c 386 idr_destroy(&_minor_idr);
1da177e4
LT
387}
388
389/*
390 * Block device functions
391 */
432a212c
MA
392int dm_deleting_md(struct mapped_device *md)
393{
394 return test_bit(DMF_DELETING, &md->flags);
395}
396
fe5f9f2c 397static int dm_blk_open(struct block_device *bdev, fmode_t mode)
1da177e4
LT
398{
399 struct mapped_device *md;
400
fba9f90e
JM
401 spin_lock(&_minor_lock);
402
fe5f9f2c 403 md = bdev->bd_disk->private_data;
fba9f90e
JM
404 if (!md)
405 goto out;
406
5c6bd75d 407 if (test_bit(DMF_FREEING, &md->flags) ||
432a212c 408 dm_deleting_md(md)) {
fba9f90e
JM
409 md = NULL;
410 goto out;
411 }
412
1da177e4 413 dm_get(md);
5c6bd75d 414 atomic_inc(&md->open_count);
fba9f90e
JM
415
416out:
417 spin_unlock(&_minor_lock);
418
419 return md ? 0 : -ENXIO;
1da177e4
LT
420}
421
db2a144b 422static void dm_blk_close(struct gendisk *disk, fmode_t mode)
1da177e4 423{
fe5f9f2c 424 struct mapped_device *md = disk->private_data;
6e9624b8 425
4a1aeb98
MB
426 spin_lock(&_minor_lock);
427
2c140a24
MP
428 if (atomic_dec_and_test(&md->open_count) &&
429 (test_bit(DMF_DEFERRED_REMOVE, &md->flags)))
acfe0ad7 430 queue_work(deferred_remove_workqueue, &deferred_remove_work);
2c140a24 431
1da177e4 432 dm_put(md);
4a1aeb98
MB
433
434 spin_unlock(&_minor_lock);
1da177e4
LT
435}
436
5c6bd75d
AK
437int dm_open_count(struct mapped_device *md)
438{
439 return atomic_read(&md->open_count);
440}
441
442/*
443 * Guarantees nothing is using the device before it's deleted.
444 */
2c140a24 445int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred)
5c6bd75d
AK
446{
447 int r = 0;
448
449 spin_lock(&_minor_lock);
450
2c140a24 451 if (dm_open_count(md)) {
5c6bd75d 452 r = -EBUSY;
2c140a24
MP
453 if (mark_deferred)
454 set_bit(DMF_DEFERRED_REMOVE, &md->flags);
455 } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags))
456 r = -EEXIST;
5c6bd75d
AK
457 else
458 set_bit(DMF_DELETING, &md->flags);
459
460 spin_unlock(&_minor_lock);
461
462 return r;
463}
464
2c140a24
MP
465int dm_cancel_deferred_remove(struct mapped_device *md)
466{
467 int r = 0;
468
469 spin_lock(&_minor_lock);
470
471 if (test_bit(DMF_DELETING, &md->flags))
472 r = -EBUSY;
473 else
474 clear_bit(DMF_DEFERRED_REMOVE, &md->flags);
475
476 spin_unlock(&_minor_lock);
477
478 return r;
479}
480
481static void do_deferred_remove(struct work_struct *w)
482{
483 dm_deferred_remove();
484}
485
fd2ed4d2
MP
486sector_t dm_get_size(struct mapped_device *md)
487{
488 return get_capacity(md->disk);
489}
490
9974fa2c
MS
491struct request_queue *dm_get_md_queue(struct mapped_device *md)
492{
493 return md->queue;
494}
495
fd2ed4d2
MP
496struct dm_stats *dm_get_stats(struct mapped_device *md)
497{
498 return &md->stats;
499}
500
3ac51e74
DW
501static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
502{
503 struct mapped_device *md = bdev->bd_disk->private_data;
504
505 return dm_get_geometry(md, geo);
506}
507
fe5f9f2c 508static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
aa129a22
MB
509 unsigned int cmd, unsigned long arg)
510{
fe5f9f2c 511 struct mapped_device *md = bdev->bd_disk->private_data;
83d5e5b0 512 int srcu_idx;
6c182cd8 513 struct dm_table *map;
aa129a22
MB
514 struct dm_target *tgt;
515 int r = -ENOTTY;
516
6c182cd8 517retry:
83d5e5b0
MP
518 map = dm_get_live_table(md, &srcu_idx);
519
aa129a22
MB
520 if (!map || !dm_table_get_size(map))
521 goto out;
522
523 /* We only support devices that have a single target */
524 if (dm_table_get_num_targets(map) != 1)
525 goto out;
526
527 tgt = dm_table_get_target(map, 0);
528
4f186f8b 529 if (dm_suspended_md(md)) {
aa129a22
MB
530 r = -EAGAIN;
531 goto out;
532 }
533
534 if (tgt->type->ioctl)
647b3d00 535 r = tgt->type->ioctl(tgt, cmd, arg);
aa129a22
MB
536
537out:
83d5e5b0 538 dm_put_live_table(md, srcu_idx);
aa129a22 539
6c182cd8
HR
540 if (r == -ENOTCONN) {
541 msleep(10);
542 goto retry;
543 }
544
aa129a22
MB
545 return r;
546}
547
028867ac 548static struct dm_io *alloc_io(struct mapped_device *md)
1da177e4
LT
549{
550 return mempool_alloc(md->io_pool, GFP_NOIO);
551}
552
028867ac 553static void free_io(struct mapped_device *md, struct dm_io *io)
1da177e4
LT
554{
555 mempool_free(io, md->io_pool);
556}
557
028867ac 558static void free_tio(struct mapped_device *md, struct dm_target_io *tio)
1da177e4 559{
dba14160 560 bio_put(&tio->clone);
1da177e4
LT
561}
562
08885643
KU
563static struct dm_rq_target_io *alloc_rq_tio(struct mapped_device *md,
564 gfp_t gfp_mask)
cec47e3d 565{
5f015204 566 return mempool_alloc(md->io_pool, gfp_mask);
cec47e3d
KU
567}
568
569static void free_rq_tio(struct dm_rq_target_io *tio)
570{
5f015204 571 mempool_free(tio, tio->md->io_pool);
cec47e3d
KU
572}
573
90abb8c4
KU
574static int md_in_flight(struct mapped_device *md)
575{
576 return atomic_read(&md->pending[READ]) +
577 atomic_read(&md->pending[WRITE]);
578}
579
3eaf840e
JNN
580static void start_io_acct(struct dm_io *io)
581{
582 struct mapped_device *md = io->md;
fd2ed4d2 583 struct bio *bio = io->bio;
c9959059 584 int cpu;
fd2ed4d2 585 int rw = bio_data_dir(bio);
3eaf840e
JNN
586
587 io->start_time = jiffies;
588
074a7aca
TH
589 cpu = part_stat_lock();
590 part_round_stats(cpu, &dm_disk(md)->part0);
591 part_stat_unlock();
1e9bb880
SL
592 atomic_set(&dm_disk(md)->part0.in_flight[rw],
593 atomic_inc_return(&md->pending[rw]));
fd2ed4d2
MP
594
595 if (unlikely(dm_stats_used(&md->stats)))
4f024f37 596 dm_stats_account_io(&md->stats, bio->bi_rw, bio->bi_iter.bi_sector,
fd2ed4d2 597 bio_sectors(bio), false, 0, &io->stats_aux);
3eaf840e
JNN
598}
599
d221d2e7 600static void end_io_acct(struct dm_io *io)
3eaf840e
JNN
601{
602 struct mapped_device *md = io->md;
603 struct bio *bio = io->bio;
604 unsigned long duration = jiffies - io->start_time;
c9959059 605 int pending, cpu;
3eaf840e
JNN
606 int rw = bio_data_dir(bio);
607
074a7aca
TH
608 cpu = part_stat_lock();
609 part_round_stats(cpu, &dm_disk(md)->part0);
610 part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration);
611 part_stat_unlock();
3eaf840e 612
fd2ed4d2 613 if (unlikely(dm_stats_used(&md->stats)))
4f024f37 614 dm_stats_account_io(&md->stats, bio->bi_rw, bio->bi_iter.bi_sector,
fd2ed4d2
MP
615 bio_sectors(bio), true, duration, &io->stats_aux);
616
af7e466a
MP
617 /*
618 * After this is decremented the bio must not be touched if it is
d87f4c14 619 * a flush.
af7e466a 620 */
1e9bb880
SL
621 pending = atomic_dec_return(&md->pending[rw]);
622 atomic_set(&dm_disk(md)->part0.in_flight[rw], pending);
316d315b 623 pending += atomic_read(&md->pending[rw^0x1]);
3eaf840e 624
d221d2e7
MP
625 /* nudge anyone waiting on suspend queue */
626 if (!pending)
627 wake_up(&md->wait);
3eaf840e
JNN
628}
629
1da177e4
LT
630/*
631 * Add the bio to the list of deferred io.
632 */
92c63902 633static void queue_io(struct mapped_device *md, struct bio *bio)
1da177e4 634{
05447420 635 unsigned long flags;
1da177e4 636
05447420 637 spin_lock_irqsave(&md->deferred_lock, flags);
1da177e4 638 bio_list_add(&md->deferred, bio);
05447420 639 spin_unlock_irqrestore(&md->deferred_lock, flags);
6a8736d1 640 queue_work(md->wq, &md->work);
1da177e4
LT
641}
642
643/*
644 * Everyone (including functions in this file), should use this
645 * function to access the md->map field, and make sure they call
83d5e5b0 646 * dm_put_live_table() when finished.
1da177e4 647 */
83d5e5b0 648struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier)
1da177e4 649{
83d5e5b0
MP
650 *srcu_idx = srcu_read_lock(&md->io_barrier);
651
652 return srcu_dereference(md->map, &md->io_barrier);
653}
1da177e4 654
83d5e5b0
MP
655void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier)
656{
657 srcu_read_unlock(&md->io_barrier, srcu_idx);
658}
659
660void dm_sync_table(struct mapped_device *md)
661{
662 synchronize_srcu(&md->io_barrier);
663 synchronize_rcu_expedited();
664}
665
666/*
667 * A fast alternative to dm_get_live_table/dm_put_live_table.
668 * The caller must not block between these two functions.
669 */
670static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU)
671{
672 rcu_read_lock();
673 return rcu_dereference(md->map);
674}
1da177e4 675
83d5e5b0
MP
676static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)
677{
678 rcu_read_unlock();
1da177e4
LT
679}
680
86f1152b
BM
681/*
682 * Open a table device so we can use it as a map destination.
683 */
684static int open_table_device(struct table_device *td, dev_t dev,
685 struct mapped_device *md)
686{
687 static char *_claim_ptr = "I belong to device-mapper";
688 struct block_device *bdev;
689
690 int r;
691
692 BUG_ON(td->dm_dev.bdev);
693
694 bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _claim_ptr);
695 if (IS_ERR(bdev))
696 return PTR_ERR(bdev);
697
698 r = bd_link_disk_holder(bdev, dm_disk(md));
699 if (r) {
700 blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL);
701 return r;
702 }
703
704 td->dm_dev.bdev = bdev;
705 return 0;
706}
707
708/*
709 * Close a table device that we've been using.
710 */
711static void close_table_device(struct table_device *td, struct mapped_device *md)
712{
713 if (!td->dm_dev.bdev)
714 return;
715
716 bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md));
717 blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL);
718 td->dm_dev.bdev = NULL;
719}
720
721static struct table_device *find_table_device(struct list_head *l, dev_t dev,
722 fmode_t mode) {
723 struct table_device *td;
724
725 list_for_each_entry(td, l, list)
726 if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode)
727 return td;
728
729 return NULL;
730}
731
732int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode,
733 struct dm_dev **result) {
734 int r;
735 struct table_device *td;
736
737 mutex_lock(&md->table_devices_lock);
738 td = find_table_device(&md->table_devices, dev, mode);
739 if (!td) {
740 td = kmalloc(sizeof(*td), GFP_KERNEL);
741 if (!td) {
742 mutex_unlock(&md->table_devices_lock);
743 return -ENOMEM;
744 }
745
746 td->dm_dev.mode = mode;
747 td->dm_dev.bdev = NULL;
748
749 if ((r = open_table_device(td, dev, md))) {
750 mutex_unlock(&md->table_devices_lock);
751 kfree(td);
752 return r;
753 }
754
755 format_dev_t(td->dm_dev.name, dev);
756
757 atomic_set(&td->count, 0);
758 list_add(&td->list, &md->table_devices);
759 }
760 atomic_inc(&td->count);
761 mutex_unlock(&md->table_devices_lock);
762
763 *result = &td->dm_dev;
764 return 0;
765}
766EXPORT_SYMBOL_GPL(dm_get_table_device);
767
768void dm_put_table_device(struct mapped_device *md, struct dm_dev *d)
769{
770 struct table_device *td = container_of(d, struct table_device, dm_dev);
771
772 mutex_lock(&md->table_devices_lock);
773 if (atomic_dec_and_test(&td->count)) {
774 close_table_device(td, md);
775 list_del(&td->list);
776 kfree(td);
777 }
778 mutex_unlock(&md->table_devices_lock);
779}
780EXPORT_SYMBOL(dm_put_table_device);
781
782static void free_table_devices(struct list_head *devices)
783{
784 struct list_head *tmp, *next;
785
786 list_for_each_safe(tmp, next, devices) {
787 struct table_device *td = list_entry(tmp, struct table_device, list);
788
789 DMWARN("dm_destroy: %s still exists with %d references",
790 td->dm_dev.name, atomic_read(&td->count));
791 kfree(td);
792 }
793}
794
3ac51e74
DW
795/*
796 * Get the geometry associated with a dm device
797 */
798int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
799{
800 *geo = md->geometry;
801
802 return 0;
803}
804
805/*
806 * Set the geometry of a device.
807 */
808int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
809{
810 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
811
812 if (geo->start > sz) {
813 DMWARN("Start sector is beyond the geometry limits.");
814 return -EINVAL;
815 }
816
817 md->geometry = *geo;
818
819 return 0;
820}
821
1da177e4
LT
822/*-----------------------------------------------------------------
823 * CRUD START:
824 * A more elegant soln is in the works that uses the queue
825 * merge fn, unfortunately there are a couple of changes to
826 * the block layer that I want to make for this. So in the
827 * interests of getting something for people to use I give
828 * you this clearly demarcated crap.
829 *---------------------------------------------------------------*/
830
2e93ccc1
KU
831static int __noflush_suspending(struct mapped_device *md)
832{
833 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
834}
835
1da177e4
LT
836/*
837 * Decrements the number of outstanding ios that a bio has been
838 * cloned into, completing the original io if necc.
839 */
858119e1 840static void dec_pending(struct dm_io *io, int error)
1da177e4 841{
2e93ccc1 842 unsigned long flags;
b35f8caa
MB
843 int io_error;
844 struct bio *bio;
845 struct mapped_device *md = io->md;
2e93ccc1
KU
846
847 /* Push-back supersedes any I/O errors */
f88fb981
KU
848 if (unlikely(error)) {
849 spin_lock_irqsave(&io->endio_lock, flags);
850 if (!(io->error > 0 && __noflush_suspending(md)))
851 io->error = error;
852 spin_unlock_irqrestore(&io->endio_lock, flags);
853 }
1da177e4
LT
854
855 if (atomic_dec_and_test(&io->io_count)) {
2e93ccc1
KU
856 if (io->error == DM_ENDIO_REQUEUE) {
857 /*
858 * Target requested pushing back the I/O.
2e93ccc1 859 */
022c2611 860 spin_lock_irqsave(&md->deferred_lock, flags);
6a8736d1
TH
861 if (__noflush_suspending(md))
862 bio_list_add_head(&md->deferred, io->bio);
863 else
2e93ccc1
KU
864 /* noflush suspend was interrupted. */
865 io->error = -EIO;
022c2611 866 spin_unlock_irqrestore(&md->deferred_lock, flags);
2e93ccc1
KU
867 }
868
b35f8caa
MB
869 io_error = io->error;
870 bio = io->bio;
6a8736d1
TH
871 end_io_acct(io);
872 free_io(md, io);
873
874 if (io_error == DM_ENDIO_REQUEUE)
875 return;
2e93ccc1 876
4f024f37 877 if ((bio->bi_rw & REQ_FLUSH) && bio->bi_iter.bi_size) {
af7e466a 878 /*
6a8736d1
TH
879 * Preflush done for flush with data, reissue
880 * without REQ_FLUSH.
af7e466a 881 */
6a8736d1
TH
882 bio->bi_rw &= ~REQ_FLUSH;
883 queue_io(md, bio);
af7e466a 884 } else {
b372d360 885 /* done with normal IO or empty flush */
0a82a8d1 886 trace_block_bio_complete(md->queue, bio, io_error);
b372d360 887 bio_endio(bio, io_error);
b35f8caa 888 }
1da177e4
LT
889 }
890}
891
7eee4ae2
MS
892static void disable_write_same(struct mapped_device *md)
893{
894 struct queue_limits *limits = dm_get_queue_limits(md);
895
896 /* device doesn't really support WRITE SAME, disable it */
897 limits->max_write_same_sectors = 0;
898}
899
6712ecf8 900static void clone_endio(struct bio *bio, int error)
1da177e4
LT
901{
902 int r = 0;
bfc6d41c 903 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
b35f8caa 904 struct dm_io *io = tio->io;
9faf400f 905 struct mapped_device *md = tio->io->md;
1da177e4
LT
906 dm_endio_fn endio = tio->ti->type->end_io;
907
1da177e4
LT
908 if (!bio_flagged(bio, BIO_UPTODATE) && !error)
909 error = -EIO;
910
911 if (endio) {
7de3ee57 912 r = endio(tio->ti, bio, error);
2e93ccc1
KU
913 if (r < 0 || r == DM_ENDIO_REQUEUE)
914 /*
915 * error and requeue request are handled
916 * in dec_pending().
917 */
1da177e4 918 error = r;
45cbcd79
KU
919 else if (r == DM_ENDIO_INCOMPLETE)
920 /* The target will handle the io */
6712ecf8 921 return;
45cbcd79
KU
922 else if (r) {
923 DMWARN("unimplemented target endio return value: %d", r);
924 BUG();
925 }
1da177e4
LT
926 }
927
7eee4ae2
MS
928 if (unlikely(r == -EREMOTEIO && (bio->bi_rw & REQ_WRITE_SAME) &&
929 !bdev_get_queue(bio->bi_bdev)->limits.max_write_same_sectors))
930 disable_write_same(md);
931
9faf400f 932 free_tio(md, tio);
b35f8caa 933 dec_pending(io, error);
1da177e4
LT
934}
935
cec47e3d
KU
936/*
937 * Partial completion handling for request-based dm
938 */
939static void end_clone_bio(struct bio *clone, int error)
940{
bfc6d41c
MP
941 struct dm_rq_clone_bio_info *info =
942 container_of(clone, struct dm_rq_clone_bio_info, clone);
cec47e3d
KU
943 struct dm_rq_target_io *tio = info->tio;
944 struct bio *bio = info->orig;
4f024f37 945 unsigned int nr_bytes = info->orig->bi_iter.bi_size;
cec47e3d
KU
946
947 bio_put(clone);
948
949 if (tio->error)
950 /*
951 * An error has already been detected on the request.
952 * Once error occurred, just let clone->end_io() handle
953 * the remainder.
954 */
955 return;
956 else if (error) {
957 /*
958 * Don't notice the error to the upper layer yet.
959 * The error handling decision is made by the target driver,
960 * when the request is completed.
961 */
962 tio->error = error;
963 return;
964 }
965
966 /*
967 * I/O for the bio successfully completed.
968 * Notice the data completion to the upper layer.
969 */
970
971 /*
972 * bios are processed from the head of the list.
973 * So the completing bio should always be rq->bio.
974 * If it's not, something wrong is happening.
975 */
976 if (tio->orig->bio != bio)
977 DMERR("bio completion is going in the middle of the request");
978
979 /*
980 * Update the original request.
981 * Do not use blk_end_request() here, because it may complete
982 * the original request before the clone, and break the ordering.
983 */
984 blk_update_request(tio->orig, 0, nr_bytes);
985}
986
987/*
988 * Don't touch any member of the md after calling this function because
989 * the md may be freed in dm_put() at the end of this function.
990 * Or do dm_get() before calling this function and dm_put() later.
991 */
b4324fee 992static void rq_completed(struct mapped_device *md, int rw, int run_queue)
cec47e3d 993{
b4324fee 994 atomic_dec(&md->pending[rw]);
cec47e3d
KU
995
996 /* nudge anyone waiting on suspend queue */
b4324fee 997 if (!md_in_flight(md))
cec47e3d
KU
998 wake_up(&md->wait);
999
a8c32a5c
JA
1000 /*
1001 * Run this off this callpath, as drivers could invoke end_io while
1002 * inside their request_fn (and holding the queue lock). Calling
1003 * back into ->request_fn() could deadlock attempting to grab the
1004 * queue lock again.
1005 */
cec47e3d 1006 if (run_queue)
a8c32a5c 1007 blk_run_queue_async(md->queue);
cec47e3d
KU
1008
1009 /*
1010 * dm_put() must be at the end of this function. See the comment above
1011 */
1012 dm_put(md);
1013}
1014
a77e28c7
KU
1015static void free_rq_clone(struct request *clone)
1016{
1017 struct dm_rq_target_io *tio = clone->end_io_data;
1018
1019 blk_rq_unprep_clone(clone);
1020 free_rq_tio(tio);
1021}
1022
980691e5
KU
1023/*
1024 * Complete the clone and the original request.
1025 * Must be called without queue lock.
1026 */
1027static void dm_end_request(struct request *clone, int error)
1028{
1029 int rw = rq_data_dir(clone);
1030 struct dm_rq_target_io *tio = clone->end_io_data;
1031 struct mapped_device *md = tio->md;
1032 struct request *rq = tio->orig;
1033
29e4013d 1034 if (rq->cmd_type == REQ_TYPE_BLOCK_PC) {
980691e5
KU
1035 rq->errors = clone->errors;
1036 rq->resid_len = clone->resid_len;
1037
1038 if (rq->sense)
1039 /*
1040 * We are using the sense buffer of the original
1041 * request.
1042 * So setting the length of the sense data is enough.
1043 */
1044 rq->sense_len = clone->sense_len;
1045 }
1046
1047 free_rq_clone(clone);
29e4013d
TH
1048 blk_end_request_all(rq, error);
1049 rq_completed(md, rw, true);
980691e5
KU
1050}
1051
cec47e3d
KU
1052static void dm_unprep_request(struct request *rq)
1053{
1054 struct request *clone = rq->special;
cec47e3d
KU
1055
1056 rq->special = NULL;
1057 rq->cmd_flags &= ~REQ_DONTPREP;
1058
a77e28c7 1059 free_rq_clone(clone);
cec47e3d
KU
1060}
1061
1062/*
1063 * Requeue the original request of a clone.
1064 */
1065void dm_requeue_unmapped_request(struct request *clone)
1066{
b4324fee 1067 int rw = rq_data_dir(clone);
cec47e3d
KU
1068 struct dm_rq_target_io *tio = clone->end_io_data;
1069 struct mapped_device *md = tio->md;
1070 struct request *rq = tio->orig;
1071 struct request_queue *q = rq->q;
1072 unsigned long flags;
1073
1074 dm_unprep_request(rq);
1075
1076 spin_lock_irqsave(q->queue_lock, flags);
cec47e3d
KU
1077 blk_requeue_request(q, rq);
1078 spin_unlock_irqrestore(q->queue_lock, flags);
1079
b4324fee 1080 rq_completed(md, rw, 0);
cec47e3d
KU
1081}
1082EXPORT_SYMBOL_GPL(dm_requeue_unmapped_request);
1083
1084static void __stop_queue(struct request_queue *q)
1085{
1086 blk_stop_queue(q);
1087}
1088
1089static void stop_queue(struct request_queue *q)
1090{
1091 unsigned long flags;
1092
1093 spin_lock_irqsave(q->queue_lock, flags);
1094 __stop_queue(q);
1095 spin_unlock_irqrestore(q->queue_lock, flags);
1096}
1097
1098static void __start_queue(struct request_queue *q)
1099{
1100 if (blk_queue_stopped(q))
1101 blk_start_queue(q);
1102}
1103
1104static void start_queue(struct request_queue *q)
1105{
1106 unsigned long flags;
1107
1108 spin_lock_irqsave(q->queue_lock, flags);
1109 __start_queue(q);
1110 spin_unlock_irqrestore(q->queue_lock, flags);
1111}
1112
11a68244 1113static void dm_done(struct request *clone, int error, bool mapped)
cec47e3d 1114{
11a68244 1115 int r = error;
cec47e3d 1116 struct dm_rq_target_io *tio = clone->end_io_data;
ba1cbad9 1117 dm_request_endio_fn rq_end_io = NULL;
cec47e3d 1118
ba1cbad9
MS
1119 if (tio->ti) {
1120 rq_end_io = tio->ti->type->rq_end_io;
1121
1122 if (mapped && rq_end_io)
1123 r = rq_end_io(tio->ti, clone, error, &tio->info);
1124 }
cec47e3d 1125
7eee4ae2
MS
1126 if (unlikely(r == -EREMOTEIO && (clone->cmd_flags & REQ_WRITE_SAME) &&
1127 !clone->q->limits.max_write_same_sectors))
1128 disable_write_same(tio->md);
1129
11a68244 1130 if (r <= 0)
cec47e3d 1131 /* The target wants to complete the I/O */
11a68244
KU
1132 dm_end_request(clone, r);
1133 else if (r == DM_ENDIO_INCOMPLETE)
cec47e3d
KU
1134 /* The target will handle the I/O */
1135 return;
11a68244 1136 else if (r == DM_ENDIO_REQUEUE)
cec47e3d
KU
1137 /* The target wants to requeue the I/O */
1138 dm_requeue_unmapped_request(clone);
1139 else {
11a68244 1140 DMWARN("unimplemented target endio return value: %d", r);
cec47e3d
KU
1141 BUG();
1142 }
1143}
1144
11a68244
KU
1145/*
1146 * Request completion handler for request-based dm
1147 */
1148static void dm_softirq_done(struct request *rq)
1149{
1150 bool mapped = true;
1151 struct request *clone = rq->completion_data;
1152 struct dm_rq_target_io *tio = clone->end_io_data;
1153
1154 if (rq->cmd_flags & REQ_FAILED)
1155 mapped = false;
1156
1157 dm_done(clone, tio->error, mapped);
1158}
1159
cec47e3d
KU
1160/*
1161 * Complete the clone and the original request with the error status
1162 * through softirq context.
1163 */
1164static void dm_complete_request(struct request *clone, int error)
1165{
1166 struct dm_rq_target_io *tio = clone->end_io_data;
1167 struct request *rq = tio->orig;
1168
1169 tio->error = error;
1170 rq->completion_data = clone;
1171 blk_complete_request(rq);
1172}
1173
1174/*
1175 * Complete the not-mapped clone and the original request with the error status
1176 * through softirq context.
1177 * Target's rq_end_io() function isn't called.
1178 * This may be used when the target's map_rq() function fails.
1179 */
1180void dm_kill_unmapped_request(struct request *clone, int error)
1181{
1182 struct dm_rq_target_io *tio = clone->end_io_data;
1183 struct request *rq = tio->orig;
1184
1185 rq->cmd_flags |= REQ_FAILED;
1186 dm_complete_request(clone, error);
1187}
1188EXPORT_SYMBOL_GPL(dm_kill_unmapped_request);
1189
1190/*
1191 * Called with the queue lock held
1192 */
1193static void end_clone_request(struct request *clone, int error)
1194{
1195 /*
1196 * For just cleaning up the information of the queue in which
1197 * the clone was dispatched.
1198 * The clone is *NOT* freed actually here because it is alloced from
1199 * dm own mempool and REQ_ALLOCED isn't set in clone->cmd_flags.
1200 */
1201 __blk_put_request(clone->q, clone);
1202
1203 /*
1204 * Actual request completion is done in a softirq context which doesn't
1205 * hold the queue lock. Otherwise, deadlock could occur because:
1206 * - another request may be submitted by the upper level driver
1207 * of the stacking during the completion
1208 * - the submission which requires queue lock may be done
1209 * against this queue
1210 */
1211 dm_complete_request(clone, error);
1212}
1213
56a67df7
MS
1214/*
1215 * Return maximum size of I/O possible at the supplied sector up to the current
1216 * target boundary.
1217 */
1218static sector_t max_io_len_target_boundary(sector_t sector, struct dm_target *ti)
1219{
1220 sector_t target_offset = dm_target_offset(ti, sector);
1221
1222 return ti->len - target_offset;
1223}
1224
1225static sector_t max_io_len(sector_t sector, struct dm_target *ti)
1da177e4 1226{
56a67df7 1227 sector_t len = max_io_len_target_boundary(sector, ti);
542f9038 1228 sector_t offset, max_len;
1da177e4
LT
1229
1230 /*
542f9038 1231 * Does the target need to split even further?
1da177e4 1232 */
542f9038
MS
1233 if (ti->max_io_len) {
1234 offset = dm_target_offset(ti, sector);
1235 if (unlikely(ti->max_io_len & (ti->max_io_len - 1)))
1236 max_len = sector_div(offset, ti->max_io_len);
1237 else
1238 max_len = offset & (ti->max_io_len - 1);
1239 max_len = ti->max_io_len - max_len;
1240
1241 if (len > max_len)
1242 len = max_len;
1da177e4
LT
1243 }
1244
1245 return len;
1246}
1247
542f9038
MS
1248int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
1249{
1250 if (len > UINT_MAX) {
1251 DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)",
1252 (unsigned long long)len, UINT_MAX);
1253 ti->error = "Maximum size of target IO is too large";
1254 return -EINVAL;
1255 }
1256
1257 ti->max_io_len = (uint32_t) len;
1258
1259 return 0;
1260}
1261EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);
1262
1dd40c3e
MP
1263/*
1264 * A target may call dm_accept_partial_bio only from the map routine. It is
1265 * allowed for all bio types except REQ_FLUSH.
1266 *
1267 * dm_accept_partial_bio informs the dm that the target only wants to process
1268 * additional n_sectors sectors of the bio and the rest of the data should be
1269 * sent in a next bio.
1270 *
1271 * A diagram that explains the arithmetics:
1272 * +--------------------+---------------+-------+
1273 * | 1 | 2 | 3 |
1274 * +--------------------+---------------+-------+
1275 *
1276 * <-------------- *tio->len_ptr --------------->
1277 * <------- bi_size ------->
1278 * <-- n_sectors -->
1279 *
1280 * Region 1 was already iterated over with bio_advance or similar function.
1281 * (it may be empty if the target doesn't use bio_advance)
1282 * Region 2 is the remaining bio size that the target wants to process.
1283 * (it may be empty if region 1 is non-empty, although there is no reason
1284 * to make it empty)
1285 * The target requires that region 3 is to be sent in the next bio.
1286 *
1287 * If the target wants to receive multiple copies of the bio (via num_*bios, etc),
1288 * the partially processed part (the sum of regions 1+2) must be the same for all
1289 * copies of the bio.
1290 */
1291void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors)
1292{
1293 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
1294 unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT;
1295 BUG_ON(bio->bi_rw & REQ_FLUSH);
1296 BUG_ON(bi_size > *tio->len_ptr);
1297 BUG_ON(n_sectors > bi_size);
1298 *tio->len_ptr -= bi_size - n_sectors;
1299 bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
1300}
1301EXPORT_SYMBOL_GPL(dm_accept_partial_bio);
1302
bd2a49b8 1303static void __map_bio(struct dm_target_io *tio)
1da177e4
LT
1304{
1305 int r;
2056a782 1306 sector_t sector;
9faf400f 1307 struct mapped_device *md;
dba14160 1308 struct bio *clone = &tio->clone;
bd2a49b8 1309 struct dm_target *ti = tio->ti;
1da177e4 1310
1da177e4 1311 clone->bi_end_io = clone_endio;
1da177e4
LT
1312
1313 /*
1314 * Map the clone. If r == 0 we don't need to do
1315 * anything, the target has assumed ownership of
1316 * this io.
1317 */
1318 atomic_inc(&tio->io->io_count);
4f024f37 1319 sector = clone->bi_iter.bi_sector;
7de3ee57 1320 r = ti->type->map(ti, clone);
45cbcd79 1321 if (r == DM_MAPIO_REMAPPED) {
1da177e4 1322 /* the bio has been remapped so dispatch it */
2056a782 1323
d07335e5
MS
1324 trace_block_bio_remap(bdev_get_queue(clone->bi_bdev), clone,
1325 tio->io->bio->bi_bdev->bd_dev, sector);
2056a782 1326
1da177e4 1327 generic_make_request(clone);
2e93ccc1
KU
1328 } else if (r < 0 || r == DM_MAPIO_REQUEUE) {
1329 /* error the io and bail out, or requeue it if needed */
9faf400f
SB
1330 md = tio->io->md;
1331 dec_pending(tio->io, r);
9faf400f 1332 free_tio(md, tio);
45cbcd79
KU
1333 } else if (r) {
1334 DMWARN("unimplemented target map return value: %d", r);
1335 BUG();
1da177e4
LT
1336 }
1337}
1338
1339struct clone_info {
1340 struct mapped_device *md;
1341 struct dm_table *map;
1342 struct bio *bio;
1343 struct dm_io *io;
1344 sector_t sector;
e0d6609a 1345 unsigned sector_count;
1da177e4
LT
1346};
1347
e0d6609a 1348static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len)
bd2a49b8 1349{
4f024f37
KO
1350 bio->bi_iter.bi_sector = sector;
1351 bio->bi_iter.bi_size = to_bytes(len);
1da177e4
LT
1352}
1353
1354/*
1355 * Creates a bio that consists of range of complete bvecs.
1356 */
dba14160 1357static void clone_bio(struct dm_target_io *tio, struct bio *bio,
1c3b13e6 1358 sector_t sector, unsigned len)
1da177e4 1359{
dba14160 1360 struct bio *clone = &tio->clone;
1da177e4 1361
1c3b13e6
KO
1362 __bio_clone_fast(clone, bio);
1363
1364 if (bio_integrity(bio))
1365 bio_integrity_clone(clone, bio, GFP_NOIO);
bd2a49b8 1366
1c3b13e6
KO
1367 bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector));
1368 clone->bi_iter.bi_size = to_bytes(len);
1369
1370 if (bio_integrity(bio))
1371 bio_integrity_trim(clone, 0, len);
1da177e4
LT
1372}
1373
9015df24 1374static struct dm_target_io *alloc_tio(struct clone_info *ci,
99778273 1375 struct dm_target *ti,
55a62eef 1376 unsigned target_bio_nr)
f9ab94ce 1377{
dba14160
MP
1378 struct dm_target_io *tio;
1379 struct bio *clone;
1380
99778273 1381 clone = bio_alloc_bioset(GFP_NOIO, 0, ci->md->bs);
dba14160 1382 tio = container_of(clone, struct dm_target_io, clone);
f9ab94ce
MP
1383
1384 tio->io = ci->io;
1385 tio->ti = ti;
55a62eef 1386 tio->target_bio_nr = target_bio_nr;
9015df24
AK
1387
1388 return tio;
1389}
1390
14fe594d
AK
1391static void __clone_and_map_simple_bio(struct clone_info *ci,
1392 struct dm_target *ti,
1dd40c3e 1393 unsigned target_bio_nr, unsigned *len)
9015df24 1394{
99778273 1395 struct dm_target_io *tio = alloc_tio(ci, ti, target_bio_nr);
dba14160 1396 struct bio *clone = &tio->clone;
9015df24 1397
1dd40c3e
MP
1398 tio->len_ptr = len;
1399
99778273 1400 __bio_clone_fast(clone, ci->bio);
bd2a49b8 1401 if (len)
1dd40c3e 1402 bio_setup_sector(clone, ci->sector, *len);
f9ab94ce 1403
bd2a49b8 1404 __map_bio(tio);
f9ab94ce
MP
1405}
1406
14fe594d 1407static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,
1dd40c3e 1408 unsigned num_bios, unsigned *len)
06a426ce 1409{
55a62eef 1410 unsigned target_bio_nr;
06a426ce 1411
55a62eef 1412 for (target_bio_nr = 0; target_bio_nr < num_bios; target_bio_nr++)
14fe594d 1413 __clone_and_map_simple_bio(ci, ti, target_bio_nr, len);
06a426ce
MS
1414}
1415
14fe594d 1416static int __send_empty_flush(struct clone_info *ci)
f9ab94ce 1417{
06a426ce 1418 unsigned target_nr = 0;
f9ab94ce
MP
1419 struct dm_target *ti;
1420
b372d360 1421 BUG_ON(bio_has_data(ci->bio));
f9ab94ce 1422 while ((ti = dm_table_get_target(ci->map, target_nr++)))
1dd40c3e 1423 __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL);
f9ab94ce 1424
f9ab94ce
MP
1425 return 0;
1426}
1427
e4c93811 1428static void __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti,
1dd40c3e 1429 sector_t sector, unsigned *len)
5ae89a87 1430{
dba14160 1431 struct bio *bio = ci->bio;
5ae89a87 1432 struct dm_target_io *tio;
b0d8ed4d
AK
1433 unsigned target_bio_nr;
1434 unsigned num_target_bios = 1;
5ae89a87 1435
b0d8ed4d
AK
1436 /*
1437 * Does the target want to receive duplicate copies of the bio?
1438 */
1439 if (bio_data_dir(bio) == WRITE && ti->num_write_bios)
1440 num_target_bios = ti->num_write_bios(ti, bio);
e4c93811 1441
b0d8ed4d 1442 for (target_bio_nr = 0; target_bio_nr < num_target_bios; target_bio_nr++) {
99778273 1443 tio = alloc_tio(ci, ti, target_bio_nr);
1dd40c3e
MP
1444 tio->len_ptr = len;
1445 clone_bio(tio, bio, sector, *len);
b0d8ed4d
AK
1446 __map_bio(tio);
1447 }
5ae89a87
MS
1448}
1449
55a62eef 1450typedef unsigned (*get_num_bios_fn)(struct dm_target *ti);
23508a96 1451
55a62eef 1452static unsigned get_num_discard_bios(struct dm_target *ti)
23508a96 1453{
55a62eef 1454 return ti->num_discard_bios;
23508a96
MS
1455}
1456
55a62eef 1457static unsigned get_num_write_same_bios(struct dm_target *ti)
23508a96 1458{
55a62eef 1459 return ti->num_write_same_bios;
23508a96
MS
1460}
1461
1462typedef bool (*is_split_required_fn)(struct dm_target *ti);
1463
1464static bool is_split_required_for_discard(struct dm_target *ti)
1465{
55a62eef 1466 return ti->split_discard_bios;
23508a96
MS
1467}
1468
14fe594d
AK
1469static int __send_changing_extent_only(struct clone_info *ci,
1470 get_num_bios_fn get_num_bios,
1471 is_split_required_fn is_split_required)
5ae89a87
MS
1472{
1473 struct dm_target *ti;
e0d6609a 1474 unsigned len;
55a62eef 1475 unsigned num_bios;
5ae89a87 1476
a79245b3
MS
1477 do {
1478 ti = dm_table_find_target(ci->map, ci->sector);
1479 if (!dm_target_is_valid(ti))
1480 return -EIO;
5ae89a87 1481
5ae89a87 1482 /*
23508a96
MS
1483 * Even though the device advertised support for this type of
1484 * request, that does not mean every target supports it, and
936688d7 1485 * reconfiguration might also have changed that since the
a79245b3 1486 * check was performed.
5ae89a87 1487 */
55a62eef
AK
1488 num_bios = get_num_bios ? get_num_bios(ti) : 0;
1489 if (!num_bios)
a79245b3 1490 return -EOPNOTSUPP;
5ae89a87 1491
23508a96 1492 if (is_split_required && !is_split_required(ti))
e0d6609a 1493 len = min((sector_t)ci->sector_count, max_io_len_target_boundary(ci->sector, ti));
7acf0277 1494 else
e0d6609a 1495 len = min((sector_t)ci->sector_count, max_io_len(ci->sector, ti));
06a426ce 1496
1dd40c3e 1497 __send_duplicate_bios(ci, ti, num_bios, &len);
a79245b3
MS
1498
1499 ci->sector += len;
1500 } while (ci->sector_count -= len);
5ae89a87
MS
1501
1502 return 0;
1503}
1504
14fe594d 1505static int __send_discard(struct clone_info *ci)
23508a96 1506{
14fe594d
AK
1507 return __send_changing_extent_only(ci, get_num_discard_bios,
1508 is_split_required_for_discard);
23508a96
MS
1509}
1510
14fe594d 1511static int __send_write_same(struct clone_info *ci)
23508a96 1512{
14fe594d 1513 return __send_changing_extent_only(ci, get_num_write_same_bios, NULL);
23508a96
MS
1514}
1515
e4c93811
AK
1516/*
1517 * Select the correct strategy for processing a non-flush bio.
1518 */
14fe594d 1519static int __split_and_process_non_flush(struct clone_info *ci)
1da177e4 1520{
dba14160 1521 struct bio *bio = ci->bio;
512875bd 1522 struct dm_target *ti;
1c3b13e6 1523 unsigned len;
1da177e4 1524
5ae89a87 1525 if (unlikely(bio->bi_rw & REQ_DISCARD))
14fe594d 1526 return __send_discard(ci);
23508a96 1527 else if (unlikely(bio->bi_rw & REQ_WRITE_SAME))
14fe594d 1528 return __send_write_same(ci);
5ae89a87 1529
512875bd
JN
1530 ti = dm_table_find_target(ci->map, ci->sector);
1531 if (!dm_target_is_valid(ti))
1532 return -EIO;
1533
1c3b13e6 1534 len = min_t(sector_t, max_io_len(ci->sector, ti), ci->sector_count);
1da177e4 1535
1dd40c3e 1536 __clone_and_map_data_bio(ci, ti, ci->sector, &len);
1da177e4 1537
1c3b13e6
KO
1538 ci->sector += len;
1539 ci->sector_count -= len;
1da177e4 1540
1c3b13e6 1541 return 0;
1da177e4
LT
1542}
1543
1544/*
14fe594d 1545 * Entry point to split a bio into clones and submit them to the targets.
1da177e4 1546 */
83d5e5b0
MP
1547static void __split_and_process_bio(struct mapped_device *md,
1548 struct dm_table *map, struct bio *bio)
1da177e4
LT
1549{
1550 struct clone_info ci;
512875bd 1551 int error = 0;
1da177e4 1552
83d5e5b0 1553 if (unlikely(!map)) {
6a8736d1 1554 bio_io_error(bio);
f0b9a450
MP
1555 return;
1556 }
692d0eb9 1557
83d5e5b0 1558 ci.map = map;
1da177e4 1559 ci.md = md;
1da177e4
LT
1560 ci.io = alloc_io(md);
1561 ci.io->error = 0;
1562 atomic_set(&ci.io->io_count, 1);
1563 ci.io->bio = bio;
1564 ci.io->md = md;
f88fb981 1565 spin_lock_init(&ci.io->endio_lock);
4f024f37 1566 ci.sector = bio->bi_iter.bi_sector;
1da177e4 1567
3eaf840e 1568 start_io_acct(ci.io);
bd2a49b8 1569
b372d360
MS
1570 if (bio->bi_rw & REQ_FLUSH) {
1571 ci.bio = &ci.md->flush_bio;
1572 ci.sector_count = 0;
14fe594d 1573 error = __send_empty_flush(&ci);
b372d360
MS
1574 /* dec_pending submits any data associated with flush */
1575 } else {
6a8736d1 1576 ci.bio = bio;
d87f4c14 1577 ci.sector_count = bio_sectors(bio);
b372d360 1578 while (ci.sector_count && !error)
14fe594d 1579 error = __split_and_process_non_flush(&ci);
d87f4c14 1580 }
1da177e4
LT
1581
1582 /* drop the extra reference count */
512875bd 1583 dec_pending(ci.io, error);
1da177e4
LT
1584}
1585/*-----------------------------------------------------------------
1586 * CRUD END
1587 *---------------------------------------------------------------*/
1588
f6fccb12
MB
1589static int dm_merge_bvec(struct request_queue *q,
1590 struct bvec_merge_data *bvm,
1591 struct bio_vec *biovec)
1592{
1593 struct mapped_device *md = q->queuedata;
83d5e5b0 1594 struct dm_table *map = dm_get_live_table_fast(md);
f6fccb12
MB
1595 struct dm_target *ti;
1596 sector_t max_sectors;
5037108a 1597 int max_size = 0;
f6fccb12
MB
1598
1599 if (unlikely(!map))
5037108a 1600 goto out;
f6fccb12
MB
1601
1602 ti = dm_table_find_target(map, bvm->bi_sector);
b01cd5ac 1603 if (!dm_target_is_valid(ti))
83d5e5b0 1604 goto out;
f6fccb12
MB
1605
1606 /*
1607 * Find maximum amount of I/O that won't need splitting
1608 */
56a67df7 1609 max_sectors = min(max_io_len(bvm->bi_sector, ti),
f6fccb12
MB
1610 (sector_t) BIO_MAX_SECTORS);
1611 max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
1612 if (max_size < 0)
1613 max_size = 0;
1614
1615 /*
1616 * merge_bvec_fn() returns number of bytes
1617 * it can accept at this offset
1618 * max is precomputed maximal io size
1619 */
1620 if (max_size && ti->type->merge)
1621 max_size = ti->type->merge(ti, bvm, biovec, max_size);
8cbeb67a
MP
1622 /*
1623 * If the target doesn't support merge method and some of the devices
1624 * provided their merge_bvec method (we know this by looking at
1625 * queue_max_hw_sectors), then we can't allow bios with multiple vector
1626 * entries. So always set max_size to 0, and the code below allows
1627 * just one page.
1628 */
1629 else if (queue_max_hw_sectors(q) <= PAGE_SIZE >> 9)
8cbeb67a 1630 max_size = 0;
f6fccb12 1631
5037108a 1632out:
83d5e5b0 1633 dm_put_live_table_fast(md);
f6fccb12
MB
1634 /*
1635 * Always allow an entire first page
1636 */
1637 if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT))
1638 max_size = biovec->bv_len;
1639
f6fccb12
MB
1640 return max_size;
1641}
1642
1da177e4
LT
1643/*
1644 * The request function that just remaps the bio built up by
1645 * dm_merge_bvec.
1646 */
5a7bbad2 1647static void _dm_request(struct request_queue *q, struct bio *bio)
1da177e4 1648{
12f03a49 1649 int rw = bio_data_dir(bio);
1da177e4 1650 struct mapped_device *md = q->queuedata;
c9959059 1651 int cpu;
83d5e5b0
MP
1652 int srcu_idx;
1653 struct dm_table *map;
1da177e4 1654
83d5e5b0 1655 map = dm_get_live_table(md, &srcu_idx);
1da177e4 1656
074a7aca
TH
1657 cpu = part_stat_lock();
1658 part_stat_inc(cpu, &dm_disk(md)->part0, ios[rw]);
1659 part_stat_add(cpu, &dm_disk(md)->part0, sectors[rw], bio_sectors(bio));
1660 part_stat_unlock();
12f03a49 1661
6a8736d1
TH
1662 /* if we're suspended, we have to queue this io for later */
1663 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
83d5e5b0 1664 dm_put_live_table(md, srcu_idx);
1da177e4 1665
6a8736d1
TH
1666 if (bio_rw(bio) != READA)
1667 queue_io(md, bio);
1668 else
54d9a1b4 1669 bio_io_error(bio);
5a7bbad2 1670 return;
1da177e4
LT
1671 }
1672
83d5e5b0
MP
1673 __split_and_process_bio(md, map, bio);
1674 dm_put_live_table(md, srcu_idx);
5a7bbad2 1675 return;
cec47e3d
KU
1676}
1677
fd2ed4d2 1678int dm_request_based(struct mapped_device *md)
cec47e3d
KU
1679{
1680 return blk_queue_stackable(md->queue);
1681}
1682
5a7bbad2 1683static void dm_request(struct request_queue *q, struct bio *bio)
cec47e3d
KU
1684{
1685 struct mapped_device *md = q->queuedata;
1686
1687 if (dm_request_based(md))
5a7bbad2
CH
1688 blk_queue_bio(q, bio);
1689 else
1690 _dm_request(q, bio);
cec47e3d
KU
1691}
1692
1693void dm_dispatch_request(struct request *rq)
1694{
1695 int r;
1696
1697 if (blk_queue_io_stat(rq->q))
1698 rq->cmd_flags |= REQ_IO_STAT;
1699
1700 rq->start_time = jiffies;
1701 r = blk_insert_cloned_request(rq->q, rq);
1702 if (r)
1703 dm_complete_request(rq, r);
1704}
1705EXPORT_SYMBOL_GPL(dm_dispatch_request);
1706
cec47e3d
KU
1707static int dm_rq_bio_constructor(struct bio *bio, struct bio *bio_orig,
1708 void *data)
1709{
1710 struct dm_rq_target_io *tio = data;
94818742
KO
1711 struct dm_rq_clone_bio_info *info =
1712 container_of(bio, struct dm_rq_clone_bio_info, clone);
cec47e3d
KU
1713
1714 info->orig = bio_orig;
1715 info->tio = tio;
1716 bio->bi_end_io = end_clone_bio;
cec47e3d
KU
1717
1718 return 0;
1719}
1720
1721static int setup_clone(struct request *clone, struct request *rq,
1722 struct dm_rq_target_io *tio)
1723{
d0bcb878 1724 int r;
cec47e3d 1725
29e4013d
TH
1726 r = blk_rq_prep_clone(clone, rq, tio->md->bs, GFP_ATOMIC,
1727 dm_rq_bio_constructor, tio);
1728 if (r)
1729 return r;
cec47e3d 1730
29e4013d
TH
1731 clone->cmd = rq->cmd;
1732 clone->cmd_len = rq->cmd_len;
1733 clone->sense = rq->sense;
cec47e3d
KU
1734 clone->end_io = end_clone_request;
1735 clone->end_io_data = tio;
1736
1737 return 0;
1738}
1739
6facdaff
KU
1740static struct request *clone_rq(struct request *rq, struct mapped_device *md,
1741 gfp_t gfp_mask)
1742{
1743 struct request *clone;
1744 struct dm_rq_target_io *tio;
1745
1746 tio = alloc_rq_tio(md, gfp_mask);
1747 if (!tio)
1748 return NULL;
1749
1750 tio->md = md;
1751 tio->ti = NULL;
1752 tio->orig = rq;
1753 tio->error = 0;
1754 memset(&tio->info, 0, sizeof(tio->info));
1755
1756 clone = &tio->clone;
1757 if (setup_clone(clone, rq, tio)) {
1758 /* -ENOMEM */
1759 free_rq_tio(tio);
1760 return NULL;
1761 }
1762
1763 return clone;
1764}
1765
cec47e3d
KU
1766/*
1767 * Called with the queue lock held.
1768 */
1769static int dm_prep_fn(struct request_queue *q, struct request *rq)
1770{
1771 struct mapped_device *md = q->queuedata;
cec47e3d
KU
1772 struct request *clone;
1773
cec47e3d
KU
1774 if (unlikely(rq->special)) {
1775 DMWARN("Already has something in rq->special.");
1776 return BLKPREP_KILL;
1777 }
1778
6facdaff
KU
1779 clone = clone_rq(rq, md, GFP_ATOMIC);
1780 if (!clone)
cec47e3d 1781 return BLKPREP_DEFER;
cec47e3d
KU
1782
1783 rq->special = clone;
1784 rq->cmd_flags |= REQ_DONTPREP;
1785
1786 return BLKPREP_OK;
1787}
1788
9eef87da
KU
1789/*
1790 * Returns:
1791 * 0 : the request has been processed (not requeued)
1792 * !0 : the request has been requeued
1793 */
1794static int map_request(struct dm_target *ti, struct request *clone,
1795 struct mapped_device *md)
cec47e3d 1796{
9eef87da 1797 int r, requeued = 0;
cec47e3d
KU
1798 struct dm_rq_target_io *tio = clone->end_io_data;
1799
cec47e3d
KU
1800 tio->ti = ti;
1801 r = ti->type->map_rq(ti, clone, &tio->info);
1802 switch (r) {
1803 case DM_MAPIO_SUBMITTED:
1804 /* The target has taken the I/O to submit by itself later */
1805 break;
1806 case DM_MAPIO_REMAPPED:
1807 /* The target has remapped the I/O so dispatch it */
6db4ccd6
JN
1808 trace_block_rq_remap(clone->q, clone, disk_devt(dm_disk(md)),
1809 blk_rq_pos(tio->orig));
cec47e3d
KU
1810 dm_dispatch_request(clone);
1811 break;
1812 case DM_MAPIO_REQUEUE:
1813 /* The target wants to requeue the I/O */
1814 dm_requeue_unmapped_request(clone);
9eef87da 1815 requeued = 1;
cec47e3d
KU
1816 break;
1817 default:
1818 if (r > 0) {
1819 DMWARN("unimplemented target map return value: %d", r);
1820 BUG();
1821 }
1822
1823 /* The target wants to complete the I/O */
1824 dm_kill_unmapped_request(clone, r);
1825 break;
1826 }
9eef87da
KU
1827
1828 return requeued;
cec47e3d
KU
1829}
1830
ba1cbad9
MS
1831static struct request *dm_start_request(struct mapped_device *md, struct request *orig)
1832{
1833 struct request *clone;
1834
1835 blk_start_request(orig);
1836 clone = orig->special;
1837 atomic_inc(&md->pending[rq_data_dir(clone)]);
1838
1839 /*
1840 * Hold the md reference here for the in-flight I/O.
1841 * We can't rely on the reference count by device opener,
1842 * because the device may be closed during the request completion
1843 * when all bios are completed.
1844 * See the comment in rq_completed() too.
1845 */
1846 dm_get(md);
1847
1848 return clone;
1849}
1850
cec47e3d
KU
1851/*
1852 * q->request_fn for request-based dm.
1853 * Called with the queue lock held.
1854 */
1855static void dm_request_fn(struct request_queue *q)
1856{
1857 struct mapped_device *md = q->queuedata;
83d5e5b0
MP
1858 int srcu_idx;
1859 struct dm_table *map = dm_get_live_table(md, &srcu_idx);
cec47e3d 1860 struct dm_target *ti;
b4324fee 1861 struct request *rq, *clone;
29e4013d 1862 sector_t pos;
cec47e3d
KU
1863
1864 /*
b4324fee
KU
1865 * For suspend, check blk_queue_stopped() and increment
1866 * ->pending within a single queue_lock not to increment the
1867 * number of in-flight I/Os after the queue is stopped in
1868 * dm_suspend().
cec47e3d 1869 */
7eaceacc 1870 while (!blk_queue_stopped(q)) {
cec47e3d
KU
1871 rq = blk_peek_request(q);
1872 if (!rq)
7eaceacc 1873 goto delay_and_out;
cec47e3d 1874
29e4013d
TH
1875 /* always use block 0 to find the target for flushes for now */
1876 pos = 0;
1877 if (!(rq->cmd_flags & REQ_FLUSH))
1878 pos = blk_rq_pos(rq);
1879
1880 ti = dm_table_find_target(map, pos);
ba1cbad9
MS
1881 if (!dm_target_is_valid(ti)) {
1882 /*
1883 * Must perform setup, that dm_done() requires,
1884 * before calling dm_kill_unmapped_request
1885 */
1886 DMERR_LIMIT("request attempted access beyond the end of device");
1887 clone = dm_start_request(md, rq);
1888 dm_kill_unmapped_request(clone, -EIO);
1889 continue;
1890 }
d0bcb878 1891
cec47e3d 1892 if (ti->type->busy && ti->type->busy(ti))
7eaceacc 1893 goto delay_and_out;
cec47e3d 1894
ba1cbad9 1895 clone = dm_start_request(md, rq);
b4324fee 1896
cec47e3d 1897 spin_unlock(q->queue_lock);
9eef87da
KU
1898 if (map_request(ti, clone, md))
1899 goto requeued;
1900
052189a2
KU
1901 BUG_ON(!irqs_disabled());
1902 spin_lock(q->queue_lock);
cec47e3d
KU
1903 }
1904
1905 goto out;
1906
9eef87da 1907requeued:
052189a2
KU
1908 BUG_ON(!irqs_disabled());
1909 spin_lock(q->queue_lock);
9eef87da 1910
7eaceacc
JA
1911delay_and_out:
1912 blk_delay_queue(q, HZ / 10);
cec47e3d 1913out:
83d5e5b0 1914 dm_put_live_table(md, srcu_idx);
cec47e3d
KU
1915}
1916
1917int dm_underlying_device_busy(struct request_queue *q)
1918{
1919 return blk_lld_busy(q);
1920}
1921EXPORT_SYMBOL_GPL(dm_underlying_device_busy);
1922
1923static int dm_lld_busy(struct request_queue *q)
1924{
1925 int r;
1926 struct mapped_device *md = q->queuedata;
83d5e5b0 1927 struct dm_table *map = dm_get_live_table_fast(md);
cec47e3d
KU
1928
1929 if (!map || test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))
1930 r = 1;
1931 else
1932 r = dm_table_any_busy_target(map);
1933
83d5e5b0 1934 dm_put_live_table_fast(md);
cec47e3d
KU
1935
1936 return r;
1937}
1938
1da177e4
LT
1939static int dm_any_congested(void *congested_data, int bdi_bits)
1940{
8a57dfc6
CS
1941 int r = bdi_bits;
1942 struct mapped_device *md = congested_data;
1943 struct dm_table *map;
1da177e4 1944
1eb787ec 1945 if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
83d5e5b0 1946 map = dm_get_live_table_fast(md);
8a57dfc6 1947 if (map) {
cec47e3d
KU
1948 /*
1949 * Request-based dm cares about only own queue for
1950 * the query about congestion status of request_queue
1951 */
1952 if (dm_request_based(md))
1953 r = md->queue->backing_dev_info.state &
1954 bdi_bits;
1955 else
1956 r = dm_table_any_congested(map, bdi_bits);
8a57dfc6 1957 }
83d5e5b0 1958 dm_put_live_table_fast(md);
8a57dfc6
CS
1959 }
1960
1da177e4
LT
1961 return r;
1962}
1963
1964/*-----------------------------------------------------------------
1965 * An IDR is used to keep track of allocated minor numbers.
1966 *---------------------------------------------------------------*/
2b06cfff 1967static void free_minor(int minor)
1da177e4 1968{
f32c10b0 1969 spin_lock(&_minor_lock);
1da177e4 1970 idr_remove(&_minor_idr, minor);
f32c10b0 1971 spin_unlock(&_minor_lock);
1da177e4
LT
1972}
1973
1974/*
1975 * See if the device with a specific minor # is free.
1976 */
cf13ab8e 1977static int specific_minor(int minor)
1da177e4 1978{
c9d76be6 1979 int r;
1da177e4
LT
1980
1981 if (minor >= (1 << MINORBITS))
1982 return -EINVAL;
1983
c9d76be6 1984 idr_preload(GFP_KERNEL);
f32c10b0 1985 spin_lock(&_minor_lock);
1da177e4 1986
c9d76be6 1987 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT);
1da177e4 1988
f32c10b0 1989 spin_unlock(&_minor_lock);
c9d76be6
TH
1990 idr_preload_end();
1991 if (r < 0)
1992 return r == -ENOSPC ? -EBUSY : r;
1993 return 0;
1da177e4
LT
1994}
1995
cf13ab8e 1996static int next_free_minor(int *minor)
1da177e4 1997{
c9d76be6 1998 int r;
62f75c2f 1999
c9d76be6 2000 idr_preload(GFP_KERNEL);
f32c10b0 2001 spin_lock(&_minor_lock);
1da177e4 2002
c9d76be6 2003 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT);
1da177e4 2004
f32c10b0 2005 spin_unlock(&_minor_lock);
c9d76be6
TH
2006 idr_preload_end();
2007 if (r < 0)
2008 return r;
2009 *minor = r;
2010 return 0;
1da177e4
LT
2011}
2012
83d5cde4 2013static const struct block_device_operations dm_blk_dops;
1da177e4 2014
53d5914f
MP
2015static void dm_wq_work(struct work_struct *work);
2016
4a0b4ddf
MS
2017static void dm_init_md_queue(struct mapped_device *md)
2018{
2019 /*
2020 * Request-based dm devices cannot be stacked on top of bio-based dm
2021 * devices. The type of this dm device has not been decided yet.
2022 * The type is decided at the first table loading time.
2023 * To prevent problematic device stacking, clear the queue flag
2024 * for request stacking support until then.
2025 *
2026 * This queue is new, so no concurrency on the queue_flags.
2027 */
2028 queue_flag_clear_unlocked(QUEUE_FLAG_STACKABLE, md->queue);
2029
2030 md->queue->queuedata = md;
2031 md->queue->backing_dev_info.congested_fn = dm_any_congested;
2032 md->queue->backing_dev_info.congested_data = md;
2033 blk_queue_make_request(md->queue, dm_request);
2034 blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
4a0b4ddf
MS
2035 blk_queue_merge_bvec(md->queue, dm_merge_bvec);
2036}
2037
1da177e4
LT
2038/*
2039 * Allocate and initialise a blank device with a given minor.
2040 */
2b06cfff 2041static struct mapped_device *alloc_dev(int minor)
1da177e4
LT
2042{
2043 int r;
cf13ab8e 2044 struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL);
ba61fdd1 2045 void *old_md;
1da177e4
LT
2046
2047 if (!md) {
2048 DMWARN("unable to allocate device, out of memory.");
2049 return NULL;
2050 }
2051
10da4f79 2052 if (!try_module_get(THIS_MODULE))
6ed7ade8 2053 goto bad_module_get;
10da4f79 2054
1da177e4 2055 /* get a minor number for the dev */
2b06cfff 2056 if (minor == DM_ANY_MINOR)
cf13ab8e 2057 r = next_free_minor(&minor);
2b06cfff 2058 else
cf13ab8e 2059 r = specific_minor(minor);
1da177e4 2060 if (r < 0)
6ed7ade8 2061 goto bad_minor;
1da177e4 2062
83d5e5b0
MP
2063 r = init_srcu_struct(&md->io_barrier);
2064 if (r < 0)
2065 goto bad_io_barrier;
2066
a5664dad 2067 md->type = DM_TYPE_NONE;
e61290a4 2068 mutex_init(&md->suspend_lock);
a5664dad 2069 mutex_init(&md->type_lock);
86f1152b 2070 mutex_init(&md->table_devices_lock);
022c2611 2071 spin_lock_init(&md->deferred_lock);
1da177e4 2072 atomic_set(&md->holders, 1);
5c6bd75d 2073 atomic_set(&md->open_count, 0);
1da177e4 2074 atomic_set(&md->event_nr, 0);
7a8c3d3b
MA
2075 atomic_set(&md->uevent_seq, 0);
2076 INIT_LIST_HEAD(&md->uevent_list);
86f1152b 2077 INIT_LIST_HEAD(&md->table_devices);
7a8c3d3b 2078 spin_lock_init(&md->uevent_lock);
1da177e4 2079
4a0b4ddf 2080 md->queue = blk_alloc_queue(GFP_KERNEL);
1da177e4 2081 if (!md->queue)
6ed7ade8 2082 goto bad_queue;
1da177e4 2083
4a0b4ddf 2084 dm_init_md_queue(md);
9faf400f 2085
1da177e4
LT
2086 md->disk = alloc_disk(1);
2087 if (!md->disk)
6ed7ade8 2088 goto bad_disk;
1da177e4 2089
316d315b
NK
2090 atomic_set(&md->pending[0], 0);
2091 atomic_set(&md->pending[1], 0);
f0b04115 2092 init_waitqueue_head(&md->wait);
53d5914f 2093 INIT_WORK(&md->work, dm_wq_work);
f0b04115 2094 init_waitqueue_head(&md->eventq);
2995fa78 2095 init_completion(&md->kobj_holder.completion);
f0b04115 2096
1da177e4
LT
2097 md->disk->major = _major;
2098 md->disk->first_minor = minor;
2099 md->disk->fops = &dm_blk_dops;
2100 md->disk->queue = md->queue;
2101 md->disk->private_data = md;
2102 sprintf(md->disk->disk_name, "dm-%d", minor);
2103 add_disk(md->disk);
7e51f257 2104 format_dev_t(md->name, MKDEV(_major, minor));
1da177e4 2105
670368a8 2106 md->wq = alloc_workqueue("kdmflush", WQ_MEM_RECLAIM, 0);
304f3f6a
MB
2107 if (!md->wq)
2108 goto bad_thread;
2109
32a926da
MP
2110 md->bdev = bdget_disk(md->disk, 0);
2111 if (!md->bdev)
2112 goto bad_bdev;
2113
6a8736d1
TH
2114 bio_init(&md->flush_bio);
2115 md->flush_bio.bi_bdev = md->bdev;
2116 md->flush_bio.bi_rw = WRITE_FLUSH;
2117
fd2ed4d2
MP
2118 dm_stats_init(&md->stats);
2119
ba61fdd1 2120 /* Populate the mapping, nobody knows we exist yet */
f32c10b0 2121 spin_lock(&_minor_lock);
ba61fdd1 2122 old_md = idr_replace(&_minor_idr, md, minor);
f32c10b0 2123 spin_unlock(&_minor_lock);
ba61fdd1
JM
2124
2125 BUG_ON(old_md != MINOR_ALLOCED);
2126
1da177e4
LT
2127 return md;
2128
32a926da
MP
2129bad_bdev:
2130 destroy_workqueue(md->wq);
304f3f6a 2131bad_thread:
03022c54 2132 del_gendisk(md->disk);
304f3f6a 2133 put_disk(md->disk);
6ed7ade8 2134bad_disk:
1312f40e 2135 blk_cleanup_queue(md->queue);
6ed7ade8 2136bad_queue:
83d5e5b0
MP
2137 cleanup_srcu_struct(&md->io_barrier);
2138bad_io_barrier:
1da177e4 2139 free_minor(minor);
6ed7ade8 2140bad_minor:
10da4f79 2141 module_put(THIS_MODULE);
6ed7ade8 2142bad_module_get:
1da177e4
LT
2143 kfree(md);
2144 return NULL;
2145}
2146
ae9da83f
JN
2147static void unlock_fs(struct mapped_device *md);
2148
1da177e4
LT
2149static void free_dev(struct mapped_device *md)
2150{
f331c029 2151 int minor = MINOR(disk_devt(md->disk));
63d94e48 2152
32a926da
MP
2153 unlock_fs(md);
2154 bdput(md->bdev);
304f3f6a 2155 destroy_workqueue(md->wq);
e6ee8c0b
KU
2156 if (md->io_pool)
2157 mempool_destroy(md->io_pool);
2158 if (md->bs)
2159 bioset_free(md->bs);
9c47008d 2160 blk_integrity_unregister(md->disk);
1da177e4 2161 del_gendisk(md->disk);
83d5e5b0 2162 cleanup_srcu_struct(&md->io_barrier);
86f1152b 2163 free_table_devices(&md->table_devices);
63d94e48 2164 free_minor(minor);
fba9f90e
JM
2165
2166 spin_lock(&_minor_lock);
2167 md->disk->private_data = NULL;
2168 spin_unlock(&_minor_lock);
2169
1da177e4 2170 put_disk(md->disk);
1312f40e 2171 blk_cleanup_queue(md->queue);
fd2ed4d2 2172 dm_stats_cleanup(&md->stats);
10da4f79 2173 module_put(THIS_MODULE);
1da177e4
LT
2174 kfree(md);
2175}
2176
e6ee8c0b
KU
2177static void __bind_mempools(struct mapped_device *md, struct dm_table *t)
2178{
c0820cf5 2179 struct dm_md_mempools *p = dm_table_get_md_mempools(t);
e6ee8c0b 2180
5f015204 2181 if (md->io_pool && md->bs) {
16245bdc
JN
2182 /* The md already has necessary mempools. */
2183 if (dm_table_get_type(t) == DM_TYPE_BIO_BASED) {
2184 /*
2185 * Reload bioset because front_pad may have changed
2186 * because a different table was loaded.
2187 */
2188 bioset_free(md->bs);
2189 md->bs = p->bs;
2190 p->bs = NULL;
2191 } else if (dm_table_get_type(t) == DM_TYPE_REQUEST_BASED) {
16245bdc
JN
2192 /*
2193 * There's no need to reload with request-based dm
2194 * because the size of front_pad doesn't change.
2195 * Note for future: If you are to reload bioset,
2196 * prep-ed requests in the queue may refer
2197 * to bio from the old bioset, so you must walk
2198 * through the queue to unprep.
2199 */
2200 }
e6ee8c0b 2201 goto out;
c0820cf5 2202 }
e6ee8c0b 2203
5f015204 2204 BUG_ON(!p || md->io_pool || md->bs);
e6ee8c0b
KU
2205
2206 md->io_pool = p->io_pool;
2207 p->io_pool = NULL;
e6ee8c0b
KU
2208 md->bs = p->bs;
2209 p->bs = NULL;
2210
2211out:
2212 /* mempool bind completed, now no need any mempools in the table */
2213 dm_table_free_md_mempools(t);
2214}
2215
1da177e4
LT
2216/*
2217 * Bind a table to the device.
2218 */
2219static void event_callback(void *context)
2220{
7a8c3d3b
MA
2221 unsigned long flags;
2222 LIST_HEAD(uevents);
1da177e4
LT
2223 struct mapped_device *md = (struct mapped_device *) context;
2224
7a8c3d3b
MA
2225 spin_lock_irqsave(&md->uevent_lock, flags);
2226 list_splice_init(&md->uevent_list, &uevents);
2227 spin_unlock_irqrestore(&md->uevent_lock, flags);
2228
ed9e1982 2229 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
7a8c3d3b 2230
1da177e4
LT
2231 atomic_inc(&md->event_nr);
2232 wake_up(&md->eventq);
2233}
2234
c217649b
MS
2235/*
2236 * Protected by md->suspend_lock obtained by dm_swap_table().
2237 */
4e90188b 2238static void __set_size(struct mapped_device *md, sector_t size)
1da177e4 2239{
4e90188b 2240 set_capacity(md->disk, size);
1da177e4 2241
db8fef4f 2242 i_size_write(md->bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
1da177e4
LT
2243}
2244
d5b9dd04
MP
2245/*
2246 * Return 1 if the queue has a compulsory merge_bvec_fn function.
2247 *
2248 * If this function returns 0, then the device is either a non-dm
2249 * device without a merge_bvec_fn, or it is a dm device that is
2250 * able to split any bios it receives that are too big.
2251 */
2252int dm_queue_merge_is_compulsory(struct request_queue *q)
2253{
2254 struct mapped_device *dev_md;
2255
2256 if (!q->merge_bvec_fn)
2257 return 0;
2258
2259 if (q->make_request_fn == dm_request) {
2260 dev_md = q->queuedata;
2261 if (test_bit(DMF_MERGE_IS_OPTIONAL, &dev_md->flags))
2262 return 0;
2263 }
2264
2265 return 1;
2266}
2267
2268static int dm_device_merge_is_compulsory(struct dm_target *ti,
2269 struct dm_dev *dev, sector_t start,
2270 sector_t len, void *data)
2271{
2272 struct block_device *bdev = dev->bdev;
2273 struct request_queue *q = bdev_get_queue(bdev);
2274
2275 return dm_queue_merge_is_compulsory(q);
2276}
2277
2278/*
2279 * Return 1 if it is acceptable to ignore merge_bvec_fn based
2280 * on the properties of the underlying devices.
2281 */
2282static int dm_table_merge_is_optional(struct dm_table *table)
2283{
2284 unsigned i = 0;
2285 struct dm_target *ti;
2286
2287 while (i < dm_table_get_num_targets(table)) {
2288 ti = dm_table_get_target(table, i++);
2289
2290 if (ti->type->iterate_devices &&
2291 ti->type->iterate_devices(ti, dm_device_merge_is_compulsory, NULL))
2292 return 0;
2293 }
2294
2295 return 1;
2296}
2297
042d2a9b
AK
2298/*
2299 * Returns old map, which caller must destroy.
2300 */
2301static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
2302 struct queue_limits *limits)
1da177e4 2303{
042d2a9b 2304 struct dm_table *old_map;
165125e1 2305 struct request_queue *q = md->queue;
1da177e4 2306 sector_t size;
d5b9dd04 2307 int merge_is_optional;
1da177e4
LT
2308
2309 size = dm_table_get_size(t);
3ac51e74
DW
2310
2311 /*
2312 * Wipe any geometry if the size of the table changed.
2313 */
fd2ed4d2 2314 if (size != dm_get_size(md))
3ac51e74
DW
2315 memset(&md->geometry, 0, sizeof(md->geometry));
2316
32a926da 2317 __set_size(md, size);
d5816876 2318
2ca3310e
AK
2319 dm_table_event_callback(t, event_callback, md);
2320
e6ee8c0b
KU
2321 /*
2322 * The queue hasn't been stopped yet, if the old table type wasn't
2323 * for request-based during suspension. So stop it to prevent
2324 * I/O mapping before resume.
2325 * This must be done before setting the queue restrictions,
2326 * because request-based dm may be run just after the setting.
2327 */
2328 if (dm_table_request_based(t) && !blk_queue_stopped(q))
2329 stop_queue(q);
2330
2331 __bind_mempools(md, t);
2332
d5b9dd04
MP
2333 merge_is_optional = dm_table_merge_is_optional(t);
2334
042d2a9b 2335 old_map = md->map;
83d5e5b0 2336 rcu_assign_pointer(md->map, t);
36a0456f
AK
2337 md->immutable_target_type = dm_table_get_immutable_target_type(t);
2338
754c5fc7 2339 dm_table_set_restrictions(t, q, limits);
d5b9dd04
MP
2340 if (merge_is_optional)
2341 set_bit(DMF_MERGE_IS_OPTIONAL, &md->flags);
2342 else
2343 clear_bit(DMF_MERGE_IS_OPTIONAL, &md->flags);
83d5e5b0 2344 dm_sync_table(md);
1da177e4 2345
042d2a9b 2346 return old_map;
1da177e4
LT
2347}
2348
a7940155
AK
2349/*
2350 * Returns unbound table for the caller to free.
2351 */
2352static struct dm_table *__unbind(struct mapped_device *md)
1da177e4
LT
2353{
2354 struct dm_table *map = md->map;
2355
2356 if (!map)
a7940155 2357 return NULL;
1da177e4
LT
2358
2359 dm_table_event_callback(map, NULL, NULL);
9cdb8520 2360 RCU_INIT_POINTER(md->map, NULL);
83d5e5b0 2361 dm_sync_table(md);
a7940155
AK
2362
2363 return map;
1da177e4
LT
2364}
2365
2366/*
2367 * Constructor for a new device.
2368 */
2b06cfff 2369int dm_create(int minor, struct mapped_device **result)
1da177e4
LT
2370{
2371 struct mapped_device *md;
2372
2b06cfff 2373 md = alloc_dev(minor);
1da177e4
LT
2374 if (!md)
2375 return -ENXIO;
2376
784aae73
MB
2377 dm_sysfs_init(md);
2378
1da177e4
LT
2379 *result = md;
2380 return 0;
2381}
2382
a5664dad
MS
2383/*
2384 * Functions to manage md->type.
2385 * All are required to hold md->type_lock.
2386 */
2387void dm_lock_md_type(struct mapped_device *md)
2388{
2389 mutex_lock(&md->type_lock);
2390}
2391
2392void dm_unlock_md_type(struct mapped_device *md)
2393{
2394 mutex_unlock(&md->type_lock);
2395}
2396
2397void dm_set_md_type(struct mapped_device *md, unsigned type)
2398{
00c4fc3b 2399 BUG_ON(!mutex_is_locked(&md->type_lock));
a5664dad
MS
2400 md->type = type;
2401}
2402
2403unsigned dm_get_md_type(struct mapped_device *md)
2404{
00c4fc3b 2405 BUG_ON(!mutex_is_locked(&md->type_lock));
a5664dad
MS
2406 return md->type;
2407}
2408
36a0456f
AK
2409struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
2410{
2411 return md->immutable_target_type;
2412}
2413
f84cb8a4
MS
2414/*
2415 * The queue_limits are only valid as long as you have a reference
2416 * count on 'md'.
2417 */
2418struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
2419{
2420 BUG_ON(!atomic_read(&md->holders));
2421 return &md->queue->limits;
2422}
2423EXPORT_SYMBOL_GPL(dm_get_queue_limits);
2424
4a0b4ddf
MS
2425/*
2426 * Fully initialize a request-based queue (->elevator, ->request_fn, etc).
2427 */
2428static int dm_init_request_based_queue(struct mapped_device *md)
2429{
2430 struct request_queue *q = NULL;
2431
2432 if (md->queue->elevator)
2433 return 1;
2434
2435 /* Fully initialize the queue */
2436 q = blk_init_allocated_queue(md->queue, dm_request_fn, NULL);
2437 if (!q)
2438 return 0;
2439
2440 md->queue = q;
4a0b4ddf
MS
2441 dm_init_md_queue(md);
2442 blk_queue_softirq_done(md->queue, dm_softirq_done);
2443 blk_queue_prep_rq(md->queue, dm_prep_fn);
2444 blk_queue_lld_busy(md->queue, dm_lld_busy);
4a0b4ddf
MS
2445
2446 elv_register_queue(md->queue);
2447
2448 return 1;
2449}
2450
2451/*
2452 * Setup the DM device's queue based on md's type
2453 */
2454int dm_setup_md_queue(struct mapped_device *md)
2455{
2456 if ((dm_get_md_type(md) == DM_TYPE_REQUEST_BASED) &&
2457 !dm_init_request_based_queue(md)) {
2458 DMWARN("Cannot initialize queue for request-based mapped device");
2459 return -EINVAL;
2460 }
2461
2462 return 0;
2463}
2464
637842cf 2465static struct mapped_device *dm_find_md(dev_t dev)
1da177e4
LT
2466{
2467 struct mapped_device *md;
1da177e4
LT
2468 unsigned minor = MINOR(dev);
2469
2470 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
2471 return NULL;
2472
f32c10b0 2473 spin_lock(&_minor_lock);
1da177e4
LT
2474
2475 md = idr_find(&_minor_idr, minor);
fba9f90e 2476 if (md && (md == MINOR_ALLOCED ||
f331c029 2477 (MINOR(disk_devt(dm_disk(md))) != minor) ||
abdc568b 2478 dm_deleting_md(md) ||
17b2f66f 2479 test_bit(DMF_FREEING, &md->flags))) {
637842cf 2480 md = NULL;
fba9f90e
JM
2481 goto out;
2482 }
1da177e4 2483
fba9f90e 2484out:
f32c10b0 2485 spin_unlock(&_minor_lock);
1da177e4 2486
637842cf
DT
2487 return md;
2488}
2489
d229a958
DT
2490struct mapped_device *dm_get_md(dev_t dev)
2491{
2492 struct mapped_device *md = dm_find_md(dev);
2493
2494 if (md)
2495 dm_get(md);
2496
2497 return md;
2498}
3cf2e4ba 2499EXPORT_SYMBOL_GPL(dm_get_md);
d229a958 2500
9ade92a9 2501void *dm_get_mdptr(struct mapped_device *md)
637842cf 2502{
9ade92a9 2503 return md->interface_ptr;
1da177e4
LT
2504}
2505
2506void dm_set_mdptr(struct mapped_device *md, void *ptr)
2507{
2508 md->interface_ptr = ptr;
2509}
2510
2511void dm_get(struct mapped_device *md)
2512{
2513 atomic_inc(&md->holders);
3f77316d 2514 BUG_ON(test_bit(DMF_FREEING, &md->flags));
1da177e4
LT
2515}
2516
72d94861
AK
2517const char *dm_device_name(struct mapped_device *md)
2518{
2519 return md->name;
2520}
2521EXPORT_SYMBOL_GPL(dm_device_name);
2522
3f77316d 2523static void __dm_destroy(struct mapped_device *md, bool wait)
1da177e4 2524{
1134e5ae 2525 struct dm_table *map;
83d5e5b0 2526 int srcu_idx;
1da177e4 2527
3f77316d 2528 might_sleep();
fba9f90e 2529
3f77316d 2530 spin_lock(&_minor_lock);
83d5e5b0 2531 map = dm_get_live_table(md, &srcu_idx);
3f77316d
KU
2532 idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
2533 set_bit(DMF_FREEING, &md->flags);
2534 spin_unlock(&_minor_lock);
2535
2536 if (!dm_suspended_md(md)) {
2537 dm_table_presuspend_targets(map);
2538 dm_table_postsuspend_targets(map);
1da177e4 2539 }
3f77316d 2540
83d5e5b0
MP
2541 /* dm_put_live_table must be before msleep, otherwise deadlock is possible */
2542 dm_put_live_table(md, srcu_idx);
2543
3f77316d
KU
2544 /*
2545 * Rare, but there may be I/O requests still going to complete,
2546 * for example. Wait for all references to disappear.
2547 * No one should increment the reference count of the mapped_device,
2548 * after the mapped_device state becomes DMF_FREEING.
2549 */
2550 if (wait)
2551 while (atomic_read(&md->holders))
2552 msleep(1);
2553 else if (atomic_read(&md->holders))
2554 DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)",
2555 dm_device_name(md), atomic_read(&md->holders));
2556
2557 dm_sysfs_exit(md);
3f77316d
KU
2558 dm_table_destroy(__unbind(md));
2559 free_dev(md);
2560}
2561
2562void dm_destroy(struct mapped_device *md)
2563{
2564 __dm_destroy(md, true);
2565}
2566
2567void dm_destroy_immediate(struct mapped_device *md)
2568{
2569 __dm_destroy(md, false);
2570}
2571
2572void dm_put(struct mapped_device *md)
2573{
2574 atomic_dec(&md->holders);
1da177e4 2575}
79eb885c 2576EXPORT_SYMBOL_GPL(dm_put);
1da177e4 2577
401600df 2578static int dm_wait_for_completion(struct mapped_device *md, int interruptible)
46125c1c
MB
2579{
2580 int r = 0;
b44ebeb0
MP
2581 DECLARE_WAITQUEUE(wait, current);
2582
b44ebeb0 2583 add_wait_queue(&md->wait, &wait);
46125c1c
MB
2584
2585 while (1) {
401600df 2586 set_current_state(interruptible);
46125c1c 2587
b4324fee 2588 if (!md_in_flight(md))
46125c1c
MB
2589 break;
2590
401600df
MP
2591 if (interruptible == TASK_INTERRUPTIBLE &&
2592 signal_pending(current)) {
46125c1c
MB
2593 r = -EINTR;
2594 break;
2595 }
2596
2597 io_schedule();
2598 }
2599 set_current_state(TASK_RUNNING);
2600
b44ebeb0
MP
2601 remove_wait_queue(&md->wait, &wait);
2602
46125c1c
MB
2603 return r;
2604}
2605
1da177e4
LT
2606/*
2607 * Process the deferred bios
2608 */
ef208587 2609static void dm_wq_work(struct work_struct *work)
1da177e4 2610{
ef208587
MP
2611 struct mapped_device *md = container_of(work, struct mapped_device,
2612 work);
6d6f10df 2613 struct bio *c;
83d5e5b0
MP
2614 int srcu_idx;
2615 struct dm_table *map;
1da177e4 2616
83d5e5b0 2617 map = dm_get_live_table(md, &srcu_idx);
ef208587 2618
3b00b203 2619 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
df12ee99
AK
2620 spin_lock_irq(&md->deferred_lock);
2621 c = bio_list_pop(&md->deferred);
2622 spin_unlock_irq(&md->deferred_lock);
2623
6a8736d1 2624 if (!c)
df12ee99 2625 break;
022c2611 2626
e6ee8c0b
KU
2627 if (dm_request_based(md))
2628 generic_make_request(c);
6a8736d1 2629 else
83d5e5b0 2630 __split_and_process_bio(md, map, c);
022c2611 2631 }
73d410c0 2632
83d5e5b0 2633 dm_put_live_table(md, srcu_idx);
1da177e4
LT
2634}
2635
9a1fb464 2636static void dm_queue_flush(struct mapped_device *md)
304f3f6a 2637{
3b00b203 2638 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
4e857c58 2639 smp_mb__after_atomic();
53d5914f 2640 queue_work(md->wq, &md->work);
304f3f6a
MB
2641}
2642
1da177e4 2643/*
042d2a9b 2644 * Swap in a new table, returning the old one for the caller to destroy.
1da177e4 2645 */
042d2a9b 2646struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table)
1da177e4 2647{
87eb5b21 2648 struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL);
754c5fc7 2649 struct queue_limits limits;
042d2a9b 2650 int r;
1da177e4 2651
e61290a4 2652 mutex_lock(&md->suspend_lock);
1da177e4
LT
2653
2654 /* device must be suspended */
4f186f8b 2655 if (!dm_suspended_md(md))
93c534ae 2656 goto out;
1da177e4 2657
3ae70656
MS
2658 /*
2659 * If the new table has no data devices, retain the existing limits.
2660 * This helps multipath with queue_if_no_path if all paths disappear,
2661 * then new I/O is queued based on these limits, and then some paths
2662 * reappear.
2663 */
2664 if (dm_table_has_no_data_devices(table)) {
83d5e5b0 2665 live_map = dm_get_live_table_fast(md);
3ae70656
MS
2666 if (live_map)
2667 limits = md->queue->limits;
83d5e5b0 2668 dm_put_live_table_fast(md);
3ae70656
MS
2669 }
2670
87eb5b21
MC
2671 if (!live_map) {
2672 r = dm_calculate_queue_limits(table, &limits);
2673 if (r) {
2674 map = ERR_PTR(r);
2675 goto out;
2676 }
042d2a9b 2677 }
754c5fc7 2678
042d2a9b 2679 map = __bind(md, table, &limits);
1da177e4 2680
93c534ae 2681out:
e61290a4 2682 mutex_unlock(&md->suspend_lock);
042d2a9b 2683 return map;
1da177e4
LT
2684}
2685
2686/*
2687 * Functions to lock and unlock any filesystem running on the
2688 * device.
2689 */
2ca3310e 2690static int lock_fs(struct mapped_device *md)
1da177e4 2691{
e39e2e95 2692 int r;
1da177e4
LT
2693
2694 WARN_ON(md->frozen_sb);
dfbe03f6 2695
db8fef4f 2696 md->frozen_sb = freeze_bdev(md->bdev);
dfbe03f6 2697 if (IS_ERR(md->frozen_sb)) {
cf222b37 2698 r = PTR_ERR(md->frozen_sb);
e39e2e95
AK
2699 md->frozen_sb = NULL;
2700 return r;
dfbe03f6
AK
2701 }
2702
aa8d7c2f
AK
2703 set_bit(DMF_FROZEN, &md->flags);
2704
1da177e4
LT
2705 return 0;
2706}
2707
2ca3310e 2708static void unlock_fs(struct mapped_device *md)
1da177e4 2709{
aa8d7c2f
AK
2710 if (!test_bit(DMF_FROZEN, &md->flags))
2711 return;
2712
db8fef4f 2713 thaw_bdev(md->bdev, md->frozen_sb);
1da177e4 2714 md->frozen_sb = NULL;
aa8d7c2f 2715 clear_bit(DMF_FROZEN, &md->flags);
1da177e4
LT
2716}
2717
2718/*
2719 * We need to be able to change a mapping table under a mounted
2720 * filesystem. For example we might want to move some data in
2721 * the background. Before the table can be swapped with
2722 * dm_bind_table, dm_suspend must be called to flush any in
2723 * flight bios and ensure that any further io gets deferred.
2724 */
cec47e3d
KU
2725/*
2726 * Suspend mechanism in request-based dm.
2727 *
9f518b27
KU
2728 * 1. Flush all I/Os by lock_fs() if needed.
2729 * 2. Stop dispatching any I/O by stopping the request_queue.
2730 * 3. Wait for all in-flight I/Os to be completed or requeued.
cec47e3d 2731 *
9f518b27 2732 * To abort suspend, start the request_queue.
cec47e3d 2733 */
a3d77d35 2734int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
1da177e4 2735{
2ca3310e 2736 struct dm_table *map = NULL;
46125c1c 2737 int r = 0;
a3d77d35 2738 int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0;
2e93ccc1 2739 int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0;
1da177e4 2740
e61290a4 2741 mutex_lock(&md->suspend_lock);
2ca3310e 2742
4f186f8b 2743 if (dm_suspended_md(md)) {
73d410c0 2744 r = -EINVAL;
d287483d 2745 goto out_unlock;
73d410c0 2746 }
1da177e4 2747
83d5e5b0 2748 map = md->map;
1da177e4 2749
2e93ccc1
KU
2750 /*
2751 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
2752 * This flag is cleared before dm_suspend returns.
2753 */
2754 if (noflush)
2755 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2756
cf222b37
AK
2757 /* This does not get reverted if there's an error later. */
2758 dm_table_presuspend_targets(map);
2759
32a926da 2760 /*
9f518b27
KU
2761 * Flush I/O to the device.
2762 * Any I/O submitted after lock_fs() may not be flushed.
2763 * noflush takes precedence over do_lockfs.
2764 * (lock_fs() flushes I/Os and waits for them to complete.)
32a926da
MP
2765 */
2766 if (!noflush && do_lockfs) {
2767 r = lock_fs(md);
2768 if (r)
83d5e5b0 2769 goto out_unlock;
aa8d7c2f 2770 }
1da177e4
LT
2771
2772 /*
3b00b203
MP
2773 * Here we must make sure that no processes are submitting requests
2774 * to target drivers i.e. no one may be executing
2775 * __split_and_process_bio. This is called from dm_request and
2776 * dm_wq_work.
2777 *
2778 * To get all processes out of __split_and_process_bio in dm_request,
2779 * we take the write lock. To prevent any process from reentering
6a8736d1
TH
2780 * __split_and_process_bio from dm_request and quiesce the thread
2781 * (dm_wq_work), we set BMF_BLOCK_IO_FOR_SUSPEND and call
2782 * flush_workqueue(md->wq).
1da177e4 2783 */
1eb787ec 2784 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
83d5e5b0 2785 synchronize_srcu(&md->io_barrier);
1da177e4 2786
d0bcb878 2787 /*
29e4013d
TH
2788 * Stop md->queue before flushing md->wq in case request-based
2789 * dm defers requests to md->wq from md->queue.
d0bcb878 2790 */
cec47e3d 2791 if (dm_request_based(md))
9f518b27 2792 stop_queue(md->queue);
cec47e3d 2793
d0bcb878
KU
2794 flush_workqueue(md->wq);
2795
1da177e4 2796 /*
3b00b203
MP
2797 * At this point no more requests are entering target request routines.
2798 * We call dm_wait_for_completion to wait for all existing requests
2799 * to finish.
1da177e4 2800 */
401600df 2801 r = dm_wait_for_completion(md, TASK_INTERRUPTIBLE);
1da177e4 2802
6d6f10df 2803 if (noflush)
022c2611 2804 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
83d5e5b0 2805 synchronize_srcu(&md->io_barrier);
2e93ccc1 2806
1da177e4 2807 /* were we interrupted ? */
46125c1c 2808 if (r < 0) {
9a1fb464 2809 dm_queue_flush(md);
73d410c0 2810
cec47e3d 2811 if (dm_request_based(md))
9f518b27 2812 start_queue(md->queue);
cec47e3d 2813
2ca3310e 2814 unlock_fs(md);
83d5e5b0 2815 goto out_unlock; /* pushback list is already flushed, so skip flush */
2ca3310e 2816 }
1da177e4 2817
3b00b203
MP
2818 /*
2819 * If dm_wait_for_completion returned 0, the device is completely
2820 * quiescent now. There is no request-processing activity. All new
2821 * requests are being added to md->deferred list.
2822 */
2823
2ca3310e 2824 set_bit(DMF_SUSPENDED, &md->flags);
b84b0287 2825
4d4471cb
KU
2826 dm_table_postsuspend_targets(map);
2827
d287483d 2828out_unlock:
e61290a4 2829 mutex_unlock(&md->suspend_lock);
cf222b37 2830 return r;
1da177e4
LT
2831}
2832
2833int dm_resume(struct mapped_device *md)
2834{
cf222b37 2835 int r = -EINVAL;
cf222b37 2836 struct dm_table *map = NULL;
1da177e4 2837
e61290a4 2838 mutex_lock(&md->suspend_lock);
4f186f8b 2839 if (!dm_suspended_md(md))
cf222b37 2840 goto out;
cf222b37 2841
83d5e5b0 2842 map = md->map;
2ca3310e 2843 if (!map || !dm_table_get_size(map))
cf222b37 2844 goto out;
1da177e4 2845
8757b776
MB
2846 r = dm_table_resume_targets(map);
2847 if (r)
2848 goto out;
2ca3310e 2849
9a1fb464 2850 dm_queue_flush(md);
2ca3310e 2851
cec47e3d
KU
2852 /*
2853 * Flushing deferred I/Os must be done after targets are resumed
2854 * so that mapping of targets can work correctly.
2855 * Request-based dm is queueing the deferred I/Os in its request_queue.
2856 */
2857 if (dm_request_based(md))
2858 start_queue(md->queue);
2859
2ca3310e
AK
2860 unlock_fs(md);
2861
2862 clear_bit(DMF_SUSPENDED, &md->flags);
2863
cf222b37
AK
2864 r = 0;
2865out:
e61290a4 2866 mutex_unlock(&md->suspend_lock);
2ca3310e 2867
cf222b37 2868 return r;
1da177e4
LT
2869}
2870
fd2ed4d2
MP
2871/*
2872 * Internal suspend/resume works like userspace-driven suspend. It waits
2873 * until all bios finish and prevents issuing new bios to the target drivers.
2874 * It may be used only from the kernel.
2875 *
2876 * Internal suspend holds md->suspend_lock, which prevents interaction with
2877 * userspace-driven suspend.
2878 */
2879
2880void dm_internal_suspend(struct mapped_device *md)
2881{
2882 mutex_lock(&md->suspend_lock);
2883 if (dm_suspended_md(md))
2884 return;
2885
2886 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2887 synchronize_srcu(&md->io_barrier);
2888 flush_workqueue(md->wq);
2889 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
2890}
2891
2892void dm_internal_resume(struct mapped_device *md)
2893{
2894 if (dm_suspended_md(md))
2895 goto done;
2896
2897 dm_queue_flush(md);
2898
2899done:
2900 mutex_unlock(&md->suspend_lock);
2901}
2902
1da177e4
LT
2903/*-----------------------------------------------------------------
2904 * Event notification.
2905 *---------------------------------------------------------------*/
3abf85b5 2906int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
60935eb2 2907 unsigned cookie)
69267a30 2908{
60935eb2
MB
2909 char udev_cookie[DM_COOKIE_LENGTH];
2910 char *envp[] = { udev_cookie, NULL };
2911
2912 if (!cookie)
3abf85b5 2913 return kobject_uevent(&disk_to_dev(md->disk)->kobj, action);
60935eb2
MB
2914 else {
2915 snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u",
2916 DM_COOKIE_ENV_VAR_NAME, cookie);
3abf85b5
PR
2917 return kobject_uevent_env(&disk_to_dev(md->disk)->kobj,
2918 action, envp);
60935eb2 2919 }
69267a30
AK
2920}
2921
7a8c3d3b
MA
2922uint32_t dm_next_uevent_seq(struct mapped_device *md)
2923{
2924 return atomic_add_return(1, &md->uevent_seq);
2925}
2926
1da177e4
LT
2927uint32_t dm_get_event_nr(struct mapped_device *md)
2928{
2929 return atomic_read(&md->event_nr);
2930}
2931
2932int dm_wait_event(struct mapped_device *md, int event_nr)
2933{
2934 return wait_event_interruptible(md->eventq,
2935 (event_nr != atomic_read(&md->event_nr)));
2936}
2937
7a8c3d3b
MA
2938void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
2939{
2940 unsigned long flags;
2941
2942 spin_lock_irqsave(&md->uevent_lock, flags);
2943 list_add(elist, &md->uevent_list);
2944 spin_unlock_irqrestore(&md->uevent_lock, flags);
2945}
2946
1da177e4
LT
2947/*
2948 * The gendisk is only valid as long as you have a reference
2949 * count on 'md'.
2950 */
2951struct gendisk *dm_disk(struct mapped_device *md)
2952{
2953 return md->disk;
2954}
2955
784aae73
MB
2956struct kobject *dm_kobject(struct mapped_device *md)
2957{
2995fa78 2958 return &md->kobj_holder.kobj;
784aae73
MB
2959}
2960
784aae73
MB
2961struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
2962{
2963 struct mapped_device *md;
2964
2995fa78 2965 md = container_of(kobj, struct mapped_device, kobj_holder.kobj);
784aae73 2966
4d89b7b4 2967 if (test_bit(DMF_FREEING, &md->flags) ||
432a212c 2968 dm_deleting_md(md))
4d89b7b4
MB
2969 return NULL;
2970
784aae73
MB
2971 dm_get(md);
2972 return md;
2973}
2974
4f186f8b 2975int dm_suspended_md(struct mapped_device *md)
1da177e4
LT
2976{
2977 return test_bit(DMF_SUSPENDED, &md->flags);
2978}
2979
2c140a24
MP
2980int dm_test_deferred_remove_flag(struct mapped_device *md)
2981{
2982 return test_bit(DMF_DEFERRED_REMOVE, &md->flags);
2983}
2984
64dbce58
KU
2985int dm_suspended(struct dm_target *ti)
2986{
ecdb2e25 2987 return dm_suspended_md(dm_table_get_md(ti->table));
64dbce58
KU
2988}
2989EXPORT_SYMBOL_GPL(dm_suspended);
2990
2e93ccc1
KU
2991int dm_noflush_suspending(struct dm_target *ti)
2992{
ecdb2e25 2993 return __noflush_suspending(dm_table_get_md(ti->table));
2e93ccc1
KU
2994}
2995EXPORT_SYMBOL_GPL(dm_noflush_suspending);
2996
c0820cf5 2997struct dm_md_mempools *dm_alloc_md_mempools(unsigned type, unsigned integrity, unsigned per_bio_data_size)
e6ee8c0b 2998{
5f015204
JN
2999 struct dm_md_mempools *pools = kzalloc(sizeof(*pools), GFP_KERNEL);
3000 struct kmem_cache *cachep;
3001 unsigned int pool_size;
3002 unsigned int front_pad;
e6ee8c0b
KU
3003
3004 if (!pools)
3005 return NULL;
3006
23e5083b 3007 if (type == DM_TYPE_BIO_BASED) {
5f015204 3008 cachep = _io_cache;
e8603136 3009 pool_size = dm_get_reserved_bio_based_ios();
5f015204
JN
3010 front_pad = roundup(per_bio_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
3011 } else if (type == DM_TYPE_REQUEST_BASED) {
3012 cachep = _rq_tio_cache;
f4790826 3013 pool_size = dm_get_reserved_rq_based_ios();
5f015204
JN
3014 front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
3015 /* per_bio_data_size is not used. See __bind_mempools(). */
3016 WARN_ON(per_bio_data_size != 0);
3017 } else
3018 goto out;
e6ee8c0b 3019
6cfa5857 3020 pools->io_pool = mempool_create_slab_pool(pool_size, cachep);
5f015204
JN
3021 if (!pools->io_pool)
3022 goto out;
e6ee8c0b 3023
3d8aab2d 3024 pools->bs = bioset_create_nobvec(pool_size, front_pad);
e6ee8c0b 3025 if (!pools->bs)
5f015204 3026 goto out;
e6ee8c0b 3027
a91a2785 3028 if (integrity && bioset_integrity_create(pools->bs, pool_size))
5f015204 3029 goto out;
a91a2785 3030
e6ee8c0b
KU
3031 return pools;
3032
5f015204
JN
3033out:
3034 dm_free_md_mempools(pools);
e6ee8c0b
KU
3035
3036 return NULL;
3037}
3038
3039void dm_free_md_mempools(struct dm_md_mempools *pools)
3040{
3041 if (!pools)
3042 return;
3043
3044 if (pools->io_pool)
3045 mempool_destroy(pools->io_pool);
3046
e6ee8c0b
KU
3047 if (pools->bs)
3048 bioset_free(pools->bs);
3049
3050 kfree(pools);
3051}
3052
83d5cde4 3053static const struct block_device_operations dm_blk_dops = {
1da177e4
LT
3054 .open = dm_blk_open,
3055 .release = dm_blk_close,
aa129a22 3056 .ioctl = dm_blk_ioctl,
3ac51e74 3057 .getgeo = dm_blk_getgeo,
1da177e4
LT
3058 .owner = THIS_MODULE
3059};
3060
1da177e4
LT
3061/*
3062 * module hooks
3063 */
3064module_init(dm_init);
3065module_exit(dm_exit);
3066
3067module_param(major, uint, 0);
3068MODULE_PARM_DESC(major, "The major number of the device mapper");
f4790826 3069
e8603136
MS
3070module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR);
3071MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools");
3072
f4790826
MS
3073module_param(reserved_rq_based_ios, uint, S_IRUGO | S_IWUSR);
3074MODULE_PARM_DESC(reserved_rq_based_ios, "Reserved IOs in request-based mempools");
3075
1da177e4
LT
3076MODULE_DESCRIPTION(DM_NAME " driver");
3077MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
3078MODULE_LICENSE("GPL");