]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blob - drivers/md/dm-table.c
Merge remote-tracking branches 'asoc/fix/davinci', 'asoc/fix/es8328', 'asoc/fix/fsl...
[mirror_ubuntu-zesty-kernel.git] / drivers / md / dm-table.c
1 /*
2 * Copyright (C) 2001 Sistina Software (UK) Limited.
3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8 #include "dm.h"
9
10 #include <linux/module.h>
11 #include <linux/vmalloc.h>
12 #include <linux/blkdev.h>
13 #include <linux/namei.h>
14 #include <linux/ctype.h>
15 #include <linux/string.h>
16 #include <linux/slab.h>
17 #include <linux/interrupt.h>
18 #include <linux/mutex.h>
19 #include <linux/delay.h>
20 #include <linux/atomic.h>
21 #include <linux/blk-mq.h>
22 #include <linux/mount.h>
23
24 #define DM_MSG_PREFIX "table"
25
26 #define MAX_DEPTH 16
27 #define NODE_SIZE L1_CACHE_BYTES
28 #define KEYS_PER_NODE (NODE_SIZE / sizeof(sector_t))
29 #define CHILDREN_PER_NODE (KEYS_PER_NODE + 1)
30
31 struct dm_table {
32 struct mapped_device *md;
33 unsigned type;
34
35 /* btree table */
36 unsigned int depth;
37 unsigned int counts[MAX_DEPTH]; /* in nodes */
38 sector_t *index[MAX_DEPTH];
39
40 unsigned int num_targets;
41 unsigned int num_allocated;
42 sector_t *highs;
43 struct dm_target *targets;
44
45 struct target_type *immutable_target_type;
46 unsigned integrity_supported:1;
47 unsigned singleton:1;
48
49 /*
50 * Indicates the rw permissions for the new logical
51 * device. This should be a combination of FMODE_READ
52 * and FMODE_WRITE.
53 */
54 fmode_t mode;
55
56 /* a list of devices used by this table */
57 struct list_head devices;
58
59 /* events get handed up using this callback */
60 void (*event_fn)(void *);
61 void *event_context;
62
63 struct dm_md_mempools *mempools;
64
65 struct list_head target_callbacks;
66 };
67
68 /*
69 * Similar to ceiling(log_size(n))
70 */
71 static unsigned int int_log(unsigned int n, unsigned int base)
72 {
73 int result = 0;
74
75 while (n > 1) {
76 n = dm_div_up(n, base);
77 result++;
78 }
79
80 return result;
81 }
82
83 /*
84 * Calculate the index of the child node of the n'th node k'th key.
85 */
86 static inline unsigned int get_child(unsigned int n, unsigned int k)
87 {
88 return (n * CHILDREN_PER_NODE) + k;
89 }
90
91 /*
92 * Return the n'th node of level l from table t.
93 */
94 static inline sector_t *get_node(struct dm_table *t,
95 unsigned int l, unsigned int n)
96 {
97 return t->index[l] + (n * KEYS_PER_NODE);
98 }
99
100 /*
101 * Return the highest key that you could lookup from the n'th
102 * node on level l of the btree.
103 */
104 static sector_t high(struct dm_table *t, unsigned int l, unsigned int n)
105 {
106 for (; l < t->depth - 1; l++)
107 n = get_child(n, CHILDREN_PER_NODE - 1);
108
109 if (n >= t->counts[l])
110 return (sector_t) - 1;
111
112 return get_node(t, l, n)[KEYS_PER_NODE - 1];
113 }
114
115 /*
116 * Fills in a level of the btree based on the highs of the level
117 * below it.
118 */
119 static int setup_btree_index(unsigned int l, struct dm_table *t)
120 {
121 unsigned int n, k;
122 sector_t *node;
123
124 for (n = 0U; n < t->counts[l]; n++) {
125 node = get_node(t, l, n);
126
127 for (k = 0U; k < KEYS_PER_NODE; k++)
128 node[k] = high(t, l + 1, get_child(n, k));
129 }
130
131 return 0;
132 }
133
134 void *dm_vcalloc(unsigned long nmemb, unsigned long elem_size)
135 {
136 unsigned long size;
137 void *addr;
138
139 /*
140 * Check that we're not going to overflow.
141 */
142 if (nmemb > (ULONG_MAX / elem_size))
143 return NULL;
144
145 size = nmemb * elem_size;
146 addr = vzalloc(size);
147
148 return addr;
149 }
150 EXPORT_SYMBOL(dm_vcalloc);
151
152 /*
153 * highs, and targets are managed as dynamic arrays during a
154 * table load.
155 */
156 static int alloc_targets(struct dm_table *t, unsigned int num)
157 {
158 sector_t *n_highs;
159 struct dm_target *n_targets;
160
161 /*
162 * Allocate both the target array and offset array at once.
163 * Append an empty entry to catch sectors beyond the end of
164 * the device.
165 */
166 n_highs = (sector_t *) dm_vcalloc(num + 1, sizeof(struct dm_target) +
167 sizeof(sector_t));
168 if (!n_highs)
169 return -ENOMEM;
170
171 n_targets = (struct dm_target *) (n_highs + num);
172
173 memset(n_highs, -1, sizeof(*n_highs) * num);
174 vfree(t->highs);
175
176 t->num_allocated = num;
177 t->highs = n_highs;
178 t->targets = n_targets;
179
180 return 0;
181 }
182
183 int dm_table_create(struct dm_table **result, fmode_t mode,
184 unsigned num_targets, struct mapped_device *md)
185 {
186 struct dm_table *t = kzalloc(sizeof(*t), GFP_KERNEL);
187
188 if (!t)
189 return -ENOMEM;
190
191 INIT_LIST_HEAD(&t->devices);
192 INIT_LIST_HEAD(&t->target_callbacks);
193
194 if (!num_targets)
195 num_targets = KEYS_PER_NODE;
196
197 num_targets = dm_round_up(num_targets, KEYS_PER_NODE);
198
199 if (!num_targets) {
200 kfree(t);
201 return -ENOMEM;
202 }
203
204 if (alloc_targets(t, num_targets)) {
205 kfree(t);
206 return -ENOMEM;
207 }
208
209 t->mode = mode;
210 t->md = md;
211 *result = t;
212 return 0;
213 }
214
215 static void free_devices(struct list_head *devices, struct mapped_device *md)
216 {
217 struct list_head *tmp, *next;
218
219 list_for_each_safe(tmp, next, devices) {
220 struct dm_dev_internal *dd =
221 list_entry(tmp, struct dm_dev_internal, list);
222 DMWARN("%s: dm_table_destroy: dm_put_device call missing for %s",
223 dm_device_name(md), dd->dm_dev->name);
224 dm_put_table_device(md, dd->dm_dev);
225 kfree(dd);
226 }
227 }
228
229 void dm_table_destroy(struct dm_table *t)
230 {
231 unsigned int i;
232
233 if (!t)
234 return;
235
236 /* free the indexes */
237 if (t->depth >= 2)
238 vfree(t->index[t->depth - 2]);
239
240 /* free the targets */
241 for (i = 0; i < t->num_targets; i++) {
242 struct dm_target *tgt = t->targets + i;
243
244 if (tgt->type->dtr)
245 tgt->type->dtr(tgt);
246
247 dm_put_target_type(tgt->type);
248 }
249
250 vfree(t->highs);
251
252 /* free the device list */
253 free_devices(&t->devices, t->md);
254
255 dm_free_md_mempools(t->mempools);
256
257 kfree(t);
258 }
259
260 /*
261 * See if we've already got a device in the list.
262 */
263 static struct dm_dev_internal *find_device(struct list_head *l, dev_t dev)
264 {
265 struct dm_dev_internal *dd;
266
267 list_for_each_entry (dd, l, list)
268 if (dd->dm_dev->bdev->bd_dev == dev)
269 return dd;
270
271 return NULL;
272 }
273
274 /*
275 * If possible, this checks an area of a destination device is invalid.
276 */
277 static int device_area_is_invalid(struct dm_target *ti, struct dm_dev *dev,
278 sector_t start, sector_t len, void *data)
279 {
280 struct request_queue *q;
281 struct queue_limits *limits = data;
282 struct block_device *bdev = dev->bdev;
283 sector_t dev_size =
284 i_size_read(bdev->bd_inode) >> SECTOR_SHIFT;
285 unsigned short logical_block_size_sectors =
286 limits->logical_block_size >> SECTOR_SHIFT;
287 char b[BDEVNAME_SIZE];
288
289 /*
290 * Some devices exist without request functions,
291 * such as loop devices not yet bound to backing files.
292 * Forbid the use of such devices.
293 */
294 q = bdev_get_queue(bdev);
295 if (!q || !q->make_request_fn) {
296 DMWARN("%s: %s is not yet initialised: "
297 "start=%llu, len=%llu, dev_size=%llu",
298 dm_device_name(ti->table->md), bdevname(bdev, b),
299 (unsigned long long)start,
300 (unsigned long long)len,
301 (unsigned long long)dev_size);
302 return 1;
303 }
304
305 if (!dev_size)
306 return 0;
307
308 if ((start >= dev_size) || (start + len > dev_size)) {
309 DMWARN("%s: %s too small for target: "
310 "start=%llu, len=%llu, dev_size=%llu",
311 dm_device_name(ti->table->md), bdevname(bdev, b),
312 (unsigned long long)start,
313 (unsigned long long)len,
314 (unsigned long long)dev_size);
315 return 1;
316 }
317
318 if (logical_block_size_sectors <= 1)
319 return 0;
320
321 if (start & (logical_block_size_sectors - 1)) {
322 DMWARN("%s: start=%llu not aligned to h/w "
323 "logical block size %u of %s",
324 dm_device_name(ti->table->md),
325 (unsigned long long)start,
326 limits->logical_block_size, bdevname(bdev, b));
327 return 1;
328 }
329
330 if (len & (logical_block_size_sectors - 1)) {
331 DMWARN("%s: len=%llu not aligned to h/w "
332 "logical block size %u of %s",
333 dm_device_name(ti->table->md),
334 (unsigned long long)len,
335 limits->logical_block_size, bdevname(bdev, b));
336 return 1;
337 }
338
339 return 0;
340 }
341
342 /*
343 * This upgrades the mode on an already open dm_dev, being
344 * careful to leave things as they were if we fail to reopen the
345 * device and not to touch the existing bdev field in case
346 * it is accessed concurrently inside dm_table_any_congested().
347 */
348 static int upgrade_mode(struct dm_dev_internal *dd, fmode_t new_mode,
349 struct mapped_device *md)
350 {
351 int r;
352 struct dm_dev *old_dev, *new_dev;
353
354 old_dev = dd->dm_dev;
355
356 r = dm_get_table_device(md, dd->dm_dev->bdev->bd_dev,
357 dd->dm_dev->mode | new_mode, &new_dev);
358 if (r)
359 return r;
360
361 dd->dm_dev = new_dev;
362 dm_put_table_device(md, old_dev);
363
364 return 0;
365 }
366
367 /*
368 * Add a device to the list, or just increment the usage count if
369 * it's already present.
370 */
371 int dm_get_device(struct dm_target *ti, const char *path, fmode_t mode,
372 struct dm_dev **result)
373 {
374 int r;
375 dev_t uninitialized_var(dev);
376 struct dm_dev_internal *dd;
377 struct dm_table *t = ti->table;
378 struct block_device *bdev;
379
380 BUG_ON(!t);
381
382 /* convert the path to a device */
383 bdev = lookup_bdev(path);
384 if (IS_ERR(bdev)) {
385 dev = name_to_dev_t(path);
386 if (!dev)
387 return -ENODEV;
388 } else {
389 dev = bdev->bd_dev;
390 bdput(bdev);
391 }
392
393 dd = find_device(&t->devices, dev);
394 if (!dd) {
395 dd = kmalloc(sizeof(*dd), GFP_KERNEL);
396 if (!dd)
397 return -ENOMEM;
398
399 if ((r = dm_get_table_device(t->md, dev, mode, &dd->dm_dev))) {
400 kfree(dd);
401 return r;
402 }
403
404 atomic_set(&dd->count, 0);
405 list_add(&dd->list, &t->devices);
406
407 } else if (dd->dm_dev->mode != (mode | dd->dm_dev->mode)) {
408 r = upgrade_mode(dd, mode, t->md);
409 if (r)
410 return r;
411 }
412 atomic_inc(&dd->count);
413
414 *result = dd->dm_dev;
415 return 0;
416 }
417 EXPORT_SYMBOL(dm_get_device);
418
419 static int dm_set_device_limits(struct dm_target *ti, struct dm_dev *dev,
420 sector_t start, sector_t len, void *data)
421 {
422 struct queue_limits *limits = data;
423 struct block_device *bdev = dev->bdev;
424 struct request_queue *q = bdev_get_queue(bdev);
425 char b[BDEVNAME_SIZE];
426
427 if (unlikely(!q)) {
428 DMWARN("%s: Cannot set limits for nonexistent device %s",
429 dm_device_name(ti->table->md), bdevname(bdev, b));
430 return 0;
431 }
432
433 if (bdev_stack_limits(limits, bdev, start) < 0)
434 DMWARN("%s: adding target device %s caused an alignment inconsistency: "
435 "physical_block_size=%u, logical_block_size=%u, "
436 "alignment_offset=%u, start=%llu",
437 dm_device_name(ti->table->md), bdevname(bdev, b),
438 q->limits.physical_block_size,
439 q->limits.logical_block_size,
440 q->limits.alignment_offset,
441 (unsigned long long) start << SECTOR_SHIFT);
442
443 return 0;
444 }
445
446 /*
447 * Decrement a device's use count and remove it if necessary.
448 */
449 void dm_put_device(struct dm_target *ti, struct dm_dev *d)
450 {
451 int found = 0;
452 struct list_head *devices = &ti->table->devices;
453 struct dm_dev_internal *dd;
454
455 list_for_each_entry(dd, devices, list) {
456 if (dd->dm_dev == d) {
457 found = 1;
458 break;
459 }
460 }
461 if (!found) {
462 DMWARN("%s: device %s not in table devices list",
463 dm_device_name(ti->table->md), d->name);
464 return;
465 }
466 if (atomic_dec_and_test(&dd->count)) {
467 dm_put_table_device(ti->table->md, d);
468 list_del(&dd->list);
469 kfree(dd);
470 }
471 }
472 EXPORT_SYMBOL(dm_put_device);
473
474 /*
475 * Checks to see if the target joins onto the end of the table.
476 */
477 static int adjoin(struct dm_table *table, struct dm_target *ti)
478 {
479 struct dm_target *prev;
480
481 if (!table->num_targets)
482 return !ti->begin;
483
484 prev = &table->targets[table->num_targets - 1];
485 return (ti->begin == (prev->begin + prev->len));
486 }
487
488 /*
489 * Used to dynamically allocate the arg array.
490 *
491 * We do first allocation with GFP_NOIO because dm-mpath and dm-thin must
492 * process messages even if some device is suspended. These messages have a
493 * small fixed number of arguments.
494 *
495 * On the other hand, dm-switch needs to process bulk data using messages and
496 * excessive use of GFP_NOIO could cause trouble.
497 */
498 static char **realloc_argv(unsigned *array_size, char **old_argv)
499 {
500 char **argv;
501 unsigned new_size;
502 gfp_t gfp;
503
504 if (*array_size) {
505 new_size = *array_size * 2;
506 gfp = GFP_KERNEL;
507 } else {
508 new_size = 8;
509 gfp = GFP_NOIO;
510 }
511 argv = kmalloc(new_size * sizeof(*argv), gfp);
512 if (argv) {
513 memcpy(argv, old_argv, *array_size * sizeof(*argv));
514 *array_size = new_size;
515 }
516
517 kfree(old_argv);
518 return argv;
519 }
520
521 /*
522 * Destructively splits up the argument list to pass to ctr.
523 */
524 int dm_split_args(int *argc, char ***argvp, char *input)
525 {
526 char *start, *end = input, *out, **argv = NULL;
527 unsigned array_size = 0;
528
529 *argc = 0;
530
531 if (!input) {
532 *argvp = NULL;
533 return 0;
534 }
535
536 argv = realloc_argv(&array_size, argv);
537 if (!argv)
538 return -ENOMEM;
539
540 while (1) {
541 /* Skip whitespace */
542 start = skip_spaces(end);
543
544 if (!*start)
545 break; /* success, we hit the end */
546
547 /* 'out' is used to remove any back-quotes */
548 end = out = start;
549 while (*end) {
550 /* Everything apart from '\0' can be quoted */
551 if (*end == '\\' && *(end + 1)) {
552 *out++ = *(end + 1);
553 end += 2;
554 continue;
555 }
556
557 if (isspace(*end))
558 break; /* end of token */
559
560 *out++ = *end++;
561 }
562
563 /* have we already filled the array ? */
564 if ((*argc + 1) > array_size) {
565 argv = realloc_argv(&array_size, argv);
566 if (!argv)
567 return -ENOMEM;
568 }
569
570 /* we know this is whitespace */
571 if (*end)
572 end++;
573
574 /* terminate the string and put it in the array */
575 *out = '\0';
576 argv[*argc] = start;
577 (*argc)++;
578 }
579
580 *argvp = argv;
581 return 0;
582 }
583
584 /*
585 * Impose necessary and sufficient conditions on a devices's table such
586 * that any incoming bio which respects its logical_block_size can be
587 * processed successfully. If it falls across the boundary between
588 * two or more targets, the size of each piece it gets split into must
589 * be compatible with the logical_block_size of the target processing it.
590 */
591 static int validate_hardware_logical_block_alignment(struct dm_table *table,
592 struct queue_limits *limits)
593 {
594 /*
595 * This function uses arithmetic modulo the logical_block_size
596 * (in units of 512-byte sectors).
597 */
598 unsigned short device_logical_block_size_sects =
599 limits->logical_block_size >> SECTOR_SHIFT;
600
601 /*
602 * Offset of the start of the next table entry, mod logical_block_size.
603 */
604 unsigned short next_target_start = 0;
605
606 /*
607 * Given an aligned bio that extends beyond the end of a
608 * target, how many sectors must the next target handle?
609 */
610 unsigned short remaining = 0;
611
612 struct dm_target *uninitialized_var(ti);
613 struct queue_limits ti_limits;
614 unsigned i = 0;
615
616 /*
617 * Check each entry in the table in turn.
618 */
619 while (i < dm_table_get_num_targets(table)) {
620 ti = dm_table_get_target(table, i++);
621
622 blk_set_stacking_limits(&ti_limits);
623
624 /* combine all target devices' limits */
625 if (ti->type->iterate_devices)
626 ti->type->iterate_devices(ti, dm_set_device_limits,
627 &ti_limits);
628
629 /*
630 * If the remaining sectors fall entirely within this
631 * table entry are they compatible with its logical_block_size?
632 */
633 if (remaining < ti->len &&
634 remaining & ((ti_limits.logical_block_size >>
635 SECTOR_SHIFT) - 1))
636 break; /* Error */
637
638 next_target_start =
639 (unsigned short) ((next_target_start + ti->len) &
640 (device_logical_block_size_sects - 1));
641 remaining = next_target_start ?
642 device_logical_block_size_sects - next_target_start : 0;
643 }
644
645 if (remaining) {
646 DMWARN("%s: table line %u (start sect %llu len %llu) "
647 "not aligned to h/w logical block size %u",
648 dm_device_name(table->md), i,
649 (unsigned long long) ti->begin,
650 (unsigned long long) ti->len,
651 limits->logical_block_size);
652 return -EINVAL;
653 }
654
655 return 0;
656 }
657
658 int dm_table_add_target(struct dm_table *t, const char *type,
659 sector_t start, sector_t len, char *params)
660 {
661 int r = -EINVAL, argc;
662 char **argv;
663 struct dm_target *tgt;
664
665 if (t->singleton) {
666 DMERR("%s: target type %s must appear alone in table",
667 dm_device_name(t->md), t->targets->type->name);
668 return -EINVAL;
669 }
670
671 BUG_ON(t->num_targets >= t->num_allocated);
672
673 tgt = t->targets + t->num_targets;
674 memset(tgt, 0, sizeof(*tgt));
675
676 if (!len) {
677 DMERR("%s: zero-length target", dm_device_name(t->md));
678 return -EINVAL;
679 }
680
681 tgt->type = dm_get_target_type(type);
682 if (!tgt->type) {
683 DMERR("%s: %s: unknown target type", dm_device_name(t->md),
684 type);
685 return -EINVAL;
686 }
687
688 if (dm_target_needs_singleton(tgt->type)) {
689 if (t->num_targets) {
690 DMERR("%s: target type %s must appear alone in table",
691 dm_device_name(t->md), type);
692 return -EINVAL;
693 }
694 t->singleton = 1;
695 }
696
697 if (dm_target_always_writeable(tgt->type) && !(t->mode & FMODE_WRITE)) {
698 DMERR("%s: target type %s may not be included in read-only tables",
699 dm_device_name(t->md), type);
700 return -EINVAL;
701 }
702
703 if (t->immutable_target_type) {
704 if (t->immutable_target_type != tgt->type) {
705 DMERR("%s: immutable target type %s cannot be mixed with other target types",
706 dm_device_name(t->md), t->immutable_target_type->name);
707 return -EINVAL;
708 }
709 } else if (dm_target_is_immutable(tgt->type)) {
710 if (t->num_targets) {
711 DMERR("%s: immutable target type %s cannot be mixed with other target types",
712 dm_device_name(t->md), tgt->type->name);
713 return -EINVAL;
714 }
715 t->immutable_target_type = tgt->type;
716 }
717
718 tgt->table = t;
719 tgt->begin = start;
720 tgt->len = len;
721 tgt->error = "Unknown error";
722
723 /*
724 * Does this target adjoin the previous one ?
725 */
726 if (!adjoin(t, tgt)) {
727 tgt->error = "Gap in table";
728 r = -EINVAL;
729 goto bad;
730 }
731
732 r = dm_split_args(&argc, &argv, params);
733 if (r) {
734 tgt->error = "couldn't split parameters (insufficient memory)";
735 goto bad;
736 }
737
738 r = tgt->type->ctr(tgt, argc, argv);
739 kfree(argv);
740 if (r)
741 goto bad;
742
743 t->highs[t->num_targets++] = tgt->begin + tgt->len - 1;
744
745 if (!tgt->num_discard_bios && tgt->discards_supported)
746 DMWARN("%s: %s: ignoring discards_supported because num_discard_bios is zero.",
747 dm_device_name(t->md), type);
748
749 return 0;
750
751 bad:
752 DMERR("%s: %s: %s", dm_device_name(t->md), type, tgt->error);
753 dm_put_target_type(tgt->type);
754 return r;
755 }
756
757 /*
758 * Target argument parsing helpers.
759 */
760 static int validate_next_arg(struct dm_arg *arg, struct dm_arg_set *arg_set,
761 unsigned *value, char **error, unsigned grouped)
762 {
763 const char *arg_str = dm_shift_arg(arg_set);
764 char dummy;
765
766 if (!arg_str ||
767 (sscanf(arg_str, "%u%c", value, &dummy) != 1) ||
768 (*value < arg->min) ||
769 (*value > arg->max) ||
770 (grouped && arg_set->argc < *value)) {
771 *error = arg->error;
772 return -EINVAL;
773 }
774
775 return 0;
776 }
777
778 int dm_read_arg(struct dm_arg *arg, struct dm_arg_set *arg_set,
779 unsigned *value, char **error)
780 {
781 return validate_next_arg(arg, arg_set, value, error, 0);
782 }
783 EXPORT_SYMBOL(dm_read_arg);
784
785 int dm_read_arg_group(struct dm_arg *arg, struct dm_arg_set *arg_set,
786 unsigned *value, char **error)
787 {
788 return validate_next_arg(arg, arg_set, value, error, 1);
789 }
790 EXPORT_SYMBOL(dm_read_arg_group);
791
792 const char *dm_shift_arg(struct dm_arg_set *as)
793 {
794 char *r;
795
796 if (as->argc) {
797 as->argc--;
798 r = *as->argv;
799 as->argv++;
800 return r;
801 }
802
803 return NULL;
804 }
805 EXPORT_SYMBOL(dm_shift_arg);
806
807 void dm_consume_args(struct dm_arg_set *as, unsigned num_args)
808 {
809 BUG_ON(as->argc < num_args);
810 as->argc -= num_args;
811 as->argv += num_args;
812 }
813 EXPORT_SYMBOL(dm_consume_args);
814
815 static bool __table_type_request_based(unsigned table_type)
816 {
817 return (table_type == DM_TYPE_REQUEST_BASED ||
818 table_type == DM_TYPE_MQ_REQUEST_BASED);
819 }
820
821 static int dm_table_set_type(struct dm_table *t)
822 {
823 unsigned i;
824 unsigned bio_based = 0, request_based = 0, hybrid = 0;
825 bool use_blk_mq = false;
826 struct dm_target *tgt;
827 struct dm_dev_internal *dd;
828 struct list_head *devices;
829 unsigned live_md_type = dm_get_md_type(t->md);
830
831 for (i = 0; i < t->num_targets; i++) {
832 tgt = t->targets + i;
833 if (dm_target_hybrid(tgt))
834 hybrid = 1;
835 else if (dm_target_request_based(tgt))
836 request_based = 1;
837 else
838 bio_based = 1;
839
840 if (bio_based && request_based) {
841 DMWARN("Inconsistent table: different target types"
842 " can't be mixed up");
843 return -EINVAL;
844 }
845 }
846
847 if (hybrid && !bio_based && !request_based) {
848 /*
849 * The targets can work either way.
850 * Determine the type from the live device.
851 * Default to bio-based if device is new.
852 */
853 if (__table_type_request_based(live_md_type))
854 request_based = 1;
855 else
856 bio_based = 1;
857 }
858
859 if (bio_based) {
860 /* We must use this table as bio-based */
861 t->type = DM_TYPE_BIO_BASED;
862 return 0;
863 }
864
865 BUG_ON(!request_based); /* No targets in this table */
866
867 /*
868 * Request-based dm supports only tables that have a single target now.
869 * To support multiple targets, request splitting support is needed,
870 * and that needs lots of changes in the block-layer.
871 * (e.g. request completion process for partial completion.)
872 */
873 if (t->num_targets > 1) {
874 DMWARN("Request-based dm doesn't support multiple targets yet");
875 return -EINVAL;
876 }
877
878 /* Non-request-stackable devices can't be used for request-based dm */
879 devices = dm_table_get_devices(t);
880 list_for_each_entry(dd, devices, list) {
881 struct request_queue *q = bdev_get_queue(dd->dm_dev->bdev);
882
883 if (!blk_queue_stackable(q)) {
884 DMERR("table load rejected: including"
885 " non-request-stackable devices");
886 return -EINVAL;
887 }
888
889 if (q->mq_ops)
890 use_blk_mq = true;
891 }
892
893 if (use_blk_mq) {
894 /* verify _all_ devices in the table are blk-mq devices */
895 list_for_each_entry(dd, devices, list)
896 if (!bdev_get_queue(dd->dm_dev->bdev)->mq_ops) {
897 DMERR("table load rejected: not all devices"
898 " are blk-mq request-stackable");
899 return -EINVAL;
900 }
901 t->type = DM_TYPE_MQ_REQUEST_BASED;
902
903 } else if (list_empty(devices) && __table_type_request_based(live_md_type)) {
904 /* inherit live MD type */
905 t->type = live_md_type;
906
907 } else
908 t->type = DM_TYPE_REQUEST_BASED;
909
910 return 0;
911 }
912
913 unsigned dm_table_get_type(struct dm_table *t)
914 {
915 return t->type;
916 }
917
918 struct target_type *dm_table_get_immutable_target_type(struct dm_table *t)
919 {
920 return t->immutable_target_type;
921 }
922
923 bool dm_table_request_based(struct dm_table *t)
924 {
925 return __table_type_request_based(dm_table_get_type(t));
926 }
927
928 bool dm_table_mq_request_based(struct dm_table *t)
929 {
930 return dm_table_get_type(t) == DM_TYPE_MQ_REQUEST_BASED;
931 }
932
933 static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device *md)
934 {
935 unsigned type = dm_table_get_type(t);
936 unsigned per_bio_data_size = 0;
937 struct dm_target *tgt;
938 unsigned i;
939
940 if (unlikely(type == DM_TYPE_NONE)) {
941 DMWARN("no table type is set, can't allocate mempools");
942 return -EINVAL;
943 }
944
945 if (type == DM_TYPE_BIO_BASED)
946 for (i = 0; i < t->num_targets; i++) {
947 tgt = t->targets + i;
948 per_bio_data_size = max(per_bio_data_size, tgt->per_bio_data_size);
949 }
950
951 t->mempools = dm_alloc_md_mempools(md, type, t->integrity_supported, per_bio_data_size);
952 if (!t->mempools)
953 return -ENOMEM;
954
955 return 0;
956 }
957
958 void dm_table_free_md_mempools(struct dm_table *t)
959 {
960 dm_free_md_mempools(t->mempools);
961 t->mempools = NULL;
962 }
963
964 struct dm_md_mempools *dm_table_get_md_mempools(struct dm_table *t)
965 {
966 return t->mempools;
967 }
968
969 static int setup_indexes(struct dm_table *t)
970 {
971 int i;
972 unsigned int total = 0;
973 sector_t *indexes;
974
975 /* allocate the space for *all* the indexes */
976 for (i = t->depth - 2; i >= 0; i--) {
977 t->counts[i] = dm_div_up(t->counts[i + 1], CHILDREN_PER_NODE);
978 total += t->counts[i];
979 }
980
981 indexes = (sector_t *) dm_vcalloc(total, (unsigned long) NODE_SIZE);
982 if (!indexes)
983 return -ENOMEM;
984
985 /* set up internal nodes, bottom-up */
986 for (i = t->depth - 2; i >= 0; i--) {
987 t->index[i] = indexes;
988 indexes += (KEYS_PER_NODE * t->counts[i]);
989 setup_btree_index(i, t);
990 }
991
992 return 0;
993 }
994
995 /*
996 * Builds the btree to index the map.
997 */
998 static int dm_table_build_index(struct dm_table *t)
999 {
1000 int r = 0;
1001 unsigned int leaf_nodes;
1002
1003 /* how many indexes will the btree have ? */
1004 leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE);
1005 t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE);
1006
1007 /* leaf layer has already been set up */
1008 t->counts[t->depth - 1] = leaf_nodes;
1009 t->index[t->depth - 1] = t->highs;
1010
1011 if (t->depth >= 2)
1012 r = setup_indexes(t);
1013
1014 return r;
1015 }
1016
1017 static bool integrity_profile_exists(struct gendisk *disk)
1018 {
1019 return !!blk_get_integrity(disk);
1020 }
1021
1022 /*
1023 * Get a disk whose integrity profile reflects the table's profile.
1024 * Returns NULL if integrity support was inconsistent or unavailable.
1025 */
1026 static struct gendisk * dm_table_get_integrity_disk(struct dm_table *t)
1027 {
1028 struct list_head *devices = dm_table_get_devices(t);
1029 struct dm_dev_internal *dd = NULL;
1030 struct gendisk *prev_disk = NULL, *template_disk = NULL;
1031
1032 list_for_each_entry(dd, devices, list) {
1033 template_disk = dd->dm_dev->bdev->bd_disk;
1034 if (!integrity_profile_exists(template_disk))
1035 goto no_integrity;
1036 else if (prev_disk &&
1037 blk_integrity_compare(prev_disk, template_disk) < 0)
1038 goto no_integrity;
1039 prev_disk = template_disk;
1040 }
1041
1042 return template_disk;
1043
1044 no_integrity:
1045 if (prev_disk)
1046 DMWARN("%s: integrity not set: %s and %s profile mismatch",
1047 dm_device_name(t->md),
1048 prev_disk->disk_name,
1049 template_disk->disk_name);
1050 return NULL;
1051 }
1052
1053 /*
1054 * Register the mapped device for blk_integrity support if the
1055 * underlying devices have an integrity profile. But all devices may
1056 * not have matching profiles (checking all devices isn't reliable
1057 * during table load because this table may use other DM device(s) which
1058 * must be resumed before they will have an initialized integity
1059 * profile). Consequently, stacked DM devices force a 2 stage integrity
1060 * profile validation: First pass during table load, final pass during
1061 * resume.
1062 */
1063 static int dm_table_register_integrity(struct dm_table *t)
1064 {
1065 struct mapped_device *md = t->md;
1066 struct gendisk *template_disk = NULL;
1067
1068 template_disk = dm_table_get_integrity_disk(t);
1069 if (!template_disk)
1070 return 0;
1071
1072 if (!integrity_profile_exists(dm_disk(md))) {
1073 t->integrity_supported = 1;
1074 /*
1075 * Register integrity profile during table load; we can do
1076 * this because the final profile must match during resume.
1077 */
1078 blk_integrity_register(dm_disk(md),
1079 blk_get_integrity(template_disk));
1080 return 0;
1081 }
1082
1083 /*
1084 * If DM device already has an initialized integrity
1085 * profile the new profile should not conflict.
1086 */
1087 if (blk_integrity_compare(dm_disk(md), template_disk) < 0) {
1088 DMWARN("%s: conflict with existing integrity profile: "
1089 "%s profile mismatch",
1090 dm_device_name(t->md),
1091 template_disk->disk_name);
1092 return 1;
1093 }
1094
1095 /* Preserve existing integrity profile */
1096 t->integrity_supported = 1;
1097 return 0;
1098 }
1099
1100 /*
1101 * Prepares the table for use by building the indices,
1102 * setting the type, and allocating mempools.
1103 */
1104 int dm_table_complete(struct dm_table *t)
1105 {
1106 int r;
1107
1108 r = dm_table_set_type(t);
1109 if (r) {
1110 DMERR("unable to set table type");
1111 return r;
1112 }
1113
1114 r = dm_table_build_index(t);
1115 if (r) {
1116 DMERR("unable to build btrees");
1117 return r;
1118 }
1119
1120 r = dm_table_register_integrity(t);
1121 if (r) {
1122 DMERR("could not register integrity profile.");
1123 return r;
1124 }
1125
1126 r = dm_table_alloc_md_mempools(t, t->md);
1127 if (r)
1128 DMERR("unable to allocate mempools");
1129
1130 return r;
1131 }
1132
1133 static DEFINE_MUTEX(_event_lock);
1134 void dm_table_event_callback(struct dm_table *t,
1135 void (*fn)(void *), void *context)
1136 {
1137 mutex_lock(&_event_lock);
1138 t->event_fn = fn;
1139 t->event_context = context;
1140 mutex_unlock(&_event_lock);
1141 }
1142
1143 void dm_table_event(struct dm_table *t)
1144 {
1145 /*
1146 * You can no longer call dm_table_event() from interrupt
1147 * context, use a bottom half instead.
1148 */
1149 BUG_ON(in_interrupt());
1150
1151 mutex_lock(&_event_lock);
1152 if (t->event_fn)
1153 t->event_fn(t->event_context);
1154 mutex_unlock(&_event_lock);
1155 }
1156 EXPORT_SYMBOL(dm_table_event);
1157
1158 sector_t dm_table_get_size(struct dm_table *t)
1159 {
1160 return t->num_targets ? (t->highs[t->num_targets - 1] + 1) : 0;
1161 }
1162 EXPORT_SYMBOL(dm_table_get_size);
1163
1164 struct dm_target *dm_table_get_target(struct dm_table *t, unsigned int index)
1165 {
1166 if (index >= t->num_targets)
1167 return NULL;
1168
1169 return t->targets + index;
1170 }
1171
1172 /*
1173 * Search the btree for the correct target.
1174 *
1175 * Caller should check returned pointer with dm_target_is_valid()
1176 * to trap I/O beyond end of device.
1177 */
1178 struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector)
1179 {
1180 unsigned int l, n = 0, k = 0;
1181 sector_t *node;
1182
1183 for (l = 0; l < t->depth; l++) {
1184 n = get_child(n, k);
1185 node = get_node(t, l, n);
1186
1187 for (k = 0; k < KEYS_PER_NODE; k++)
1188 if (node[k] >= sector)
1189 break;
1190 }
1191
1192 return &t->targets[(KEYS_PER_NODE * n) + k];
1193 }
1194
1195 static int count_device(struct dm_target *ti, struct dm_dev *dev,
1196 sector_t start, sector_t len, void *data)
1197 {
1198 unsigned *num_devices = data;
1199
1200 (*num_devices)++;
1201
1202 return 0;
1203 }
1204
1205 /*
1206 * Check whether a table has no data devices attached using each
1207 * target's iterate_devices method.
1208 * Returns false if the result is unknown because a target doesn't
1209 * support iterate_devices.
1210 */
1211 bool dm_table_has_no_data_devices(struct dm_table *table)
1212 {
1213 struct dm_target *uninitialized_var(ti);
1214 unsigned i = 0, num_devices = 0;
1215
1216 while (i < dm_table_get_num_targets(table)) {
1217 ti = dm_table_get_target(table, i++);
1218
1219 if (!ti->type->iterate_devices)
1220 return false;
1221
1222 ti->type->iterate_devices(ti, count_device, &num_devices);
1223 if (num_devices)
1224 return false;
1225 }
1226
1227 return true;
1228 }
1229
1230 /*
1231 * Establish the new table's queue_limits and validate them.
1232 */
1233 int dm_calculate_queue_limits(struct dm_table *table,
1234 struct queue_limits *limits)
1235 {
1236 struct dm_target *uninitialized_var(ti);
1237 struct queue_limits ti_limits;
1238 unsigned i = 0;
1239
1240 blk_set_stacking_limits(limits);
1241
1242 while (i < dm_table_get_num_targets(table)) {
1243 blk_set_stacking_limits(&ti_limits);
1244
1245 ti = dm_table_get_target(table, i++);
1246
1247 if (!ti->type->iterate_devices)
1248 goto combine_limits;
1249
1250 /*
1251 * Combine queue limits of all the devices this target uses.
1252 */
1253 ti->type->iterate_devices(ti, dm_set_device_limits,
1254 &ti_limits);
1255
1256 /* Set I/O hints portion of queue limits */
1257 if (ti->type->io_hints)
1258 ti->type->io_hints(ti, &ti_limits);
1259
1260 /*
1261 * Check each device area is consistent with the target's
1262 * overall queue limits.
1263 */
1264 if (ti->type->iterate_devices(ti, device_area_is_invalid,
1265 &ti_limits))
1266 return -EINVAL;
1267
1268 combine_limits:
1269 /*
1270 * Merge this target's queue limits into the overall limits
1271 * for the table.
1272 */
1273 if (blk_stack_limits(limits, &ti_limits, 0) < 0)
1274 DMWARN("%s: adding target device "
1275 "(start sect %llu len %llu) "
1276 "caused an alignment inconsistency",
1277 dm_device_name(table->md),
1278 (unsigned long long) ti->begin,
1279 (unsigned long long) ti->len);
1280 }
1281
1282 return validate_hardware_logical_block_alignment(table, limits);
1283 }
1284
1285 /*
1286 * Verify that all devices have an integrity profile that matches the
1287 * DM device's registered integrity profile. If the profiles don't
1288 * match then unregister the DM device's integrity profile.
1289 */
1290 static void dm_table_verify_integrity(struct dm_table *t)
1291 {
1292 struct gendisk *template_disk = NULL;
1293
1294 if (t->integrity_supported) {
1295 /*
1296 * Verify that the original integrity profile
1297 * matches all the devices in this table.
1298 */
1299 template_disk = dm_table_get_integrity_disk(t);
1300 if (template_disk &&
1301 blk_integrity_compare(dm_disk(t->md), template_disk) >= 0)
1302 return;
1303 }
1304
1305 if (integrity_profile_exists(dm_disk(t->md))) {
1306 DMWARN("%s: unable to establish an integrity profile",
1307 dm_device_name(t->md));
1308 blk_integrity_unregister(dm_disk(t->md));
1309 }
1310 }
1311
1312 static int device_flush_capable(struct dm_target *ti, struct dm_dev *dev,
1313 sector_t start, sector_t len, void *data)
1314 {
1315 unsigned flush = (*(unsigned *)data);
1316 struct request_queue *q = bdev_get_queue(dev->bdev);
1317
1318 return q && (q->flush_flags & flush);
1319 }
1320
1321 static bool dm_table_supports_flush(struct dm_table *t, unsigned flush)
1322 {
1323 struct dm_target *ti;
1324 unsigned i = 0;
1325
1326 /*
1327 * Require at least one underlying device to support flushes.
1328 * t->devices includes internal dm devices such as mirror logs
1329 * so we need to use iterate_devices here, which targets
1330 * supporting flushes must provide.
1331 */
1332 while (i < dm_table_get_num_targets(t)) {
1333 ti = dm_table_get_target(t, i++);
1334
1335 if (!ti->num_flush_bios)
1336 continue;
1337
1338 if (ti->flush_supported)
1339 return true;
1340
1341 if (ti->type->iterate_devices &&
1342 ti->type->iterate_devices(ti, device_flush_capable, &flush))
1343 return true;
1344 }
1345
1346 return false;
1347 }
1348
1349 static bool dm_table_discard_zeroes_data(struct dm_table *t)
1350 {
1351 struct dm_target *ti;
1352 unsigned i = 0;
1353
1354 /* Ensure that all targets supports discard_zeroes_data. */
1355 while (i < dm_table_get_num_targets(t)) {
1356 ti = dm_table_get_target(t, i++);
1357
1358 if (ti->discard_zeroes_data_unsupported)
1359 return false;
1360 }
1361
1362 return true;
1363 }
1364
1365 static int device_is_nonrot(struct dm_target *ti, struct dm_dev *dev,
1366 sector_t start, sector_t len, void *data)
1367 {
1368 struct request_queue *q = bdev_get_queue(dev->bdev);
1369
1370 return q && blk_queue_nonrot(q);
1371 }
1372
1373 static int device_is_not_random(struct dm_target *ti, struct dm_dev *dev,
1374 sector_t start, sector_t len, void *data)
1375 {
1376 struct request_queue *q = bdev_get_queue(dev->bdev);
1377
1378 return q && !blk_queue_add_random(q);
1379 }
1380
1381 static int queue_supports_sg_merge(struct dm_target *ti, struct dm_dev *dev,
1382 sector_t start, sector_t len, void *data)
1383 {
1384 struct request_queue *q = bdev_get_queue(dev->bdev);
1385
1386 return q && !test_bit(QUEUE_FLAG_NO_SG_MERGE, &q->queue_flags);
1387 }
1388
1389 static bool dm_table_all_devices_attribute(struct dm_table *t,
1390 iterate_devices_callout_fn func)
1391 {
1392 struct dm_target *ti;
1393 unsigned i = 0;
1394
1395 while (i < dm_table_get_num_targets(t)) {
1396 ti = dm_table_get_target(t, i++);
1397
1398 if (!ti->type->iterate_devices ||
1399 !ti->type->iterate_devices(ti, func, NULL))
1400 return false;
1401 }
1402
1403 return true;
1404 }
1405
1406 static int device_not_write_same_capable(struct dm_target *ti, struct dm_dev *dev,
1407 sector_t start, sector_t len, void *data)
1408 {
1409 struct request_queue *q = bdev_get_queue(dev->bdev);
1410
1411 return q && !q->limits.max_write_same_sectors;
1412 }
1413
1414 static bool dm_table_supports_write_same(struct dm_table *t)
1415 {
1416 struct dm_target *ti;
1417 unsigned i = 0;
1418
1419 while (i < dm_table_get_num_targets(t)) {
1420 ti = dm_table_get_target(t, i++);
1421
1422 if (!ti->num_write_same_bios)
1423 return false;
1424
1425 if (!ti->type->iterate_devices ||
1426 ti->type->iterate_devices(ti, device_not_write_same_capable, NULL))
1427 return false;
1428 }
1429
1430 return true;
1431 }
1432
1433 static int device_discard_capable(struct dm_target *ti, struct dm_dev *dev,
1434 sector_t start, sector_t len, void *data)
1435 {
1436 struct request_queue *q = bdev_get_queue(dev->bdev);
1437
1438 return q && blk_queue_discard(q);
1439 }
1440
1441 static bool dm_table_supports_discards(struct dm_table *t)
1442 {
1443 struct dm_target *ti;
1444 unsigned i = 0;
1445
1446 /*
1447 * Unless any target used by the table set discards_supported,
1448 * require at least one underlying device to support discards.
1449 * t->devices includes internal dm devices such as mirror logs
1450 * so we need to use iterate_devices here, which targets
1451 * supporting discard selectively must provide.
1452 */
1453 while (i < dm_table_get_num_targets(t)) {
1454 ti = dm_table_get_target(t, i++);
1455
1456 if (!ti->num_discard_bios)
1457 continue;
1458
1459 if (ti->discards_supported)
1460 return true;
1461
1462 if (ti->type->iterate_devices &&
1463 ti->type->iterate_devices(ti, device_discard_capable, NULL))
1464 return true;
1465 }
1466
1467 return false;
1468 }
1469
1470 void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q,
1471 struct queue_limits *limits)
1472 {
1473 unsigned flush = 0;
1474
1475 /*
1476 * Copy table's limits to the DM device's request_queue
1477 */
1478 q->limits = *limits;
1479
1480 if (!dm_table_supports_discards(t))
1481 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q);
1482 else
1483 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q);
1484
1485 if (dm_table_supports_flush(t, REQ_FLUSH)) {
1486 flush |= REQ_FLUSH;
1487 if (dm_table_supports_flush(t, REQ_FUA))
1488 flush |= REQ_FUA;
1489 }
1490 blk_queue_flush(q, flush);
1491
1492 if (!dm_table_discard_zeroes_data(t))
1493 q->limits.discard_zeroes_data = 0;
1494
1495 /* Ensure that all underlying devices are non-rotational. */
1496 if (dm_table_all_devices_attribute(t, device_is_nonrot))
1497 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q);
1498 else
1499 queue_flag_clear_unlocked(QUEUE_FLAG_NONROT, q);
1500
1501 if (!dm_table_supports_write_same(t))
1502 q->limits.max_write_same_sectors = 0;
1503
1504 if (dm_table_all_devices_attribute(t, queue_supports_sg_merge))
1505 queue_flag_clear_unlocked(QUEUE_FLAG_NO_SG_MERGE, q);
1506 else
1507 queue_flag_set_unlocked(QUEUE_FLAG_NO_SG_MERGE, q);
1508
1509 dm_table_verify_integrity(t);
1510
1511 /*
1512 * Determine whether or not this queue's I/O timings contribute
1513 * to the entropy pool, Only request-based targets use this.
1514 * Clear QUEUE_FLAG_ADD_RANDOM if any underlying device does not
1515 * have it set.
1516 */
1517 if (blk_queue_add_random(q) && dm_table_all_devices_attribute(t, device_is_not_random))
1518 queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, q);
1519
1520 /*
1521 * QUEUE_FLAG_STACKABLE must be set after all queue settings are
1522 * visible to other CPUs because, once the flag is set, incoming bios
1523 * are processed by request-based dm, which refers to the queue
1524 * settings.
1525 * Until the flag set, bios are passed to bio-based dm and queued to
1526 * md->deferred where queue settings are not needed yet.
1527 * Those bios are passed to request-based dm at the resume time.
1528 */
1529 smp_mb();
1530 if (dm_table_request_based(t))
1531 queue_flag_set_unlocked(QUEUE_FLAG_STACKABLE, q);
1532 }
1533
1534 unsigned int dm_table_get_num_targets(struct dm_table *t)
1535 {
1536 return t->num_targets;
1537 }
1538
1539 struct list_head *dm_table_get_devices(struct dm_table *t)
1540 {
1541 return &t->devices;
1542 }
1543
1544 fmode_t dm_table_get_mode(struct dm_table *t)
1545 {
1546 return t->mode;
1547 }
1548 EXPORT_SYMBOL(dm_table_get_mode);
1549
1550 enum suspend_mode {
1551 PRESUSPEND,
1552 PRESUSPEND_UNDO,
1553 POSTSUSPEND,
1554 };
1555
1556 static void suspend_targets(struct dm_table *t, enum suspend_mode mode)
1557 {
1558 int i = t->num_targets;
1559 struct dm_target *ti = t->targets;
1560
1561 while (i--) {
1562 switch (mode) {
1563 case PRESUSPEND:
1564 if (ti->type->presuspend)
1565 ti->type->presuspend(ti);
1566 break;
1567 case PRESUSPEND_UNDO:
1568 if (ti->type->presuspend_undo)
1569 ti->type->presuspend_undo(ti);
1570 break;
1571 case POSTSUSPEND:
1572 if (ti->type->postsuspend)
1573 ti->type->postsuspend(ti);
1574 break;
1575 }
1576 ti++;
1577 }
1578 }
1579
1580 void dm_table_presuspend_targets(struct dm_table *t)
1581 {
1582 if (!t)
1583 return;
1584
1585 suspend_targets(t, PRESUSPEND);
1586 }
1587
1588 void dm_table_presuspend_undo_targets(struct dm_table *t)
1589 {
1590 if (!t)
1591 return;
1592
1593 suspend_targets(t, PRESUSPEND_UNDO);
1594 }
1595
1596 void dm_table_postsuspend_targets(struct dm_table *t)
1597 {
1598 if (!t)
1599 return;
1600
1601 suspend_targets(t, POSTSUSPEND);
1602 }
1603
1604 int dm_table_resume_targets(struct dm_table *t)
1605 {
1606 int i, r = 0;
1607
1608 for (i = 0; i < t->num_targets; i++) {
1609 struct dm_target *ti = t->targets + i;
1610
1611 if (!ti->type->preresume)
1612 continue;
1613
1614 r = ti->type->preresume(ti);
1615 if (r) {
1616 DMERR("%s: %s: preresume failed, error = %d",
1617 dm_device_name(t->md), ti->type->name, r);
1618 return r;
1619 }
1620 }
1621
1622 for (i = 0; i < t->num_targets; i++) {
1623 struct dm_target *ti = t->targets + i;
1624
1625 if (ti->type->resume)
1626 ti->type->resume(ti);
1627 }
1628
1629 return 0;
1630 }
1631
1632 void dm_table_add_target_callbacks(struct dm_table *t, struct dm_target_callbacks *cb)
1633 {
1634 list_add(&cb->list, &t->target_callbacks);
1635 }
1636 EXPORT_SYMBOL_GPL(dm_table_add_target_callbacks);
1637
1638 int dm_table_any_congested(struct dm_table *t, int bdi_bits)
1639 {
1640 struct dm_dev_internal *dd;
1641 struct list_head *devices = dm_table_get_devices(t);
1642 struct dm_target_callbacks *cb;
1643 int r = 0;
1644
1645 list_for_each_entry(dd, devices, list) {
1646 struct request_queue *q = bdev_get_queue(dd->dm_dev->bdev);
1647 char b[BDEVNAME_SIZE];
1648
1649 if (likely(q))
1650 r |= bdi_congested(&q->backing_dev_info, bdi_bits);
1651 else
1652 DMWARN_LIMIT("%s: any_congested: nonexistent device %s",
1653 dm_device_name(t->md),
1654 bdevname(dd->dm_dev->bdev, b));
1655 }
1656
1657 list_for_each_entry(cb, &t->target_callbacks, list)
1658 if (cb->congested_fn)
1659 r |= cb->congested_fn(cb, bdi_bits);
1660
1661 return r;
1662 }
1663
1664 struct mapped_device *dm_table_get_md(struct dm_table *t)
1665 {
1666 return t->md;
1667 }
1668 EXPORT_SYMBOL(dm_table_get_md);
1669
1670 void dm_table_run_md_queue_async(struct dm_table *t)
1671 {
1672 struct mapped_device *md;
1673 struct request_queue *queue;
1674 unsigned long flags;
1675
1676 if (!dm_table_request_based(t))
1677 return;
1678
1679 md = dm_table_get_md(t);
1680 queue = dm_get_md_queue(md);
1681 if (queue) {
1682 if (queue->mq_ops)
1683 blk_mq_run_hw_queues(queue, true);
1684 else {
1685 spin_lock_irqsave(queue->queue_lock, flags);
1686 blk_run_queue_async(queue);
1687 spin_unlock_irqrestore(queue->queue_lock, flags);
1688 }
1689 }
1690 }
1691 EXPORT_SYMBOL(dm_table_run_md_queue_async);
1692