]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blob - drivers/md/dm-mpath.c
projects / mirror_ubuntu-artful-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'work.sendmsg' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs
[mirror_ubuntu-artful-kernel.git] / drivers / md / dm-mpath.c
1 /*
2 * Copyright (C) 2003 Sistina Software Limited.
3 * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8 #include <linux/device-mapper.h>
9
10 #include "dm-rq.h"
11 #include "dm-bio-record.h"
12 #include "dm-path-selector.h"
13 #include "dm-uevent.h"
14
15 #include <linux/blkdev.h>
16 #include <linux/ctype.h>
17 #include <linux/init.h>
18 #include <linux/mempool.h>
19 #include <linux/module.h>
20 #include <linux/pagemap.h>
21 #include <linux/slab.h>
22 #include <linux/time.h>
23 #include <linux/workqueue.h>
24 #include <linux/delay.h>
25 #include <scsi/scsi_dh.h>
26 #include <linux/atomic.h>
27 #include <linux/blk-mq.h>
28
29 #define DM_MSG_PREFIX "multipath"
30 #define DM_PG_INIT_DELAY_MSECS 2000
31 #define DM_PG_INIT_DELAY_DEFAULT ((unsigned) -1)
32
33 /* Path properties */
34 struct pgpath {
35 struct list_head list;
36
37 struct priority_group *pg; /* Owning PG */
38 unsigned fail_count; /* Cumulative failure count */
39
40 struct dm_path path;
41 struct delayed_work activate_path;
42
43 bool is_active:1; /* Path status */
44 };
45
46 #define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path)
47
48 /*
49 * Paths are grouped into Priority Groups and numbered from 1 upwards.
50 * Each has a path selector which controls which path gets used.
51 */
52 struct priority_group {
53 struct list_head list;
54
55 struct multipath *m; /* Owning multipath instance */
56 struct path_selector ps;
57
58 unsigned pg_num; /* Reference number */
59 unsigned nr_pgpaths; /* Number of paths in PG */
60 struct list_head pgpaths;
61
62 bool bypassed:1; /* Temporarily bypass this PG? */
63 };
64
65 /* Multipath context */
66 struct multipath {
67 struct list_head list;
68 struct dm_target *ti;
69
70 const char *hw_handler_name;
71 char *hw_handler_params;
72
73 spinlock_t lock;
74
75 unsigned nr_priority_groups;
76 struct list_head priority_groups;
77
78 wait_queue_head_t pg_init_wait; /* Wait for pg_init completion */
79
80 struct pgpath *current_pgpath;
81 struct priority_group *current_pg;
82 struct priority_group *next_pg; /* Switch to this PG if set */
83
84 unsigned long flags; /* Multipath state flags */
85
86 unsigned pg_init_retries; /* Number of times to retry pg_init */
87 unsigned pg_init_delay_msecs; /* Number of msecs before pg_init retry */
88
89 atomic_t nr_valid_paths; /* Total number of usable paths */
90 atomic_t pg_init_in_progress; /* Only one pg_init allowed at once */
91 atomic_t pg_init_count; /* Number of times pg_init called */
92
93 unsigned queue_mode;
94
95 struct mutex work_mutex;
96 struct work_struct trigger_event;
97
98 struct work_struct process_queued_bios;
99 struct bio_list queued_bios;
100 };
101
102 /*
103 * Context information attached to each io we process.
104 */
105 struct dm_mpath_io {
106 struct pgpath *pgpath;
107 size_t nr_bytes;
108 };
109
110 typedef int (*action_fn) (struct pgpath *pgpath);
111
112 static struct workqueue_struct *kmultipathd, *kmpath_handlerd;
113 static void trigger_event(struct work_struct *work);
114 static void activate_path(struct work_struct *work);
115 static void process_queued_bios(struct work_struct *work);
116
117 /*-----------------------------------------------
118 * Multipath state flags.
119 *-----------------------------------------------*/
120
121 #define MPATHF_QUEUE_IO 0 /* Must we queue all I/O? */
122 #define MPATHF_QUEUE_IF_NO_PATH 1 /* Queue I/O if last path fails? */
123 #define MPATHF_SAVED_QUEUE_IF_NO_PATH 2 /* Saved state during suspension */
124 #define MPATHF_RETAIN_ATTACHED_HW_HANDLER 3 /* If there's already a hw_handler present, don't change it. */
125 #define MPATHF_PG_INIT_DISABLED 4 /* pg_init is not currently allowed */
126 #define MPATHF_PG_INIT_REQUIRED 5 /* pg_init needs calling? */
127 #define MPATHF_PG_INIT_DELAY_RETRY 6 /* Delay pg_init retry? */
128
129 /*-----------------------------------------------
130 * Allocation routines
131 *-----------------------------------------------*/
132
133 static struct pgpath *alloc_pgpath(void)
134 {
135 struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL);
136
137 if (pgpath) {
138 pgpath->is_active = true;
139 INIT_DELAYED_WORK(&pgpath->activate_path, activate_path);
140 }
141
142 return pgpath;
143 }
144
145 static void free_pgpath(struct pgpath *pgpath)
146 {
147 kfree(pgpath);
148 }
149
150 static struct priority_group *alloc_priority_group(void)
151 {
152 struct priority_group *pg;
153
154 pg = kzalloc(sizeof(*pg), GFP_KERNEL);
155
156 if (pg)
157 INIT_LIST_HEAD(&pg->pgpaths);
158
159 return pg;
160 }
161
162 static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti)
163 {
164 struct pgpath *pgpath, *tmp;
165
166 list_for_each_entry_safe(pgpath, tmp, pgpaths, list) {
167 list_del(&pgpath->list);
168 dm_put_device(ti, pgpath->path.dev);
169 free_pgpath(pgpath);
170 }
171 }
172
173 static void free_priority_group(struct priority_group *pg,
174 struct dm_target *ti)
175 {
176 struct path_selector *ps = &pg->ps;
177
178 if (ps->type) {
179 ps->type->destroy(ps);
180 dm_put_path_selector(ps->type);
181 }
182
183 free_pgpaths(&pg->pgpaths, ti);
184 kfree(pg);
185 }
186
187 static struct multipath *alloc_multipath(struct dm_target *ti)
188 {
189 struct multipath *m;
190
191 m = kzalloc(sizeof(*m), GFP_KERNEL);
192 if (m) {
193 INIT_LIST_HEAD(&m->priority_groups);
194 spin_lock_init(&m->lock);
195 set_bit(MPATHF_QUEUE_IO, &m->flags);
196 atomic_set(&m->nr_valid_paths, 0);
197 atomic_set(&m->pg_init_in_progress, 0);
198 atomic_set(&m->pg_init_count, 0);
199 m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT;
200 INIT_WORK(&m->trigger_event, trigger_event);
201 init_waitqueue_head(&m->pg_init_wait);
202 mutex_init(&m->work_mutex);
203
204 m->queue_mode = DM_TYPE_NONE;
205
206 m->ti = ti;
207 ti->private = m;
208 }
209
210 return m;
211 }
212
213 static int alloc_multipath_stage2(struct dm_target *ti, struct multipath *m)
214 {
215 if (m->queue_mode == DM_TYPE_NONE) {
216 /*
217 * Default to request-based.
218 */
219 if (dm_use_blk_mq(dm_table_get_md(ti->table)))
220 m->queue_mode = DM_TYPE_MQ_REQUEST_BASED;
221 else
222 m->queue_mode = DM_TYPE_REQUEST_BASED;
223 } else if (m->queue_mode == DM_TYPE_BIO_BASED) {
224 INIT_WORK(&m->process_queued_bios, process_queued_bios);
225 /*
226 * bio-based doesn't support any direct scsi_dh management;
227 * it just discovers if a scsi_dh is attached.
228 */
229 set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags);
230 }
231
232 dm_table_set_type(ti->table, m->queue_mode);
233
234 return 0;
235 }
236
237 static void free_multipath(struct multipath *m)
238 {
239 struct priority_group *pg, *tmp;
240
241 list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) {
242 list_del(&pg->list);
243 free_priority_group(pg, m->ti);
244 }
245
246 kfree(m->hw_handler_name);
247 kfree(m->hw_handler_params);
248 kfree(m);
249 }
250
251 static struct dm_mpath_io *get_mpio(union map_info *info)
252 {
253 return info->ptr;
254 }
255
256 static size_t multipath_per_bio_data_size(void)
257 {
258 return sizeof(struct dm_mpath_io) + sizeof(struct dm_bio_details);
259 }
260
261 static struct dm_mpath_io *get_mpio_from_bio(struct bio *bio)
262 {
263 return dm_per_bio_data(bio, multipath_per_bio_data_size());
264 }
265
266 static struct dm_bio_details *get_bio_details_from_bio(struct bio *bio)
267 {
268 /* dm_bio_details is immediately after the dm_mpath_io in bio's per-bio-data */
269 struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
270 void *bio_details = mpio + 1;
271
272 return bio_details;
273 }
274
275 static void multipath_init_per_bio_data(struct bio *bio, struct dm_mpath_io **mpio_p,
276 struct dm_bio_details **bio_details_p)
277 {
278 struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
279 struct dm_bio_details *bio_details = get_bio_details_from_bio(bio);
280
281 memset(mpio, 0, sizeof(*mpio));
282 memset(bio_details, 0, sizeof(*bio_details));
283 dm_bio_record(bio_details, bio);
284
285 if (mpio_p)
286 *mpio_p = mpio;
287 if (bio_details_p)
288 *bio_details_p = bio_details;
289 }
290
291 /*-----------------------------------------------
292 * Path selection
293 *-----------------------------------------------*/
294
295 static int __pg_init_all_paths(struct multipath *m)
296 {
297 struct pgpath *pgpath;
298 unsigned long pg_init_delay = 0;
299
300 if (atomic_read(&m->pg_init_in_progress) || test_bit(MPATHF_PG_INIT_DISABLED, &m->flags))
301 return 0;
302
303 atomic_inc(&m->pg_init_count);
304 clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
305
306 /* Check here to reset pg_init_required */
307 if (!m->current_pg)
308 return 0;
309
310 if (test_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags))
311 pg_init_delay = msecs_to_jiffies(m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT ?
312 m->pg_init_delay_msecs : DM_PG_INIT_DELAY_MSECS);
313 list_for_each_entry(pgpath, &m->current_pg->pgpaths, list) {
314 /* Skip failed paths */
315 if (!pgpath->is_active)
316 continue;
317 if (queue_delayed_work(kmpath_handlerd, &pgpath->activate_path,
318 pg_init_delay))
319 atomic_inc(&m->pg_init_in_progress);
320 }
321 return atomic_read(&m->pg_init_in_progress);
322 }
323
324 static void pg_init_all_paths(struct multipath *m)
325 {
326 unsigned long flags;
327
328 spin_lock_irqsave(&m->lock, flags);
329 __pg_init_all_paths(m);
330 spin_unlock_irqrestore(&m->lock, flags);
331 }
332
333 static void __switch_pg(struct multipath *m, struct priority_group *pg)
334 {
335 m->current_pg = pg;
336
337 /* Must we initialise the PG first, and queue I/O till it's ready? */
338 if (m->hw_handler_name) {
339 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
340 set_bit(MPATHF_QUEUE_IO, &m->flags);
341 } else {
342 clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
343 clear_bit(MPATHF_QUEUE_IO, &m->flags);
344 }
345
346 atomic_set(&m->pg_init_count, 0);
347 }
348
349 static struct pgpath *choose_path_in_pg(struct multipath *m,
350 struct priority_group *pg,
351 size_t nr_bytes)
352 {
353 unsigned long flags;
354 struct dm_path *path;
355 struct pgpath *pgpath;
356
357 path = pg->ps.type->select_path(&pg->ps, nr_bytes);
358 if (!path)
359 return ERR_PTR(-ENXIO);
360
361 pgpath = path_to_pgpath(path);
362
363 if (unlikely(lockless_dereference(m->current_pg) != pg)) {
364 /* Only update current_pgpath if pg changed */
365 spin_lock_irqsave(&m->lock, flags);
366 m->current_pgpath = pgpath;
367 __switch_pg(m, pg);
368 spin_unlock_irqrestore(&m->lock, flags);
369 }
370
371 return pgpath;
372 }
373
374 static struct pgpath *choose_pgpath(struct multipath *m, size_t nr_bytes)
375 {
376 unsigned long flags;
377 struct priority_group *pg;
378 struct pgpath *pgpath;
379 unsigned bypassed = 1;
380
381 if (!atomic_read(&m->nr_valid_paths)) {
382 clear_bit(MPATHF_QUEUE_IO, &m->flags);
383 goto failed;
384 }
385
386 /* Were we instructed to switch PG? */
387 if (lockless_dereference(m->next_pg)) {
388 spin_lock_irqsave(&m->lock, flags);
389 pg = m->next_pg;
390 if (!pg) {
391 spin_unlock_irqrestore(&m->lock, flags);
392 goto check_current_pg;
393 }
394 m->next_pg = NULL;
395 spin_unlock_irqrestore(&m->lock, flags);
396 pgpath = choose_path_in_pg(m, pg, nr_bytes);
397 if (!IS_ERR_OR_NULL(pgpath))
398 return pgpath;
399 }
400
401 /* Don't change PG until it has no remaining paths */
402 check_current_pg:
403 pg = lockless_dereference(m->current_pg);
404 if (pg) {
405 pgpath = choose_path_in_pg(m, pg, nr_bytes);
406 if (!IS_ERR_OR_NULL(pgpath))
407 return pgpath;
408 }
409
410 /*
411 * Loop through priority groups until we find a valid path.
412 * First time we skip PGs marked 'bypassed'.
413 * Second time we only try the ones we skipped, but set
414 * pg_init_delay_retry so we do not hammer controllers.
415 */
416 do {
417 list_for_each_entry(pg, &m->priority_groups, list) {
418 if (pg->bypassed == !!bypassed)
419 continue;
420 pgpath = choose_path_in_pg(m, pg, nr_bytes);
421 if (!IS_ERR_OR_NULL(pgpath)) {
422 if (!bypassed)
423 set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
424 return pgpath;
425 }
426 }
427 } while (bypassed--);
428
429 failed:
430 spin_lock_irqsave(&m->lock, flags);
431 m->current_pgpath = NULL;
432 m->current_pg = NULL;
433 spin_unlock_irqrestore(&m->lock, flags);
434
435 return NULL;
436 }
437
438 /*
439 * Check whether bios must be queued in the device-mapper core rather
440 * than here in the target.
441 *
442 * If m->queue_if_no_path and m->saved_queue_if_no_path hold the
443 * same value then we are not between multipath_presuspend()
444 * and multipath_resume() calls and we have no need to check
445 * for the DMF_NOFLUSH_SUSPENDING flag.
446 */
447 static bool __must_push_back(struct multipath *m)
448 {
449 return ((test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) !=
450 test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags)) &&
451 dm_noflush_suspending(m->ti));
452 }
453
454 static bool must_push_back_rq(struct multipath *m)
455 {
456 bool r;
457 unsigned long flags;
458
459 spin_lock_irqsave(&m->lock, flags);
460 r = (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) ||
461 __must_push_back(m));
462 spin_unlock_irqrestore(&m->lock, flags);
463
464 return r;
465 }
466
467 static bool must_push_back_bio(struct multipath *m)
468 {
469 bool r;
470 unsigned long flags;
471
472 spin_lock_irqsave(&m->lock, flags);
473 r = __must_push_back(m);
474 spin_unlock_irqrestore(&m->lock, flags);
475
476 return r;
477 }
478
479 /*
480 * Map cloned requests (request-based multipath)
481 */
482 static int multipath_clone_and_map(struct dm_target *ti, struct request *rq,
483 union map_info *map_context,
484 struct request **__clone)
485 {
486 struct multipath *m = ti->private;
487 int r = DM_MAPIO_REQUEUE;
488 size_t nr_bytes = blk_rq_bytes(rq);
489 struct pgpath *pgpath;
490 struct block_device *bdev;
491 struct dm_mpath_io *mpio = get_mpio(map_context);
492 struct request *clone;
493
494 /* Do we need to select a new pgpath? */
495 pgpath = lockless_dereference(m->current_pgpath);
496 if (!pgpath || !test_bit(MPATHF_QUEUE_IO, &m->flags))
497 pgpath = choose_pgpath(m, nr_bytes);
498
499 if (!pgpath) {
500 if (must_push_back_rq(m))
501 return DM_MAPIO_DELAY_REQUEUE;
502 return -EIO; /* Failed */
503 } else if (test_bit(MPATHF_QUEUE_IO, &m->flags) ||
504 test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) {
505 pg_init_all_paths(m);
506 return r;
507 }
508
509 memset(mpio, 0, sizeof(*mpio));
510 mpio->pgpath = pgpath;
511 mpio->nr_bytes = nr_bytes;
512
513 bdev = pgpath->path.dev->bdev;
514
515 clone = blk_get_request(bdev_get_queue(bdev),
516 rq->cmd_flags | REQ_NOMERGE,
517 GFP_ATOMIC);
518 if (IS_ERR(clone)) {
519 /* EBUSY, ENODEV or EWOULDBLOCK: requeue */
520 return r;
521 }
522 clone->bio = clone->biotail = NULL;
523 clone->rq_disk = bdev->bd_disk;
524 clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
525 *__clone = clone;
526
527 if (pgpath->pg->ps.type->start_io)
528 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
529 &pgpath->path,
530 nr_bytes);
531 return DM_MAPIO_REMAPPED;
532 }
533
534 static void multipath_release_clone(struct request *clone)
535 {
536 blk_put_request(clone);
537 }
538
539 /*
540 * Map cloned bios (bio-based multipath)
541 */
542 static int __multipath_map_bio(struct multipath *m, struct bio *bio, struct dm_mpath_io *mpio)
543 {
544 size_t nr_bytes = bio->bi_iter.bi_size;
545 struct pgpath *pgpath;
546 unsigned long flags;
547 bool queue_io;
548
549 /* Do we need to select a new pgpath? */
550 pgpath = lockless_dereference(m->current_pgpath);
551 queue_io = test_bit(MPATHF_QUEUE_IO, &m->flags);
552 if (!pgpath || !queue_io)
553 pgpath = choose_pgpath(m, nr_bytes);
554
555 if ((pgpath && queue_io) ||
556 (!pgpath && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))) {
557 /* Queue for the daemon to resubmit */
558 spin_lock_irqsave(&m->lock, flags);
559 bio_list_add(&m->queued_bios, bio);
560 spin_unlock_irqrestore(&m->lock, flags);
561 /* PG_INIT_REQUIRED cannot be set without QUEUE_IO */
562 if (queue_io || test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
563 pg_init_all_paths(m);
564 else if (!queue_io)
565 queue_work(kmultipathd, &m->process_queued_bios);
566 return DM_MAPIO_SUBMITTED;
567 }
568
569 if (!pgpath) {
570 if (!must_push_back_bio(m))
571 return -EIO;
572 return DM_MAPIO_REQUEUE;
573 }
574
575 mpio->pgpath = pgpath;
576 mpio->nr_bytes = nr_bytes;
577
578 bio->bi_error = 0;
579 bio->bi_bdev = pgpath->path.dev->bdev;
580 bio->bi_opf |= REQ_FAILFAST_TRANSPORT;
581
582 if (pgpath->pg->ps.type->start_io)
583 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
584 &pgpath->path,
585 nr_bytes);
586 return DM_MAPIO_REMAPPED;
587 }
588
589 static int multipath_map_bio(struct dm_target *ti, struct bio *bio)
590 {
591 struct multipath *m = ti->private;
592 struct dm_mpath_io *mpio = NULL;
593
594 multipath_init_per_bio_data(bio, &mpio, NULL);
595
596 return __multipath_map_bio(m, bio, mpio);
597 }
598
599 static void process_queued_io_list(struct multipath *m)
600 {
601 if (m->queue_mode == DM_TYPE_MQ_REQUEST_BASED)
602 dm_mq_kick_requeue_list(dm_table_get_md(m->ti->table));
603 else if (m->queue_mode == DM_TYPE_BIO_BASED)
604 queue_work(kmultipathd, &m->process_queued_bios);
605 }
606
607 static void process_queued_bios(struct work_struct *work)
608 {
609 int r;
610 unsigned long flags;
611 struct bio *bio;
612 struct bio_list bios;
613 struct blk_plug plug;
614 struct multipath *m =
615 container_of(work, struct multipath, process_queued_bios);
616
617 bio_list_init(&bios);
618
619 spin_lock_irqsave(&m->lock, flags);
620
621 if (bio_list_empty(&m->queued_bios)) {
622 spin_unlock_irqrestore(&m->lock, flags);
623 return;
624 }
625
626 bio_list_merge(&bios, &m->queued_bios);
627 bio_list_init(&m->queued_bios);
628
629 spin_unlock_irqrestore(&m->lock, flags);
630
631 blk_start_plug(&plug);
632 while ((bio = bio_list_pop(&bios))) {
633 r = __multipath_map_bio(m, bio, get_mpio_from_bio(bio));
634 if (r < 0 || r == DM_MAPIO_REQUEUE) {
635 bio->bi_error = r;
636 bio_endio(bio);
637 } else if (r == DM_MAPIO_REMAPPED)
638 generic_make_request(bio);
639 }
640 blk_finish_plug(&plug);
641 }
642
643 /*
644 * If we run out of usable paths, should we queue I/O or error it?
645 */
646 static int queue_if_no_path(struct multipath *m, bool queue_if_no_path,
647 bool save_old_value)
648 {
649 unsigned long flags;
650
651 spin_lock_irqsave(&m->lock, flags);
652
653 if (save_old_value) {
654 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
655 set_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags);
656 else
657 clear_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags);
658 } else {
659 if (queue_if_no_path)
660 set_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags);
661 else
662 clear_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags);
663 }
664 if (queue_if_no_path)
665 set_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags);
666 else
667 clear_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags);
668
669 spin_unlock_irqrestore(&m->lock, flags);
670
671 if (!queue_if_no_path) {
672 dm_table_run_md_queue_async(m->ti->table);
673 process_queued_io_list(m);
674 }
675
676 return 0;
677 }
678
679 /*
680 * An event is triggered whenever a path is taken out of use.
681 * Includes path failure and PG bypass.
682 */
683 static void trigger_event(struct work_struct *work)
684 {
685 struct multipath *m =
686 container_of(work, struct multipath, trigger_event);
687
688 dm_table_event(m->ti->table);
689 }
690
691 /*-----------------------------------------------------------------
692 * Constructor/argument parsing:
693 * <#multipath feature args> [<arg>]*
694 * <#hw_handler args> [hw_handler [<arg>]*]
695 * <#priority groups>
696 * <initial priority group>
697 * [<selector> <#selector args> [<arg>]*
698 * <#paths> <#per-path selector args>
699 * [<path> [<arg>]* ]+ ]+
700 *---------------------------------------------------------------*/
701 static int parse_path_selector(struct dm_arg_set *as, struct priority_group *pg,
702 struct dm_target *ti)
703 {
704 int r;
705 struct path_selector_type *pst;
706 unsigned ps_argc;
707
708 static struct dm_arg _args[] = {
709 {0, 1024, "invalid number of path selector args"},
710 };
711
712 pst = dm_get_path_selector(dm_shift_arg(as));
713 if (!pst) {
714 ti->error = "unknown path selector type";
715 return -EINVAL;
716 }
717
718 r = dm_read_arg_group(_args, as, &ps_argc, &ti->error);
719 if (r) {
720 dm_put_path_selector(pst);
721 return -EINVAL;
722 }
723
724 r = pst->create(&pg->ps, ps_argc, as->argv);
725 if (r) {
726 dm_put_path_selector(pst);
727 ti->error = "path selector constructor failed";
728 return r;
729 }
730
731 pg->ps.type = pst;
732 dm_consume_args(as, ps_argc);
733
734 return 0;
735 }
736
737 static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps,
738 struct dm_target *ti)
739 {
740 int r;
741 struct pgpath *p;
742 struct multipath *m = ti->private;
743 struct request_queue *q = NULL;
744 const char *attached_handler_name;
745
746 /* we need at least a path arg */
747 if (as->argc < 1) {
748 ti->error = "no device given";
749 return ERR_PTR(-EINVAL);
750 }
751
752 p = alloc_pgpath();
753 if (!p)
754 return ERR_PTR(-ENOMEM);
755
756 r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table),
757 &p->path.dev);
758 if (r) {
759 ti->error = "error getting device";
760 goto bad;
761 }
762
763 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags) || m->hw_handler_name)
764 q = bdev_get_queue(p->path.dev->bdev);
765
766 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags)) {
767 retain:
768 attached_handler_name = scsi_dh_attached_handler_name(q, GFP_KERNEL);
769 if (attached_handler_name) {
770 /*
771 * Clear any hw_handler_params associated with a
772 * handler that isn't already attached.
773 */
774 if (m->hw_handler_name && strcmp(attached_handler_name, m->hw_handler_name)) {
775 kfree(m->hw_handler_params);
776 m->hw_handler_params = NULL;
777 }
778
779 /*
780 * Reset hw_handler_name to match the attached handler
781 *
782 * NB. This modifies the table line to show the actual
783 * handler instead of the original table passed in.
784 */
785 kfree(m->hw_handler_name);
786 m->hw_handler_name = attached_handler_name;
787 }
788 }
789
790 if (m->hw_handler_name) {
791 r = scsi_dh_attach(q, m->hw_handler_name);
792 if (r == -EBUSY) {
793 char b[BDEVNAME_SIZE];
794
795 printk(KERN_INFO "dm-mpath: retaining handler on device %s\n",
796 bdevname(p->path.dev->bdev, b));
797 goto retain;
798 }
799 if (r < 0) {
800 ti->error = "error attaching hardware handler";
801 dm_put_device(ti, p->path.dev);
802 goto bad;
803 }
804
805 if (m->hw_handler_params) {
806 r = scsi_dh_set_params(q, m->hw_handler_params);
807 if (r < 0) {
808 ti->error = "unable to set hardware "
809 "handler parameters";
810 dm_put_device(ti, p->path.dev);
811 goto bad;
812 }
813 }
814 }
815
816 r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error);
817 if (r) {
818 dm_put_device(ti, p->path.dev);
819 goto bad;
820 }
821
822 return p;
823
824 bad:
825 free_pgpath(p);
826 return ERR_PTR(r);
827 }
828
829 static struct priority_group *parse_priority_group(struct dm_arg_set *as,
830 struct multipath *m)
831 {
832 static struct dm_arg _args[] = {
833 {1, 1024, "invalid number of paths"},
834 {0, 1024, "invalid number of selector args"}
835 };
836
837 int r;
838 unsigned i, nr_selector_args, nr_args;
839 struct priority_group *pg;
840 struct dm_target *ti = m->ti;
841
842 if (as->argc < 2) {
843 as->argc = 0;
844 ti->error = "not enough priority group arguments";
845 return ERR_PTR(-EINVAL);
846 }
847
848 pg = alloc_priority_group();
849 if (!pg) {
850 ti->error = "couldn't allocate priority group";
851 return ERR_PTR(-ENOMEM);
852 }
853 pg->m = m;
854
855 r = parse_path_selector(as, pg, ti);
856 if (r)
857 goto bad;
858
859 /*
860 * read the paths
861 */
862 r = dm_read_arg(_args, as, &pg->nr_pgpaths, &ti->error);
863 if (r)
864 goto bad;
865
866 r = dm_read_arg(_args + 1, as, &nr_selector_args, &ti->error);
867 if (r)
868 goto bad;
869
870 nr_args = 1 + nr_selector_args;
871 for (i = 0; i < pg->nr_pgpaths; i++) {
872 struct pgpath *pgpath;
873 struct dm_arg_set path_args;
874
875 if (as->argc < nr_args) {
876 ti->error = "not enough path parameters";
877 r = -EINVAL;
878 goto bad;
879 }
880
881 path_args.argc = nr_args;
882 path_args.argv = as->argv;
883
884 pgpath = parse_path(&path_args, &pg->ps, ti);
885 if (IS_ERR(pgpath)) {
886 r = PTR_ERR(pgpath);
887 goto bad;
888 }
889
890 pgpath->pg = pg;
891 list_add_tail(&pgpath->list, &pg->pgpaths);
892 dm_consume_args(as, nr_args);
893 }
894
895 return pg;
896
897 bad:
898 free_priority_group(pg, ti);
899 return ERR_PTR(r);
900 }
901
902 static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m)
903 {
904 unsigned hw_argc;
905 int ret;
906 struct dm_target *ti = m->ti;
907
908 static struct dm_arg _args[] = {
909 {0, 1024, "invalid number of hardware handler args"},
910 };
911
912 if (dm_read_arg_group(_args, as, &hw_argc, &ti->error))
913 return -EINVAL;
914
915 if (!hw_argc)
916 return 0;
917
918 if (m->queue_mode == DM_TYPE_BIO_BASED) {
919 dm_consume_args(as, hw_argc);
920 DMERR("bio-based multipath doesn't allow hardware handler args");
921 return 0;
922 }
923
924 m->hw_handler_name = kstrdup(dm_shift_arg(as), GFP_KERNEL);
925 if (!m->hw_handler_name)
926 return -EINVAL;
927
928 if (hw_argc > 1) {
929 char *p;
930 int i, j, len = 4;
931
932 for (i = 0; i <= hw_argc - 2; i++)
933 len += strlen(as->argv[i]) + 1;
934 p = m->hw_handler_params = kzalloc(len, GFP_KERNEL);
935 if (!p) {
936 ti->error = "memory allocation failed";
937 ret = -ENOMEM;
938 goto fail;
939 }
940 j = sprintf(p, "%d", hw_argc - 1);
941 for (i = 0, p+=j+1; i <= hw_argc - 2; i++, p+=j+1)
942 j = sprintf(p, "%s", as->argv[i]);
943 }
944 dm_consume_args(as, hw_argc - 1);
945
946 return 0;
947 fail:
948 kfree(m->hw_handler_name);
949 m->hw_handler_name = NULL;
950 return ret;
951 }
952
953 static int parse_features(struct dm_arg_set *as, struct multipath *m)
954 {
955 int r;
956 unsigned argc;
957 struct dm_target *ti = m->ti;
958 const char *arg_name;
959
960 static struct dm_arg _args[] = {
961 {0, 8, "invalid number of feature args"},
962 {1, 50, "pg_init_retries must be between 1 and 50"},
963 {0, 60000, "pg_init_delay_msecs must be between 0 and 60000"},
964 };
965
966 r = dm_read_arg_group(_args, as, &argc, &ti->error);
967 if (r)
968 return -EINVAL;
969
970 if (!argc)
971 return 0;
972
973 do {
974 arg_name = dm_shift_arg(as);
975 argc--;
976
977 if (!strcasecmp(arg_name, "queue_if_no_path")) {
978 r = queue_if_no_path(m, true, false);
979 continue;
980 }
981
982 if (!strcasecmp(arg_name, "retain_attached_hw_handler")) {
983 set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags);
984 continue;
985 }
986
987 if (!strcasecmp(arg_name, "pg_init_retries") &&
988 (argc >= 1)) {
989 r = dm_read_arg(_args + 1, as, &m->pg_init_retries, &ti->error);
990 argc--;
991 continue;
992 }
993
994 if (!strcasecmp(arg_name, "pg_init_delay_msecs") &&
995 (argc >= 1)) {
996 r = dm_read_arg(_args + 2, as, &m->pg_init_delay_msecs, &ti->error);
997 argc--;
998 continue;
999 }
1000
1001 if (!strcasecmp(arg_name, "queue_mode") &&
1002 (argc >= 1)) {
1003 const char *queue_mode_name = dm_shift_arg(as);
1004
1005 if (!strcasecmp(queue_mode_name, "bio"))
1006 m->queue_mode = DM_TYPE_BIO_BASED;
1007 else if (!strcasecmp(queue_mode_name, "rq"))
1008 m->queue_mode = DM_TYPE_REQUEST_BASED;
1009 else if (!strcasecmp(queue_mode_name, "mq"))
1010 m->queue_mode = DM_TYPE_MQ_REQUEST_BASED;
1011 else {
1012 ti->error = "Unknown 'queue_mode' requested";
1013 r = -EINVAL;
1014 }
1015 argc--;
1016 continue;
1017 }
1018
1019 ti->error = "Unrecognised multipath feature request";
1020 r = -EINVAL;
1021 } while (argc && !r);
1022
1023 return r;
1024 }
1025
1026 static int multipath_ctr(struct dm_target *ti, unsigned argc, char **argv)
1027 {
1028 /* target arguments */
1029 static struct dm_arg _args[] = {
1030 {0, 1024, "invalid number of priority groups"},
1031 {0, 1024, "invalid initial priority group number"},
1032 };
1033
1034 int r;
1035 struct multipath *m;
1036 struct dm_arg_set as;
1037 unsigned pg_count = 0;
1038 unsigned next_pg_num;
1039
1040 as.argc = argc;
1041 as.argv = argv;
1042
1043 m = alloc_multipath(ti);
1044 if (!m) {
1045 ti->error = "can't allocate multipath";
1046 return -EINVAL;
1047 }
1048
1049 r = parse_features(&as, m);
1050 if (r)
1051 goto bad;
1052
1053 r = alloc_multipath_stage2(ti, m);
1054 if (r)
1055 goto bad;
1056
1057 r = parse_hw_handler(&as, m);
1058 if (r)
1059 goto bad;
1060
1061 r = dm_read_arg(_args, &as, &m->nr_priority_groups, &ti->error);
1062 if (r)
1063 goto bad;
1064
1065 r = dm_read_arg(_args + 1, &as, &next_pg_num, &ti->error);
1066 if (r)
1067 goto bad;
1068
1069 if ((!m->nr_priority_groups && next_pg_num) ||
1070 (m->nr_priority_groups && !next_pg_num)) {
1071 ti->error = "invalid initial priority group";
1072 r = -EINVAL;
1073 goto bad;
1074 }
1075
1076 /* parse the priority groups */
1077 while (as.argc) {
1078 struct priority_group *pg;
1079 unsigned nr_valid_paths = atomic_read(&m->nr_valid_paths);
1080
1081 pg = parse_priority_group(&as, m);
1082 if (IS_ERR(pg)) {
1083 r = PTR_ERR(pg);
1084 goto bad;
1085 }
1086
1087 nr_valid_paths += pg->nr_pgpaths;
1088 atomic_set(&m->nr_valid_paths, nr_valid_paths);
1089
1090 list_add_tail(&pg->list, &m->priority_groups);
1091 pg_count++;
1092 pg->pg_num = pg_count;
1093 if (!--next_pg_num)
1094 m->next_pg = pg;
1095 }
1096
1097 if (pg_count != m->nr_priority_groups) {
1098 ti->error = "priority group count mismatch";
1099 r = -EINVAL;
1100 goto bad;
1101 }
1102
1103 ti->num_flush_bios = 1;
1104 ti->num_discard_bios = 1;
1105 ti->num_write_same_bios = 1;
1106 if (m->queue_mode == DM_TYPE_BIO_BASED)
1107 ti->per_io_data_size = multipath_per_bio_data_size();
1108 else
1109 ti->per_io_data_size = sizeof(struct dm_mpath_io);
1110
1111 return 0;
1112
1113 bad:
1114 free_multipath(m);
1115 return r;
1116 }
1117
1118 static void multipath_wait_for_pg_init_completion(struct multipath *m)
1119 {
1120 DEFINE_WAIT(wait);
1121
1122 while (1) {
1123 prepare_to_wait(&m->pg_init_wait, &wait, TASK_UNINTERRUPTIBLE);
1124
1125 if (!atomic_read(&m->pg_init_in_progress))
1126 break;
1127
1128 io_schedule();
1129 }
1130 finish_wait(&m->pg_init_wait, &wait);
1131 }
1132
1133 static void flush_multipath_work(struct multipath *m)
1134 {
1135 set_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
1136 smp_mb__after_atomic();
1137
1138 flush_workqueue(kmpath_handlerd);
1139 multipath_wait_for_pg_init_completion(m);
1140 flush_workqueue(kmultipathd);
1141 flush_work(&m->trigger_event);
1142
1143 clear_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
1144 smp_mb__after_atomic();
1145 }
1146
1147 static void multipath_dtr(struct dm_target *ti)
1148 {
1149 struct multipath *m = ti->private;
1150
1151 flush_multipath_work(m);
1152 free_multipath(m);
1153 }
1154
1155 /*
1156 * Take a path out of use.
1157 */
1158 static int fail_path(struct pgpath *pgpath)
1159 {
1160 unsigned long flags;
1161 struct multipath *m = pgpath->pg->m;
1162
1163 spin_lock_irqsave(&m->lock, flags);
1164
1165 if (!pgpath->is_active)
1166 goto out;
1167
1168 DMWARN("Failing path %s.", pgpath->path.dev->name);
1169
1170 pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path);
1171 pgpath->is_active = false;
1172 pgpath->fail_count++;
1173
1174 atomic_dec(&m->nr_valid_paths);
1175
1176 if (pgpath == m->current_pgpath)
1177 m->current_pgpath = NULL;
1178
1179 dm_path_uevent(DM_UEVENT_PATH_FAILED, m->ti,
1180 pgpath->path.dev->name, atomic_read(&m->nr_valid_paths));
1181
1182 schedule_work(&m->trigger_event);
1183
1184 out:
1185 spin_unlock_irqrestore(&m->lock, flags);
1186
1187 return 0;
1188 }
1189
1190 /*
1191 * Reinstate a previously-failed path
1192 */
1193 static int reinstate_path(struct pgpath *pgpath)
1194 {
1195 int r = 0, run_queue = 0;
1196 unsigned long flags;
1197 struct multipath *m = pgpath->pg->m;
1198 unsigned nr_valid_paths;
1199
1200 spin_lock_irqsave(&m->lock, flags);
1201
1202 if (pgpath->is_active)
1203 goto out;
1204
1205 DMWARN("Reinstating path %s.", pgpath->path.dev->name);
1206
1207 r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path);
1208 if (r)
1209 goto out;
1210
1211 pgpath->is_active = true;
1212
1213 nr_valid_paths = atomic_inc_return(&m->nr_valid_paths);
1214 if (nr_valid_paths == 1) {
1215 m->current_pgpath = NULL;
1216 run_queue = 1;
1217 } else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) {
1218 if (queue_work(kmpath_handlerd, &pgpath->activate_path.work))
1219 atomic_inc(&m->pg_init_in_progress);
1220 }
1221
1222 dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti,
1223 pgpath->path.dev->name, nr_valid_paths);
1224
1225 schedule_work(&m->trigger_event);
1226
1227 out:
1228 spin_unlock_irqrestore(&m->lock, flags);
1229 if (run_queue) {
1230 dm_table_run_md_queue_async(m->ti->table);
1231 process_queued_io_list(m);
1232 }
1233
1234 return r;
1235 }
1236
1237 /*
1238 * Fail or reinstate all paths that match the provided struct dm_dev.
1239 */
1240 static int action_dev(struct multipath *m, struct dm_dev *dev,
1241 action_fn action)
1242 {
1243 int r = -EINVAL;
1244 struct pgpath *pgpath;
1245 struct priority_group *pg;
1246
1247 list_for_each_entry(pg, &m->priority_groups, list) {
1248 list_for_each_entry(pgpath, &pg->pgpaths, list) {
1249 if (pgpath->path.dev == dev)
1250 r = action(pgpath);
1251 }
1252 }
1253
1254 return r;
1255 }
1256
1257 /*
1258 * Temporarily try to avoid having to use the specified PG
1259 */
1260 static void bypass_pg(struct multipath *m, struct priority_group *pg,
1261 bool bypassed)
1262 {
1263 unsigned long flags;
1264
1265 spin_lock_irqsave(&m->lock, flags);
1266
1267 pg->bypassed = bypassed;
1268 m->current_pgpath = NULL;
1269 m->current_pg = NULL;
1270
1271 spin_unlock_irqrestore(&m->lock, flags);
1272
1273 schedule_work(&m->trigger_event);
1274 }
1275
1276 /*
1277 * Switch to using the specified PG from the next I/O that gets mapped
1278 */
1279 static int switch_pg_num(struct multipath *m, const char *pgstr)
1280 {
1281 struct priority_group *pg;
1282 unsigned pgnum;
1283 unsigned long flags;
1284 char dummy;
1285
1286 if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
1287 !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) {
1288 DMWARN("invalid PG number supplied to switch_pg_num");
1289 return -EINVAL;
1290 }
1291
1292 spin_lock_irqsave(&m->lock, flags);
1293 list_for_each_entry(pg, &m->priority_groups, list) {
1294 pg->bypassed = false;
1295 if (--pgnum)
1296 continue;
1297
1298 m->current_pgpath = NULL;
1299 m->current_pg = NULL;
1300 m->next_pg = pg;
1301 }
1302 spin_unlock_irqrestore(&m->lock, flags);
1303
1304 schedule_work(&m->trigger_event);
1305 return 0;
1306 }
1307
1308 /*
1309 * Set/clear bypassed status of a PG.
1310 * PGs are numbered upwards from 1 in the order they were declared.
1311 */
1312 static int bypass_pg_num(struct multipath *m, const char *pgstr, bool bypassed)
1313 {
1314 struct priority_group *pg;
1315 unsigned pgnum;
1316 char dummy;
1317
1318 if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
1319 !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) {
1320 DMWARN("invalid PG number supplied to bypass_pg");
1321 return -EINVAL;
1322 }
1323
1324 list_for_each_entry(pg, &m->priority_groups, list) {
1325 if (!--pgnum)
1326 break;
1327 }
1328
1329 bypass_pg(m, pg, bypassed);
1330 return 0;
1331 }
1332
1333 /*
1334 * Should we retry pg_init immediately?
1335 */
1336 static bool pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath)
1337 {
1338 unsigned long flags;
1339 bool limit_reached = false;
1340
1341 spin_lock_irqsave(&m->lock, flags);
1342
1343 if (atomic_read(&m->pg_init_count) <= m->pg_init_retries &&
1344 !test_bit(MPATHF_PG_INIT_DISABLED, &m->flags))
1345 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
1346 else
1347 limit_reached = true;
1348
1349 spin_unlock_irqrestore(&m->lock, flags);
1350
1351 return limit_reached;
1352 }
1353
1354 static void pg_init_done(void *data, int errors)
1355 {
1356 struct pgpath *pgpath = data;
1357 struct priority_group *pg = pgpath->pg;
1358 struct multipath *m = pg->m;
1359 unsigned long flags;
1360 bool delay_retry = false;
1361
1362 /* device or driver problems */
1363 switch (errors) {
1364 case SCSI_DH_OK:
1365 break;
1366 case SCSI_DH_NOSYS:
1367 if (!m->hw_handler_name) {
1368 errors = 0;
1369 break;
1370 }
1371 DMERR("Could not failover the device: Handler scsi_dh_%s "
1372 "Error %d.", m->hw_handler_name, errors);
1373 /*
1374 * Fail path for now, so we do not ping pong
1375 */
1376 fail_path(pgpath);
1377 break;
1378 case SCSI_DH_DEV_TEMP_BUSY:
1379 /*
1380 * Probably doing something like FW upgrade on the
1381 * controller so try the other pg.
1382 */
1383 bypass_pg(m, pg, true);
1384 break;
1385 case SCSI_DH_RETRY:
1386 /* Wait before retrying. */
1387 delay_retry = 1;
1388 case SCSI_DH_IMM_RETRY:
1389 case SCSI_DH_RES_TEMP_UNAVAIL:
1390 if (pg_init_limit_reached(m, pgpath))
1391 fail_path(pgpath);
1392 errors = 0;
1393 break;
1394 case SCSI_DH_DEV_OFFLINED:
1395 default:
1396 /*
1397 * We probably do not want to fail the path for a device
1398 * error, but this is what the old dm did. In future
1399 * patches we can do more advanced handling.
1400 */
1401 fail_path(pgpath);
1402 }
1403
1404 spin_lock_irqsave(&m->lock, flags);
1405 if (errors) {
1406 if (pgpath == m->current_pgpath) {
1407 DMERR("Could not failover device. Error %d.", errors);
1408 m->current_pgpath = NULL;
1409 m->current_pg = NULL;
1410 }
1411 } else if (!test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
1412 pg->bypassed = false;
1413
1414 if (atomic_dec_return(&m->pg_init_in_progress) > 0)
1415 /* Activations of other paths are still on going */
1416 goto out;
1417
1418 if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) {
1419 if (delay_retry)
1420 set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
1421 else
1422 clear_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
1423
1424 if (__pg_init_all_paths(m))
1425 goto out;
1426 }
1427 clear_bit(MPATHF_QUEUE_IO, &m->flags);
1428
1429 process_queued_io_list(m);
1430
1431 /*
1432 * Wake up any thread waiting to suspend.
1433 */
1434 wake_up(&m->pg_init_wait);
1435
1436 out:
1437 spin_unlock_irqrestore(&m->lock, flags);
1438 }
1439
1440 static void activate_path(struct work_struct *work)
1441 {
1442 struct pgpath *pgpath =
1443 container_of(work, struct pgpath, activate_path.work);
1444 struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
1445
1446 if (pgpath->is_active && !blk_queue_dying(q))
1447 scsi_dh_activate(q, pg_init_done, pgpath);
1448 else
1449 pg_init_done(pgpath, SCSI_DH_DEV_OFFLINED);
1450 }
1451
1452 static int noretry_error(int error)
1453 {
1454 switch (error) {
1455 case -EBADE:
1456 /*
1457 * EBADE signals an reservation conflict.
1458 * We shouldn't fail the path here as we can communicate with
1459 * the target. We should failover to the next path, but in
1460 * doing so we might be causing a ping-pong between paths.
1461 * So just return the reservation conflict error.
1462 */
1463 case -EOPNOTSUPP:
1464 case -EREMOTEIO:
1465 case -EILSEQ:
1466 case -ENODATA:
1467 case -ENOSPC:
1468 return 1;
1469 }
1470
1471 /* Anything else could be a path failure, so should be retried */
1472 return 0;
1473 }
1474
1475 /*
1476 * end_io handling
1477 */
1478 static int do_end_io(struct multipath *m, struct request *clone,
1479 int error, struct dm_mpath_io *mpio)
1480 {
1481 /*
1482 * We don't queue any clone request inside the multipath target
1483 * during end I/O handling, since those clone requests don't have
1484 * bio clones. If we queue them inside the multipath target,
1485 * we need to make bio clones, that requires memory allocation.
1486 * (See drivers/md/dm-rq.c:end_clone_bio() about why the clone requests
1487 * don't have bio clones.)
1488 * Instead of queueing the clone request here, we queue the original
1489 * request into dm core, which will remake a clone request and
1490 * clone bios for it and resubmit it later.
1491 */
1492 int r = DM_ENDIO_REQUEUE;
1493
1494 if (!error && !clone->errors)
1495 return 0; /* I/O complete */
1496
1497 if (noretry_error(error))
1498 return error;
1499
1500 if (mpio->pgpath)
1501 fail_path(mpio->pgpath);
1502
1503 if (!atomic_read(&m->nr_valid_paths)) {
1504 if (!test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) {
1505 if (!must_push_back_rq(m))
1506 r = -EIO;
1507 }
1508 }
1509
1510 return r;
1511 }
1512
1513 static int multipath_end_io(struct dm_target *ti, struct request *clone,
1514 int error, union map_info *map_context)
1515 {
1516 struct multipath *m = ti->private;
1517 struct dm_mpath_io *mpio = get_mpio(map_context);
1518 struct pgpath *pgpath;
1519 struct path_selector *ps;
1520 int r;
1521
1522 BUG_ON(!mpio);
1523
1524 r = do_end_io(m, clone, error, mpio);
1525 pgpath = mpio->pgpath;
1526 if (pgpath) {
1527 ps = &pgpath->pg->ps;
1528 if (ps->type->end_io)
1529 ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
1530 }
1531
1532 return r;
1533 }
1534
1535 static int do_end_io_bio(struct multipath *m, struct bio *clone,
1536 int error, struct dm_mpath_io *mpio)
1537 {
1538 unsigned long flags;
1539
1540 if (!error)
1541 return 0; /* I/O complete */
1542
1543 if (noretry_error(error))
1544 return error;
1545
1546 if (mpio->pgpath)
1547 fail_path(mpio->pgpath);
1548
1549 if (!atomic_read(&m->nr_valid_paths)) {
1550 if (!test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) {
1551 if (!must_push_back_bio(m))
1552 return -EIO;
1553 return DM_ENDIO_REQUEUE;
1554 }
1555 }
1556
1557 /* Queue for the daemon to resubmit */
1558 dm_bio_restore(get_bio_details_from_bio(clone), clone);
1559
1560 spin_lock_irqsave(&m->lock, flags);
1561 bio_list_add(&m->queued_bios, clone);
1562 spin_unlock_irqrestore(&m->lock, flags);
1563 if (!test_bit(MPATHF_QUEUE_IO, &m->flags))
1564 queue_work(kmultipathd, &m->process_queued_bios);
1565
1566 return DM_ENDIO_INCOMPLETE;
1567 }
1568
1569 static int multipath_end_io_bio(struct dm_target *ti, struct bio *clone, int error)
1570 {
1571 struct multipath *m = ti->private;
1572 struct dm_mpath_io *mpio = get_mpio_from_bio(clone);
1573 struct pgpath *pgpath;
1574 struct path_selector *ps;
1575 int r;
1576
1577 BUG_ON(!mpio);
1578
1579 r = do_end_io_bio(m, clone, error, mpio);
1580 pgpath = mpio->pgpath;
1581 if (pgpath) {
1582 ps = &pgpath->pg->ps;
1583 if (ps->type->end_io)
1584 ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
1585 }
1586
1587 return r;
1588 }
1589
1590 /*
1591 * Suspend can't complete until all the I/O is processed so if
1592 * the last path fails we must error any remaining I/O.
1593 * Note that if the freeze_bdev fails while suspending, the
1594 * queue_if_no_path state is lost - userspace should reset it.
1595 */
1596 static void multipath_presuspend(struct dm_target *ti)
1597 {
1598 struct multipath *m = ti->private;
1599
1600 queue_if_no_path(m, false, true);
1601 }
1602
1603 static void multipath_postsuspend(struct dm_target *ti)
1604 {
1605 struct multipath *m = ti->private;
1606
1607 mutex_lock(&m->work_mutex);
1608 flush_multipath_work(m);
1609 mutex_unlock(&m->work_mutex);
1610 }
1611
1612 /*
1613 * Restore the queue_if_no_path setting.
1614 */
1615 static void multipath_resume(struct dm_target *ti)
1616 {
1617 struct multipath *m = ti->private;
1618 unsigned long flags;
1619
1620 spin_lock_irqsave(&m->lock, flags);
1621 if (test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags))
1622 set_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags);
1623 else
1624 clear_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags);
1625 spin_unlock_irqrestore(&m->lock, flags);
1626 }
1627
1628 /*
1629 * Info output has the following format:
1630 * num_multipath_feature_args [multipath_feature_args]*
1631 * num_handler_status_args [handler_status_args]*
1632 * num_groups init_group_number
1633 * [A|D|E num_ps_status_args [ps_status_args]*
1634 * num_paths num_selector_args
1635 * [path_dev A|F fail_count [selector_args]* ]+ ]+
1636 *
1637 * Table output has the following format (identical to the constructor string):
1638 * num_feature_args [features_args]*
1639 * num_handler_args hw_handler [hw_handler_args]*
1640 * num_groups init_group_number
1641 * [priority selector-name num_ps_args [ps_args]*
1642 * num_paths num_selector_args [path_dev [selector_args]* ]+ ]+
1643 */
1644 static void multipath_status(struct dm_target *ti, status_type_t type,
1645 unsigned status_flags, char *result, unsigned maxlen)
1646 {
1647 int sz = 0;
1648 unsigned long flags;
1649 struct multipath *m = ti->private;
1650 struct priority_group *pg;
1651 struct pgpath *p;
1652 unsigned pg_num;
1653 char state;
1654
1655 spin_lock_irqsave(&m->lock, flags);
1656
1657 /* Features */
1658 if (type == STATUSTYPE_INFO)
1659 DMEMIT("2 %u %u ", test_bit(MPATHF_QUEUE_IO, &m->flags),
1660 atomic_read(&m->pg_init_count));
1661 else {
1662 DMEMIT("%u ", test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) +
1663 (m->pg_init_retries > 0) * 2 +
1664 (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) * 2 +
1665 test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags) +
1666 (m->queue_mode != DM_TYPE_REQUEST_BASED) * 2);
1667
1668 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
1669 DMEMIT("queue_if_no_path ");
1670 if (m->pg_init_retries)
1671 DMEMIT("pg_init_retries %u ", m->pg_init_retries);
1672 if (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT)
1673 DMEMIT("pg_init_delay_msecs %u ", m->pg_init_delay_msecs);
1674 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags))
1675 DMEMIT("retain_attached_hw_handler ");
1676 if (m->queue_mode != DM_TYPE_REQUEST_BASED) {
1677 switch(m->queue_mode) {
1678 case DM_TYPE_BIO_BASED:
1679 DMEMIT("queue_mode bio ");
1680 break;
1681 case DM_TYPE_MQ_REQUEST_BASED:
1682 DMEMIT("queue_mode mq ");
1683 break;
1684 }
1685 }
1686 }
1687
1688 if (!m->hw_handler_name || type == STATUSTYPE_INFO)
1689 DMEMIT("0 ");
1690 else
1691 DMEMIT("1 %s ", m->hw_handler_name);
1692
1693 DMEMIT("%u ", m->nr_priority_groups);
1694
1695 if (m->next_pg)
1696 pg_num = m->next_pg->pg_num;
1697 else if (m->current_pg)
1698 pg_num = m->current_pg->pg_num;
1699 else
1700 pg_num = (m->nr_priority_groups ? 1 : 0);
1701
1702 DMEMIT("%u ", pg_num);
1703
1704 switch (type) {
1705 case STATUSTYPE_INFO:
1706 list_for_each_entry(pg, &m->priority_groups, list) {
1707 if (pg->bypassed)
1708 state = 'D'; /* Disabled */
1709 else if (pg == m->current_pg)
1710 state = 'A'; /* Currently Active */
1711 else
1712 state = 'E'; /* Enabled */
1713
1714 DMEMIT("%c ", state);
1715
1716 if (pg->ps.type->status)
1717 sz += pg->ps.type->status(&pg->ps, NULL, type,
1718 result + sz,
1719 maxlen - sz);
1720 else
1721 DMEMIT("0 ");
1722
1723 DMEMIT("%u %u ", pg->nr_pgpaths,
1724 pg->ps.type->info_args);
1725
1726 list_for_each_entry(p, &pg->pgpaths, list) {
1727 DMEMIT("%s %s %u ", p->path.dev->name,
1728 p->is_active ? "A" : "F",
1729 p->fail_count);
1730 if (pg->ps.type->status)
1731 sz += pg->ps.type->status(&pg->ps,
1732 &p->path, type, result + sz,
1733 maxlen - sz);
1734 }
1735 }
1736 break;
1737
1738 case STATUSTYPE_TABLE:
1739 list_for_each_entry(pg, &m->priority_groups, list) {
1740 DMEMIT("%s ", pg->ps.type->name);
1741
1742 if (pg->ps.type->status)
1743 sz += pg->ps.type->status(&pg->ps, NULL, type,
1744 result + sz,
1745 maxlen - sz);
1746 else
1747 DMEMIT("0 ");
1748
1749 DMEMIT("%u %u ", pg->nr_pgpaths,
1750 pg->ps.type->table_args);
1751
1752 list_for_each_entry(p, &pg->pgpaths, list) {
1753 DMEMIT("%s ", p->path.dev->name);
1754 if (pg->ps.type->status)
1755 sz += pg->ps.type->status(&pg->ps,
1756 &p->path, type, result + sz,
1757 maxlen - sz);
1758 }
1759 }
1760 break;
1761 }
1762
1763 spin_unlock_irqrestore(&m->lock, flags);
1764 }
1765
1766 static int multipath_message(struct dm_target *ti, unsigned argc, char **argv)
1767 {
1768 int r = -EINVAL;
1769 struct dm_dev *dev;
1770 struct multipath *m = ti->private;
1771 action_fn action;
1772
1773 mutex_lock(&m->work_mutex);
1774
1775 if (dm_suspended(ti)) {
1776 r = -EBUSY;
1777 goto out;
1778 }
1779
1780 if (argc == 1) {
1781 if (!strcasecmp(argv[0], "queue_if_no_path")) {
1782 r = queue_if_no_path(m, true, false);
1783 goto out;
1784 } else if (!strcasecmp(argv[0], "fail_if_no_path")) {
1785 r = queue_if_no_path(m, false, false);
1786 goto out;
1787 }
1788 }
1789
1790 if (argc != 2) {
1791 DMWARN("Invalid multipath message arguments. Expected 2 arguments, got %d.", argc);
1792 goto out;
1793 }
1794
1795 if (!strcasecmp(argv[0], "disable_group")) {
1796 r = bypass_pg_num(m, argv[1], true);
1797 goto out;
1798 } else if (!strcasecmp(argv[0], "enable_group")) {
1799 r = bypass_pg_num(m, argv[1], false);
1800 goto out;
1801 } else if (!strcasecmp(argv[0], "switch_group")) {
1802 r = switch_pg_num(m, argv[1]);
1803 goto out;
1804 } else if (!strcasecmp(argv[0], "reinstate_path"))
1805 action = reinstate_path;
1806 else if (!strcasecmp(argv[0], "fail_path"))
1807 action = fail_path;
1808 else {
1809 DMWARN("Unrecognised multipath message received: %s", argv[0]);
1810 goto out;
1811 }
1812
1813 r = dm_get_device(ti, argv[1], dm_table_get_mode(ti->table), &dev);
1814 if (r) {
1815 DMWARN("message: error getting device %s",
1816 argv[1]);
1817 goto out;
1818 }
1819
1820 r = action_dev(m, dev, action);
1821
1822 dm_put_device(ti, dev);
1823
1824 out:
1825 mutex_unlock(&m->work_mutex);
1826 return r;
1827 }
1828
1829 static int multipath_prepare_ioctl(struct dm_target *ti,
1830 struct block_device **bdev, fmode_t *mode)
1831 {
1832 struct multipath *m = ti->private;
1833 struct pgpath *current_pgpath;
1834 int r;
1835
1836 current_pgpath = lockless_dereference(m->current_pgpath);
1837 if (!current_pgpath)
1838 current_pgpath = choose_pgpath(m, 0);
1839
1840 if (current_pgpath) {
1841 if (!test_bit(MPATHF_QUEUE_IO, &m->flags)) {
1842 *bdev = current_pgpath->path.dev->bdev;
1843 *mode = current_pgpath->path.dev->mode;
1844 r = 0;
1845 } else {
1846 /* pg_init has not started or completed */
1847 r = -ENOTCONN;
1848 }
1849 } else {
1850 /* No path is available */
1851 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
1852 r = -ENOTCONN;
1853 else
1854 r = -EIO;
1855 }
1856
1857 if (r == -ENOTCONN) {
1858 if (!lockless_dereference(m->current_pg)) {
1859 /* Path status changed, redo selection */
1860 (void) choose_pgpath(m, 0);
1861 }
1862 if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
1863 pg_init_all_paths(m);
1864 dm_table_run_md_queue_async(m->ti->table);
1865 process_queued_io_list(m);
1866 }
1867
1868 /*
1869 * Only pass ioctls through if the device sizes match exactly.
1870 */
1871 if (!r && ti->len != i_size_read((*bdev)->bd_inode) >> SECTOR_SHIFT)
1872 return 1;
1873 return r;
1874 }
1875
1876 static int multipath_iterate_devices(struct dm_target *ti,
1877 iterate_devices_callout_fn fn, void *data)
1878 {
1879 struct multipath *m = ti->private;
1880 struct priority_group *pg;
1881 struct pgpath *p;
1882 int ret = 0;
1883
1884 list_for_each_entry(pg, &m->priority_groups, list) {
1885 list_for_each_entry(p, &pg->pgpaths, list) {
1886 ret = fn(ti, p->path.dev, ti->begin, ti->len, data);
1887 if (ret)
1888 goto out;
1889 }
1890 }
1891
1892 out:
1893 return ret;
1894 }
1895
1896 static int pgpath_busy(struct pgpath *pgpath)
1897 {
1898 struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
1899
1900 return blk_lld_busy(q);
1901 }
1902
1903 /*
1904 * We return "busy", only when we can map I/Os but underlying devices
1905 * are busy (so even if we map I/Os now, the I/Os will wait on
1906 * the underlying queue).
1907 * In other words, if we want to kill I/Os or queue them inside us
1908 * due to map unavailability, we don't return "busy". Otherwise,
1909 * dm core won't give us the I/Os and we can't do what we want.
1910 */
1911 static int multipath_busy(struct dm_target *ti)
1912 {
1913 bool busy = false, has_active = false;
1914 struct multipath *m = ti->private;
1915 struct priority_group *pg, *next_pg;
1916 struct pgpath *pgpath;
1917
1918 /* pg_init in progress */
1919 if (atomic_read(&m->pg_init_in_progress))
1920 return true;
1921
1922 /* no paths available, for blk-mq: rely on IO mapping to delay requeue */
1923 if (!atomic_read(&m->nr_valid_paths) && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
1924 return (m->queue_mode != DM_TYPE_MQ_REQUEST_BASED);
1925
1926 /* Guess which priority_group will be used at next mapping time */
1927 pg = lockless_dereference(m->current_pg);
1928 next_pg = lockless_dereference(m->next_pg);
1929 if (unlikely(!lockless_dereference(m->current_pgpath) && next_pg))
1930 pg = next_pg;
1931
1932 if (!pg) {
1933 /*
1934 * We don't know which pg will be used at next mapping time.
1935 * We don't call choose_pgpath() here to avoid to trigger
1936 * pg_init just by busy checking.
1937 * So we don't know whether underlying devices we will be using
1938 * at next mapping time are busy or not. Just try mapping.
1939 */
1940 return busy;
1941 }
1942
1943 /*
1944 * If there is one non-busy active path at least, the path selector
1945 * will be able to select it. So we consider such a pg as not busy.
1946 */
1947 busy = true;
1948 list_for_each_entry(pgpath, &pg->pgpaths, list) {
1949 if (pgpath->is_active) {
1950 has_active = true;
1951 if (!pgpath_busy(pgpath)) {
1952 busy = false;
1953 break;
1954 }
1955 }
1956 }
1957
1958 if (!has_active) {
1959 /*
1960 * No active path in this pg, so this pg won't be used and
1961 * the current_pg will be changed at next mapping time.
1962 * We need to try mapping to determine it.
1963 */
1964 busy = false;
1965 }
1966
1967 return busy;
1968 }
1969
1970 /*-----------------------------------------------------------------
1971 * Module setup
1972 *---------------------------------------------------------------*/
1973 static struct target_type multipath_target = {
1974 .name = "multipath",
1975 .version = {1, 12, 0},
1976 .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE,
1977 .module = THIS_MODULE,
1978 .ctr = multipath_ctr,
1979 .dtr = multipath_dtr,
1980 .clone_and_map_rq = multipath_clone_and_map,
1981 .release_clone_rq = multipath_release_clone,
1982 .rq_end_io = multipath_end_io,
1983 .map = multipath_map_bio,
1984 .end_io = multipath_end_io_bio,
1985 .presuspend = multipath_presuspend,
1986 .postsuspend = multipath_postsuspend,
1987 .resume = multipath_resume,
1988 .status = multipath_status,
1989 .message = multipath_message,
1990 .prepare_ioctl = multipath_prepare_ioctl,
1991 .iterate_devices = multipath_iterate_devices,
1992 .busy = multipath_busy,
1993 };
1994
1995 static int __init dm_multipath_init(void)
1996 {
1997 int r;
1998
1999 r = dm_register_target(&multipath_target);
2000 if (r < 0) {
2001 DMERR("request-based register failed %d", r);
2002 r = -EINVAL;
2003 goto bad_register_target;
2004 }
2005
2006 kmultipathd = alloc_workqueue("kmpathd", WQ_MEM_RECLAIM, 0);
2007 if (!kmultipathd) {
2008 DMERR("failed to create workqueue kmpathd");
2009 r = -ENOMEM;
2010 goto bad_alloc_kmultipathd;
2011 }
2012
2013 /*
2014 * A separate workqueue is used to handle the device handlers
2015 * to avoid overloading existing workqueue. Overloading the
2016 * old workqueue would also create a bottleneck in the
2017 * path of the storage hardware device activation.
2018 */
2019 kmpath_handlerd = alloc_ordered_workqueue("kmpath_handlerd",
2020 WQ_MEM_RECLAIM);
2021 if (!kmpath_handlerd) {
2022 DMERR("failed to create workqueue kmpath_handlerd");
2023 r = -ENOMEM;
2024 goto bad_alloc_kmpath_handlerd;
2025 }
2026
2027 return 0;
2028
2029 bad_alloc_kmpath_handlerd:
2030 destroy_workqueue(kmultipathd);
2031 bad_alloc_kmultipathd:
2032 dm_unregister_target(&multipath_target);
2033 bad_register_target:
2034 return r;
2035 }
2036
2037 static void __exit dm_multipath_exit(void)
2038 {
2039 destroy_workqueue(kmpath_handlerd);
2040 destroy_workqueue(kmultipathd);
2041
2042 dm_unregister_target(&multipath_target);
2043 }
2044
2045 module_init(dm_multipath_init);
2046 module_exit(dm_multipath_exit);
2047
2048 MODULE_DESCRIPTION(DM_NAME " multipath target");
2049 MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>");
2050 MODULE_LICENSE("GPL");
Ubuntu Artful kernel mirror