]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame_incremental - drivers/md/raid5.c
projects / mirror_ubuntu-artful-kernel.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
md/raid456: downlevel multicore operations to raid_run_ops
[mirror_ubuntu-artful-kernel.git] / drivers / md / raid5.c
... / ...
CommitLineData
1/*
2 * raid5.c : Multiple Devices driver for Linux
3 * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
4 * Copyright (C) 1999, 2000 Ingo Molnar
5 * Copyright (C) 2002, 2003 H. Peter Anvin
6 *
7 * RAID-4/5/6 management functions.
8 * Thanks to Penguin Computing for making the RAID-6 development possible
9 * by donating a test server!
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2, or (at your option)
14 * any later version.
15 *
16 * You should have received a copy of the GNU General Public License
17 * (for example /usr/src/linux/COPYING); if not, write to the Free
18 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19 */
20
21/*
22 * BITMAP UNPLUGGING:
23 *
24 * The sequencing for updating the bitmap reliably is a little
25 * subtle (and I got it wrong the first time) so it deserves some
26 * explanation.
27 *
28 * We group bitmap updates into batches. Each batch has a number.
29 * We may write out several batches at once, but that isn't very important.
30 * conf->bm_write is the number of the last batch successfully written.
31 * conf->bm_flush is the number of the last batch that was closed to
32 * new additions.
33 * When we discover that we will need to write to any block in a stripe
34 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq
35 * the number of the batch it will be in. This is bm_flush+1.
36 * When we are ready to do a write, if that batch hasn't been written yet,
37 * we plug the array and queue the stripe for later.
38 * When an unplug happens, we increment bm_flush, thus closing the current
39 * batch.
40 * When we notice that bm_flush > bm_write, we write out all pending updates
41 * to the bitmap, and advance bm_write to where bm_flush was.
42 * This may occasionally write a bit out twice, but is sure never to
43 * miss any bits.
44 */
45
46#include <linux/blkdev.h>
47#include <linux/kthread.h>
48#include <linux/raid/pq.h>
49#include <linux/async_tx.h>
50#include <linux/async.h>
51#include <linux/seq_file.h>
52#include <linux/cpu.h>
53#include "md.h"
54#include "raid5.h"
55#include "bitmap.h"
56
57/*
58 * Stripe cache
59 */
60
61#define NR_STRIPES 256
62#define STRIPE_SIZE PAGE_SIZE
63#define STRIPE_SHIFT (PAGE_SHIFT - 9)
64#define STRIPE_SECTORS (STRIPE_SIZE>>9)
65#define IO_THRESHOLD 1
66#define BYPASS_THRESHOLD 1
67#define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head))
68#define HASH_MASK (NR_HASH - 1)
69
70#define stripe_hash(conf, sect) (&((conf)->stripe_hashtbl[((sect) >> STRIPE_SHIFT) & HASH_MASK]))
71
72/* bio's attached to a stripe+device for I/O are linked together in bi_sector
73 * order without overlap. There may be several bio's per stripe+device, and
74 * a bio could span several devices.
75 * When walking this list for a particular stripe+device, we must never proceed
76 * beyond a bio that extends past this device, as the next bio might no longer
77 * be valid.
78 * This macro is used to determine the 'next' bio in the list, given the sector
79 * of the current stripe+device
80 */
81#define r5_next_bio(bio, sect) ( ( (bio)->bi_sector + ((bio)->bi_size>>9) < sect + STRIPE_SECTORS) ? (bio)->bi_next : NULL)
82/*
83 * The following can be used to debug the driver
84 */
85#define RAID5_PARANOIA 1
86#if RAID5_PARANOIA && defined(CONFIG_SMP)
87# define CHECK_DEVLOCK() assert_spin_locked(&conf->device_lock)
88#else
89# define CHECK_DEVLOCK()
90#endif
91
92#ifdef DEBUG
93#define inline
94#define __inline__
95#endif
96
97#define printk_rl(args...) ((void) (printk_ratelimit() && printk(args)))
98
99/*
100 * We maintain a biased count of active stripes in the bottom 16 bits of
101 * bi_phys_segments, and a count of processed stripes in the upper 16 bits
102 */
103static inline int raid5_bi_phys_segments(struct bio *bio)
104{
105 return bio->bi_phys_segments & 0xffff;
106}
107
108static inline int raid5_bi_hw_segments(struct bio *bio)
109{
110 return (bio->bi_phys_segments >> 16) & 0xffff;
111}
112
113static inline int raid5_dec_bi_phys_segments(struct bio *bio)
114{
115 --bio->bi_phys_segments;
116 return raid5_bi_phys_segments(bio);
117}
118
119static inline int raid5_dec_bi_hw_segments(struct bio *bio)
120{
121 unsigned short val = raid5_bi_hw_segments(bio);
122
123 --val;
124 bio->bi_phys_segments = (val << 16) | raid5_bi_phys_segments(bio);
125 return val;
126}
127
128static inline void raid5_set_bi_hw_segments(struct bio *bio, unsigned int cnt)
129{
130 bio->bi_phys_segments = raid5_bi_phys_segments(bio) || (cnt << 16);
131}
132
133/* Find first data disk in a raid6 stripe */
134static inline int raid6_d0(struct stripe_head *sh)
135{
136 if (sh->ddf_layout)
137 /* ddf always start from first device */
138 return 0;
139 /* md starts just after Q block */
140 if (sh->qd_idx == sh->disks - 1)
141 return 0;
142 else
143 return sh->qd_idx + 1;
144}
145static inline int raid6_next_disk(int disk, int raid_disks)
146{
147 disk++;
148 return (disk < raid_disks) ? disk : 0;
149}
150
151/* When walking through the disks in a raid5, starting at raid6_d0,
152 * We need to map each disk to a 'slot', where the data disks are slot
153 * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk
154 * is raid_disks-1. This help does that mapping.
155 */
156static int raid6_idx_to_slot(int idx, struct stripe_head *sh,
157 int *count, int syndrome_disks)
158{
159 int slot;
160
161 if (idx == sh->pd_idx)
162 return syndrome_disks;
163 if (idx == sh->qd_idx)
164 return syndrome_disks + 1;
165 slot = (*count)++;
166 return slot;
167}
168
169static void return_io(struct bio *return_bi)
170{
171 struct bio *bi = return_bi;
172 while (bi) {
173
174 return_bi = bi->bi_next;
175 bi->bi_next = NULL;
176 bi->bi_size = 0;
177 bio_endio(bi, 0);
178 bi = return_bi;
179 }
180}
181
182static void print_raid5_conf (raid5_conf_t *conf);
183
184static int stripe_operations_active(struct stripe_head *sh)
185{
186 return sh->check_state || sh->reconstruct_state ||
187 test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||
188 test_bit(STRIPE_COMPUTE_RUN, &sh->state);
189}
190
191static void __release_stripe(raid5_conf_t *conf, struct stripe_head *sh)
192{
193 if (atomic_dec_and_test(&sh->count)) {
194 BUG_ON(!list_empty(&sh->lru));
195 BUG_ON(atomic_read(&conf->active_stripes)==0);
196 if (test_bit(STRIPE_HANDLE, &sh->state)) {
197 if (test_bit(STRIPE_DELAYED, &sh->state)) {
198 list_add_tail(&sh->lru, &conf->delayed_list);
199 blk_plug_device(conf->mddev->queue);
200 } else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
201 sh->bm_seq - conf->seq_write > 0) {
202 list_add_tail(&sh->lru, &conf->bitmap_list);
203 blk_plug_device(conf->mddev->queue);
204 } else {
205 clear_bit(STRIPE_BIT_DELAY, &sh->state);
206 list_add_tail(&sh->lru, &conf->handle_list);
207 }
208 md_wakeup_thread(conf->mddev->thread);
209 } else {
210 BUG_ON(stripe_operations_active(sh));
211 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
212 atomic_dec(&conf->preread_active_stripes);
213 if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD)
214 md_wakeup_thread(conf->mddev->thread);
215 }
216 atomic_dec(&conf->active_stripes);
217 if (!test_bit(STRIPE_EXPANDING, &sh->state)) {
218 list_add_tail(&sh->lru, &conf->inactive_list);
219 wake_up(&conf->wait_for_stripe);
220 if (conf->retry_read_aligned)
221 md_wakeup_thread(conf->mddev->thread);
222 }
223 }
224 }
225}
226
227static void release_stripe(struct stripe_head *sh)
228{
229 raid5_conf_t *conf = sh->raid_conf;
230 unsigned long flags;
231
232 spin_lock_irqsave(&conf->device_lock, flags);
233 __release_stripe(conf, sh);
234 spin_unlock_irqrestore(&conf->device_lock, flags);
235}
236
237static inline void remove_hash(struct stripe_head *sh)
238{
239 pr_debug("remove_hash(), stripe %llu\n",
240 (unsigned long long)sh->sector);
241
242 hlist_del_init(&sh->hash);
243}
244
245static inline void insert_hash(raid5_conf_t *conf, struct stripe_head *sh)
246{
247 struct hlist_head *hp = stripe_hash(conf, sh->sector);
248
249 pr_debug("insert_hash(), stripe %llu\n",
250 (unsigned long long)sh->sector);
251
252 CHECK_DEVLOCK();
253 hlist_add_head(&sh->hash, hp);
254}
255
256
257/* find an idle stripe, make sure it is unhashed, and return it. */
258static struct stripe_head *get_free_stripe(raid5_conf_t *conf)
259{
260 struct stripe_head *sh = NULL;
261 struct list_head *first;
262
263 CHECK_DEVLOCK();
264 if (list_empty(&conf->inactive_list))
265 goto out;
266 first = conf->inactive_list.next;
267 sh = list_entry(first, struct stripe_head, lru);
268 list_del_init(first);
269 remove_hash(sh);
270 atomic_inc(&conf->active_stripes);
271out:
272 return sh;
273}
274
275static void shrink_buffers(struct stripe_head *sh, int num)
276{
277 struct page *p;
278 int i;
279
280 for (i=0; i<num ; i++) {
281 p = sh->dev[i].page;
282 if (!p)
283 continue;
284 sh->dev[i].page = NULL;
285 put_page(p);
286 }
287}
288
289static int grow_buffers(struct stripe_head *sh, int num)
290{
291 int i;
292
293 for (i=0; i<num; i++) {
294 struct page *page;
295
296 if (!(page = alloc_page(GFP_KERNEL))) {
297 return 1;
298 }
299 sh->dev[i].page = page;
300 }
301 return 0;
302}
303
304static void raid5_build_block(struct stripe_head *sh, int i, int previous);
305static void stripe_set_idx(sector_t stripe, raid5_conf_t *conf, int previous,
306 struct stripe_head *sh);
307
308static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)
309{
310 raid5_conf_t *conf = sh->raid_conf;
311 int i;
312
313 BUG_ON(atomic_read(&sh->count) != 0);
314 BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
315 BUG_ON(stripe_operations_active(sh));
316
317 CHECK_DEVLOCK();
318 pr_debug("init_stripe called, stripe %llu\n",
319 (unsigned long long)sh->sector);
320
321 remove_hash(sh);
322
323 sh->generation = conf->generation - previous;
324 sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;
325 sh->sector = sector;
326 stripe_set_idx(sector, conf, previous, sh);
327 sh->state = 0;
328
329
330 for (i = sh->disks; i--; ) {
331 struct r5dev *dev = &sh->dev[i];
332
333 if (dev->toread || dev->read || dev->towrite || dev->written ||
334 test_bit(R5_LOCKED, &dev->flags)) {
335 printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n",
336 (unsigned long long)sh->sector, i, dev->toread,
337 dev->read, dev->towrite, dev->written,
338 test_bit(R5_LOCKED, &dev->flags));
339 BUG();
340 }
341 dev->flags = 0;
342 raid5_build_block(sh, i, previous);
343 }
344 insert_hash(conf, sh);
345}
346
347static struct stripe_head *__find_stripe(raid5_conf_t *conf, sector_t sector,
348 short generation)
349{
350 struct stripe_head *sh;
351 struct hlist_node *hn;
352
353 CHECK_DEVLOCK();
354 pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector);
355 hlist_for_each_entry(sh, hn, stripe_hash(conf, sector), hash)
356 if (sh->sector == sector && sh->generation == generation)
357 return sh;
358 pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector);
359 return NULL;
360}
361
362static void unplug_slaves(mddev_t *mddev);
363static void raid5_unplug_device(struct request_queue *q);
364
365static struct stripe_head *
366get_active_stripe(raid5_conf_t *conf, sector_t sector,
367 int previous, int noblock, int noquiesce)
368{
369 struct stripe_head *sh;
370
371 pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);
372
373 spin_lock_irq(&conf->device_lock);
374
375 do {
376 wait_event_lock_irq(conf->wait_for_stripe,
377 conf->quiesce == 0 || noquiesce,
378 conf->device_lock, /* nothing */);
379 sh = __find_stripe(conf, sector, conf->generation - previous);
380 if (!sh) {
381 if (!conf->inactive_blocked)
382 sh = get_free_stripe(conf);
383 if (noblock && sh == NULL)
384 break;
385 if (!sh) {
386 conf->inactive_blocked = 1;
387 wait_event_lock_irq(conf->wait_for_stripe,
388 !list_empty(&conf->inactive_list) &&
389 (atomic_read(&conf->active_stripes)
390 < (conf->max_nr_stripes *3/4)
391 || !conf->inactive_blocked),
392 conf->device_lock,
393 raid5_unplug_device(conf->mddev->queue)
394 );
395 conf->inactive_blocked = 0;
396 } else
397 init_stripe(sh, sector, previous);
398 } else {
399 if (atomic_read(&sh->count)) {
400 BUG_ON(!list_empty(&sh->lru)
401 && !test_bit(STRIPE_EXPANDING, &sh->state));
402 } else {
403 if (!test_bit(STRIPE_HANDLE, &sh->state))
404 atomic_inc(&conf->active_stripes);
405 if (list_empty(&sh->lru) &&
406 !test_bit(STRIPE_EXPANDING, &sh->state))
407 BUG();
408 list_del_init(&sh->lru);
409 }
410 }
411 } while (sh == NULL);
412
413 if (sh)
414 atomic_inc(&sh->count);
415
416 spin_unlock_irq(&conf->device_lock);
417 return sh;
418}
419
420static void
421raid5_end_read_request(struct bio *bi, int error);
422static void
423raid5_end_write_request(struct bio *bi, int error);
424
425static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
426{
427 raid5_conf_t *conf = sh->raid_conf;
428 int i, disks = sh->disks;
429
430 might_sleep();
431
432 for (i = disks; i--; ) {
433 int rw;
434 struct bio *bi;
435 mdk_rdev_t *rdev;
436 if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags))
437 rw = WRITE;
438 else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
439 rw = READ;
440 else
441 continue;
442
443 bi = &sh->dev[i].req;
444
445 bi->bi_rw = rw;
446 if (rw == WRITE)
447 bi->bi_end_io = raid5_end_write_request;
448 else
449 bi->bi_end_io = raid5_end_read_request;
450
451 rcu_read_lock();
452 rdev = rcu_dereference(conf->disks[i].rdev);
453 if (rdev && test_bit(Faulty, &rdev->flags))
454 rdev = NULL;
455 if (rdev)
456 atomic_inc(&rdev->nr_pending);
457 rcu_read_unlock();
458
459 if (rdev) {
460 if (s->syncing || s->expanding || s->expanded)
461 md_sync_acct(rdev->bdev, STRIPE_SECTORS);
462
463 set_bit(STRIPE_IO_STARTED, &sh->state);
464
465 bi->bi_bdev = rdev->bdev;
466 pr_debug("%s: for %llu schedule op %ld on disc %d\n",
467 __func__, (unsigned long long)sh->sector,
468 bi->bi_rw, i);
469 atomic_inc(&sh->count);
470 bi->bi_sector = sh->sector + rdev->data_offset;
471 bi->bi_flags = 1 << BIO_UPTODATE;
472 bi->bi_vcnt = 1;
473 bi->bi_max_vecs = 1;
474 bi->bi_idx = 0;
475 bi->bi_io_vec = &sh->dev[i].vec;
476 bi->bi_io_vec[0].bv_len = STRIPE_SIZE;
477 bi->bi_io_vec[0].bv_offset = 0;
478 bi->bi_size = STRIPE_SIZE;
479 bi->bi_next = NULL;
480 if (rw == WRITE &&
481 test_bit(R5_ReWrite, &sh->dev[i].flags))
482 atomic_add(STRIPE_SECTORS,
483 &rdev->corrected_errors);
484 generic_make_request(bi);
485 } else {
486 if (rw == WRITE)
487 set_bit(STRIPE_DEGRADED, &sh->state);
488 pr_debug("skip op %ld on disc %d for sector %llu\n",
489 bi->bi_rw, i, (unsigned long long)sh->sector);
490 clear_bit(R5_LOCKED, &sh->dev[i].flags);
491 set_bit(STRIPE_HANDLE, &sh->state);
492 }
493 }
494}
495
496static struct dma_async_tx_descriptor *
497async_copy_data(int frombio, struct bio *bio, struct page *page,
498 sector_t sector, struct dma_async_tx_descriptor *tx)
499{
500 struct bio_vec *bvl;
501 struct page *bio_page;
502 int i;
503 int page_offset;
504 struct async_submit_ctl submit;
505 enum async_tx_flags flags = 0;
506
507 if (bio->bi_sector >= sector)
508 page_offset = (signed)(bio->bi_sector - sector) * 512;
509 else
510 page_offset = (signed)(sector - bio->bi_sector) * -512;
511
512 if (frombio)
513 flags |= ASYNC_TX_FENCE;
514 init_async_submit(&submit, flags, tx, NULL, NULL, NULL);
515
516 bio_for_each_segment(bvl, bio, i) {
517 int len = bio_iovec_idx(bio, i)->bv_len;
518 int clen;
519 int b_offset = 0;
520
521 if (page_offset < 0) {
522 b_offset = -page_offset;
523 page_offset += b_offset;
524 len -= b_offset;
525 }
526
527 if (len > 0 && page_offset + len > STRIPE_SIZE)
528 clen = STRIPE_SIZE - page_offset;
529 else
530 clen = len;
531
532 if (clen > 0) {
533 b_offset += bio_iovec_idx(bio, i)->bv_offset;
534 bio_page = bio_iovec_idx(bio, i)->bv_page;
535 if (frombio)
536 tx = async_memcpy(page, bio_page, page_offset,
537 b_offset, clen, &submit);
538 else
539 tx = async_memcpy(bio_page, page, b_offset,
540 page_offset, clen, &submit);
541 }
542 /* chain the operations */
543 submit.depend_tx = tx;
544
545 if (clen < len) /* hit end of page */
546 break;
547 page_offset += len;
548 }
549
550 return tx;
551}
552
553static void ops_complete_biofill(void *stripe_head_ref)
554{
555 struct stripe_head *sh = stripe_head_ref;
556 struct bio *return_bi = NULL;
557 raid5_conf_t *conf = sh->raid_conf;
558 int i;
559
560 pr_debug("%s: stripe %llu\n", __func__,
561 (unsigned long long)sh->sector);
562
563 /* clear completed biofills */
564 spin_lock_irq(&conf->device_lock);
565 for (i = sh->disks; i--; ) {
566 struct r5dev *dev = &sh->dev[i];
567
568 /* acknowledge completion of a biofill operation */
569 /* and check if we need to reply to a read request,
570 * new R5_Wantfill requests are held off until
571 * !STRIPE_BIOFILL_RUN
572 */
573 if (test_and_clear_bit(R5_Wantfill, &dev->flags)) {
574 struct bio *rbi, *rbi2;
575
576 BUG_ON(!dev->read);
577 rbi = dev->read;
578 dev->read = NULL;
579 while (rbi && rbi->bi_sector <
580 dev->sector + STRIPE_SECTORS) {
581 rbi2 = r5_next_bio(rbi, dev->sector);
582 if (!raid5_dec_bi_phys_segments(rbi)) {
583 rbi->bi_next = return_bi;
584 return_bi = rbi;
585 }
586 rbi = rbi2;
587 }
588 }
589 }
590 spin_unlock_irq(&conf->device_lock);
591 clear_bit(STRIPE_BIOFILL_RUN, &sh->state);
592
593 return_io(return_bi);
594
595 set_bit(STRIPE_HANDLE, &sh->state);
596 release_stripe(sh);
597}
598
599static void ops_run_biofill(struct stripe_head *sh)
600{
601 struct dma_async_tx_descriptor *tx = NULL;
602 raid5_conf_t *conf = sh->raid_conf;
603 struct async_submit_ctl submit;
604 int i;
605
606 pr_debug("%s: stripe %llu\n", __func__,
607 (unsigned long long)sh->sector);
608
609 for (i = sh->disks; i--; ) {
610 struct r5dev *dev = &sh->dev[i];
611 if (test_bit(R5_Wantfill, &dev->flags)) {
612 struct bio *rbi;
613 spin_lock_irq(&conf->device_lock);
614 dev->read = rbi = dev->toread;
615 dev->toread = NULL;
616 spin_unlock_irq(&conf->device_lock);
617 while (rbi && rbi->bi_sector <
618 dev->sector + STRIPE_SECTORS) {
619 tx = async_copy_data(0, rbi, dev->page,
620 dev->sector, tx);
621 rbi = r5_next_bio(rbi, dev->sector);
622 }
623 }
624 }
625
626 atomic_inc(&sh->count);
627 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);
628 async_trigger_callback(&submit);
629}
630
631static void mark_target_uptodate(struct stripe_head *sh, int target)
632{
633 struct r5dev *tgt;
634
635 if (target < 0)
636 return;
637
638 tgt = &sh->dev[target];
639 set_bit(R5_UPTODATE, &tgt->flags);
640 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
641 clear_bit(R5_Wantcompute, &tgt->flags);
642}
643
644static void ops_complete_compute(void *stripe_head_ref)
645{
646 struct stripe_head *sh = stripe_head_ref;
647
648 pr_debug("%s: stripe %llu\n", __func__,
649 (unsigned long long)sh->sector);
650
651 /* mark the computed target(s) as uptodate */
652 mark_target_uptodate(sh, sh->ops.target);
653 mark_target_uptodate(sh, sh->ops.target2);
654
655 clear_bit(STRIPE_COMPUTE_RUN, &sh->state);
656 if (sh->check_state == check_state_compute_run)
657 sh->check_state = check_state_compute_result;
658 set_bit(STRIPE_HANDLE, &sh->state);
659 release_stripe(sh);
660}
661
662/* return a pointer to the address conversion region of the scribble buffer */
663static addr_conv_t *to_addr_conv(struct stripe_head *sh,
664 struct raid5_percpu *percpu)
665{
666 return percpu->scribble + sizeof(struct page *) * (sh->disks + 2);
667}
668
669static struct dma_async_tx_descriptor *
670ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
671{
672 int disks = sh->disks;
673 struct page **xor_srcs = percpu->scribble;
674 int target = sh->ops.target;
675 struct r5dev *tgt = &sh->dev[target];
676 struct page *xor_dest = tgt->page;
677 int count = 0;
678 struct dma_async_tx_descriptor *tx;
679 struct async_submit_ctl submit;
680 int i;
681
682 pr_debug("%s: stripe %llu block: %d\n",
683 __func__, (unsigned long long)sh->sector, target);
684 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
685
686 for (i = disks; i--; )
687 if (i != target)
688 xor_srcs[count++] = sh->dev[i].page;
689
690 atomic_inc(&sh->count);
691
692 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL,
693 ops_complete_compute, sh, to_addr_conv(sh, percpu));
694 if (unlikely(count == 1))
695 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
696 else
697 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
698
699 return tx;
700}
701
702/* set_syndrome_sources - populate source buffers for gen_syndrome
703 * @srcs - (struct page *) array of size sh->disks
704 * @sh - stripe_head to parse
705 *
706 * Populates srcs in proper layout order for the stripe and returns the
707 * 'count' of sources to be used in a call to async_gen_syndrome. The P
708 * destination buffer is recorded in srcs[count] and the Q destination
709 * is recorded in srcs[count+1]].
710 */
711static int set_syndrome_sources(struct page **srcs, struct stripe_head *sh)
712{
713 int disks = sh->disks;
714 int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);
715 int d0_idx = raid6_d0(sh);
716 int count;
717 int i;
718
719 for (i = 0; i < disks; i++)
720 srcs[i] = (void *)raid6_empty_zero_page;
721
722 count = 0;
723 i = d0_idx;
724 do {
725 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
726
727 srcs[slot] = sh->dev[i].page;
728 i = raid6_next_disk(i, disks);
729 } while (i != d0_idx);
730 BUG_ON(count != syndrome_disks);
731
732 return count;
733}
734
735static struct dma_async_tx_descriptor *
736ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
737{
738 int disks = sh->disks;
739 struct page **blocks = percpu->scribble;
740 int target;
741 int qd_idx = sh->qd_idx;
742 struct dma_async_tx_descriptor *tx;
743 struct async_submit_ctl submit;
744 struct r5dev *tgt;
745 struct page *dest;
746 int i;
747 int count;
748
749 if (sh->ops.target < 0)
750 target = sh->ops.target2;
751 else if (sh->ops.target2 < 0)
752 target = sh->ops.target;
753 else
754 /* we should only have one valid target */
755 BUG();
756 BUG_ON(target < 0);
757 pr_debug("%s: stripe %llu block: %d\n",
758 __func__, (unsigned long long)sh->sector, target);
759
760 tgt = &sh->dev[target];
761 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
762 dest = tgt->page;
763
764 atomic_inc(&sh->count);
765
766 if (target == qd_idx) {
767 count = set_syndrome_sources(blocks, sh);
768 blocks[count] = NULL; /* regenerating p is not necessary */
769 BUG_ON(blocks[count+1] != dest); /* q should already be set */
770 init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
771 ops_complete_compute, sh,
772 to_addr_conv(sh, percpu));
773 tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
774 } else {
775 /* Compute any data- or p-drive using XOR */
776 count = 0;
777 for (i = disks; i-- ; ) {
778 if (i == target || i == qd_idx)
779 continue;
780 blocks[count++] = sh->dev[i].page;
781 }
782
783 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
784 NULL, ops_complete_compute, sh,
785 to_addr_conv(sh, percpu));
786 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit);
787 }
788
789 return tx;
790}
791
792static struct dma_async_tx_descriptor *
793ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
794{
795 int i, count, disks = sh->disks;
796 int syndrome_disks = sh->ddf_layout ? disks : disks-2;
797 int d0_idx = raid6_d0(sh);
798 int faila = -1, failb = -1;
799 int target = sh->ops.target;
800 int target2 = sh->ops.target2;
801 struct r5dev *tgt = &sh->dev[target];
802 struct r5dev *tgt2 = &sh->dev[target2];
803 struct dma_async_tx_descriptor *tx;
804 struct page **blocks = percpu->scribble;
805 struct async_submit_ctl submit;
806
807 pr_debug("%s: stripe %llu block1: %d block2: %d\n",
808 __func__, (unsigned long long)sh->sector, target, target2);
809 BUG_ON(target < 0 || target2 < 0);
810 BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
811 BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags));
812
813 /* we need to open-code set_syndrome_sources to handle the
814 * slot number conversion for 'faila' and 'failb'
815 */
816 for (i = 0; i < disks ; i++)
817 blocks[i] = (void *)raid6_empty_zero_page;
818 count = 0;
819 i = d0_idx;
820 do {
821 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
822
823 blocks[slot] = sh->dev[i].page;
824
825 if (i == target)
826 faila = slot;
827 if (i == target2)
828 failb = slot;
829 i = raid6_next_disk(i, disks);
830 } while (i != d0_idx);
831 BUG_ON(count != syndrome_disks);
832
833 BUG_ON(faila == failb);
834 if (failb < faila)
835 swap(faila, failb);
836 pr_debug("%s: stripe: %llu faila: %d failb: %d\n",
837 __func__, (unsigned long long)sh->sector, faila, failb);
838
839 atomic_inc(&sh->count);
840
841 if (failb == syndrome_disks+1) {
842 /* Q disk is one of the missing disks */
843 if (faila == syndrome_disks) {
844 /* Missing P+Q, just recompute */
845 init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
846 ops_complete_compute, sh,
847 to_addr_conv(sh, percpu));
848 return async_gen_syndrome(blocks, 0, count+2,
849 STRIPE_SIZE, &submit);
850 } else {
851 struct page *dest;
852 int data_target;
853 int qd_idx = sh->qd_idx;
854
855 /* Missing D+Q: recompute D from P, then recompute Q */
856 if (target == qd_idx)
857 data_target = target2;
858 else
859 data_target = target;
860
861 count = 0;
862 for (i = disks; i-- ; ) {
863 if (i == data_target || i == qd_idx)
864 continue;
865 blocks[count++] = sh->dev[i].page;
866 }
867 dest = sh->dev[data_target].page;
868 init_async_submit(&submit,
869 ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
870 NULL, NULL, NULL,
871 to_addr_conv(sh, percpu));
872 tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE,
873 &submit);
874
875 count = set_syndrome_sources(blocks, sh);
876 init_async_submit(&submit, ASYNC_TX_FENCE, tx,
877 ops_complete_compute, sh,
878 to_addr_conv(sh, percpu));
879 return async_gen_syndrome(blocks, 0, count+2,
880 STRIPE_SIZE, &submit);
881 }
882 } else {
883 init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
884 ops_complete_compute, sh,
885 to_addr_conv(sh, percpu));
886 if (failb == syndrome_disks) {
887 /* We're missing D+P. */
888 return async_raid6_datap_recov(syndrome_disks+2,
889 STRIPE_SIZE, faila,
890 blocks, &submit);
891 } else {
892 /* We're missing D+D. */
893 return async_raid6_2data_recov(syndrome_disks+2,
894 STRIPE_SIZE, faila, failb,
895 blocks, &submit);
896 }
897 }
898}
899
900
901static void ops_complete_prexor(void *stripe_head_ref)
902{
903 struct stripe_head *sh = stripe_head_ref;
904
905 pr_debug("%s: stripe %llu\n", __func__,
906 (unsigned long long)sh->sector);
907}
908
909static struct dma_async_tx_descriptor *
910ops_run_prexor(struct stripe_head *sh, struct raid5_percpu *percpu,
911 struct dma_async_tx_descriptor *tx)
912{
913 int disks = sh->disks;
914 struct page **xor_srcs = percpu->scribble;
915 int count = 0, pd_idx = sh->pd_idx, i;
916 struct async_submit_ctl submit;
917
918 /* existing parity data subtracted */
919 struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
920
921 pr_debug("%s: stripe %llu\n", __func__,
922 (unsigned long long)sh->sector);
923
924 for (i = disks; i--; ) {
925 struct r5dev *dev = &sh->dev[i];
926 /* Only process blocks that are known to be uptodate */
927 if (test_bit(R5_Wantdrain, &dev->flags))
928 xor_srcs[count++] = dev->page;
929 }
930
931 init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
932 ops_complete_prexor, sh, to_addr_conv(sh, percpu));
933 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
934
935 return tx;
936}
937
938static struct dma_async_tx_descriptor *
939ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
940{
941 int disks = sh->disks;
942 int i;
943
944 pr_debug("%s: stripe %llu\n", __func__,
945 (unsigned long long)sh->sector);
946
947 for (i = disks; i--; ) {
948 struct r5dev *dev = &sh->dev[i];
949 struct bio *chosen;
950
951 if (test_and_clear_bit(R5_Wantdrain, &dev->flags)) {
952 struct bio *wbi;
953
954 spin_lock(&sh->lock);
955 chosen = dev->towrite;
956 dev->towrite = NULL;
957 BUG_ON(dev->written);
958 wbi = dev->written = chosen;
959 spin_unlock(&sh->lock);
960
961 while (wbi && wbi->bi_sector <
962 dev->sector + STRIPE_SECTORS) {
963 tx = async_copy_data(1, wbi, dev->page,
964 dev->sector, tx);
965 wbi = r5_next_bio(wbi, dev->sector);
966 }
967 }
968 }
969
970 return tx;
971}
972
973static void ops_complete_reconstruct(void *stripe_head_ref)
974{
975 struct stripe_head *sh = stripe_head_ref;
976 int disks = sh->disks;
977 int pd_idx = sh->pd_idx;
978 int qd_idx = sh->qd_idx;
979 int i;
980
981 pr_debug("%s: stripe %llu\n", __func__,
982 (unsigned long long)sh->sector);
983
984 for (i = disks; i--; ) {
985 struct r5dev *dev = &sh->dev[i];
986
987 if (dev->written || i == pd_idx || i == qd_idx)
988 set_bit(R5_UPTODATE, &dev->flags);
989 }
990
991 if (sh->reconstruct_state == reconstruct_state_drain_run)
992 sh->reconstruct_state = reconstruct_state_drain_result;
993 else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)
994 sh->reconstruct_state = reconstruct_state_prexor_drain_result;
995 else {
996 BUG_ON(sh->reconstruct_state != reconstruct_state_run);
997 sh->reconstruct_state = reconstruct_state_result;
998 }
999
1000 set_bit(STRIPE_HANDLE, &sh->state);
1001 release_stripe(sh);
1002}
1003
1004static void
1005ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
1006 struct dma_async_tx_descriptor *tx)
1007{
1008 int disks = sh->disks;
1009 struct page **xor_srcs = percpu->scribble;
1010 struct async_submit_ctl submit;
1011 int count = 0, pd_idx = sh->pd_idx, i;
1012 struct page *xor_dest;
1013 int prexor = 0;
1014 unsigned long flags;
1015
1016 pr_debug("%s: stripe %llu\n", __func__,
1017 (unsigned long long)sh->sector);
1018
1019 /* check if prexor is active which means only process blocks
1020 * that are part of a read-modify-write (written)
1021 */
1022 if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
1023 prexor = 1;
1024 xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
1025 for (i = disks; i--; ) {
1026 struct r5dev *dev = &sh->dev[i];
1027 if (dev->written)
1028 xor_srcs[count++] = dev->page;
1029 }
1030 } else {
1031 xor_dest = sh->dev[pd_idx].page;
1032 for (i = disks; i--; ) {
1033 struct r5dev *dev = &sh->dev[i];
1034 if (i != pd_idx)
1035 xor_srcs[count++] = dev->page;
1036 }
1037 }
1038
1039 /* 1/ if we prexor'd then the dest is reused as a source
1040 * 2/ if we did not prexor then we are redoing the parity
1041 * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST
1042 * for the synchronous xor case
1043 */
1044 flags = ASYNC_TX_ACK |
1045 (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST);
1046
1047 atomic_inc(&sh->count);
1048
1049 init_async_submit(&submit, flags, tx, ops_complete_reconstruct, sh,
1050 to_addr_conv(sh, percpu));
1051 if (unlikely(count == 1))
1052 tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
1053 else
1054 tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1055}
1056
1057static void
1058ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
1059 struct dma_async_tx_descriptor *tx)
1060{
1061 struct async_submit_ctl submit;
1062 struct page **blocks = percpu->scribble;
1063 int count;
1064
1065 pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);
1066
1067 count = set_syndrome_sources(blocks, sh);
1068
1069 atomic_inc(&sh->count);
1070
1071 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_reconstruct,
1072 sh, to_addr_conv(sh, percpu));
1073 async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
1074}
1075
1076static void ops_complete_check(void *stripe_head_ref)
1077{
1078 struct stripe_head *sh = stripe_head_ref;
1079
1080 pr_debug("%s: stripe %llu\n", __func__,
1081 (unsigned long long)sh->sector);
1082
1083 sh->check_state = check_state_check_result;
1084 set_bit(STRIPE_HANDLE, &sh->state);
1085 release_stripe(sh);
1086}
1087
1088static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
1089{
1090 int disks = sh->disks;
1091 int pd_idx = sh->pd_idx;
1092 int qd_idx = sh->qd_idx;
1093 struct page *xor_dest;
1094 struct page **xor_srcs = percpu->scribble;
1095 struct dma_async_tx_descriptor *tx;
1096 struct async_submit_ctl submit;
1097 int count;
1098 int i;
1099
1100 pr_debug("%s: stripe %llu\n", __func__,
1101 (unsigned long long)sh->sector);
1102
1103 count = 0;
1104 xor_dest = sh->dev[pd_idx].page;
1105 xor_srcs[count++] = xor_dest;
1106 for (i = disks; i--; ) {
1107 if (i == pd_idx || i == qd_idx)
1108 continue;
1109 xor_srcs[count++] = sh->dev[i].page;
1110 }
1111
1112 init_async_submit(&submit, 0, NULL, NULL, NULL,
1113 to_addr_conv(sh, percpu));
1114 tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
1115 &sh->ops.zero_sum_result, &submit);
1116
1117 atomic_inc(&sh->count);
1118 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);
1119 tx = async_trigger_callback(&submit);
1120}
1121
1122static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)
1123{
1124 struct page **srcs = percpu->scribble;
1125 struct async_submit_ctl submit;
1126 int count;
1127
1128 pr_debug("%s: stripe %llu checkp: %d\n", __func__,
1129 (unsigned long long)sh->sector, checkp);
1130
1131 count = set_syndrome_sources(srcs, sh);
1132 if (!checkp)
1133 srcs[count] = NULL;
1134
1135 atomic_inc(&sh->count);
1136 init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
1137 sh, to_addr_conv(sh, percpu));
1138 async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE,
1139 &sh->ops.zero_sum_result, percpu->spare_page, &submit);
1140}
1141
1142static void __raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
1143{
1144 int overlap_clear = 0, i, disks = sh->disks;
1145 struct dma_async_tx_descriptor *tx = NULL;
1146 raid5_conf_t *conf = sh->raid_conf;
1147 int level = conf->level;
1148 struct raid5_percpu *percpu;
1149 unsigned long cpu;
1150
1151 cpu = get_cpu();
1152 percpu = per_cpu_ptr(conf->percpu, cpu);
1153 if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
1154 ops_run_biofill(sh);
1155 overlap_clear++;
1156 }
1157
1158 if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) {
1159 if (level < 6)
1160 tx = ops_run_compute5(sh, percpu);
1161 else {
1162 if (sh->ops.target2 < 0 || sh->ops.target < 0)
1163 tx = ops_run_compute6_1(sh, percpu);
1164 else
1165 tx = ops_run_compute6_2(sh, percpu);
1166 }
1167 /* terminate the chain if reconstruct is not set to be run */
1168 if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request))
1169 async_tx_ack(tx);
1170 }
1171
1172 if (test_bit(STRIPE_OP_PREXOR, &ops_request))
1173 tx = ops_run_prexor(sh, percpu, tx);
1174
1175 if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {
1176 tx = ops_run_biodrain(sh, tx);
1177 overlap_clear++;
1178 }
1179
1180 if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) {
1181 if (level < 6)
1182 ops_run_reconstruct5(sh, percpu, tx);
1183 else
1184 ops_run_reconstruct6(sh, percpu, tx);
1185 }
1186
1187 if (test_bit(STRIPE_OP_CHECK, &ops_request)) {
1188 if (sh->check_state == check_state_run)
1189 ops_run_check_p(sh, percpu);
1190 else if (sh->check_state == check_state_run_q)
1191 ops_run_check_pq(sh, percpu, 0);
1192 else if (sh->check_state == check_state_run_pq)
1193 ops_run_check_pq(sh, percpu, 1);
1194 else
1195 BUG();
1196 }
1197
1198 if (overlap_clear)
1199 for (i = disks; i--; ) {
1200 struct r5dev *dev = &sh->dev[i];
1201 if (test_and_clear_bit(R5_Overlap, &dev->flags))
1202 wake_up(&sh->raid_conf->wait_for_overlap);
1203 }
1204 put_cpu();
1205}
1206
1207#ifdef CONFIG_MULTICORE_RAID456
1208static void async_run_ops(void *param, async_cookie_t cookie)
1209{
1210 struct stripe_head *sh = param;
1211 unsigned long ops_request = sh->ops.request;
1212
1213 clear_bit_unlock(STRIPE_OPS_REQ_PENDING, &sh->state);
1214 wake_up(&sh->ops.wait_for_ops);
1215
1216 __raid_run_ops(sh, ops_request);
1217 release_stripe(sh);
1218}
1219
1220static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
1221{
1222 /* since handle_stripe can be called outside of raid5d context
1223 * we need to ensure sh->ops.request is de-staged before another
1224 * request arrives
1225 */
1226 wait_event(sh->ops.wait_for_ops,
1227 !test_and_set_bit_lock(STRIPE_OPS_REQ_PENDING, &sh->state));
1228 sh->ops.request = ops_request;
1229
1230 atomic_inc(&sh->count);
1231 async_schedule(async_run_ops, sh);
1232}
1233#else
1234#define raid_run_ops __raid_run_ops
1235#endif
1236
1237static int grow_one_stripe(raid5_conf_t *conf)
1238{
1239 struct stripe_head *sh;
1240 sh = kmem_cache_alloc(conf->slab_cache, GFP_KERNEL);
1241 if (!sh)
1242 return 0;
1243 memset(sh, 0, sizeof(*sh) + (conf->raid_disks-1)*sizeof(struct r5dev));
1244 sh->raid_conf = conf;
1245 spin_lock_init(&sh->lock);
1246 #ifdef CONFIG_MULTICORE_RAID456
1247 init_waitqueue_head(&sh->ops.wait_for_ops);
1248 #endif
1249
1250 if (grow_buffers(sh, conf->raid_disks)) {
1251 shrink_buffers(sh, conf->raid_disks);
1252 kmem_cache_free(conf->slab_cache, sh);
1253 return 0;
1254 }
1255 sh->disks = conf->raid_disks;
1256 /* we just created an active stripe so... */
1257 atomic_set(&sh->count, 1);
1258 atomic_inc(&conf->active_stripes);
1259 INIT_LIST_HEAD(&sh->lru);
1260 release_stripe(sh);
1261 return 1;
1262}
1263
1264static int grow_stripes(raid5_conf_t *conf, int num)
1265{
1266 struct kmem_cache *sc;
1267 int devs = conf->raid_disks;
1268
1269 sprintf(conf->cache_name[0],
1270 "raid%d-%s", conf->level, mdname(conf->mddev));
1271 sprintf(conf->cache_name[1],
1272 "raid%d-%s-alt", conf->level, mdname(conf->mddev));
1273 conf->active_name = 0;
1274 sc = kmem_cache_create(conf->cache_name[conf->active_name],
1275 sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev),
1276 0, 0, NULL);
1277 if (!sc)
1278 return 1;
1279 conf->slab_cache = sc;
1280 conf->pool_size = devs;
1281 while (num--)
1282 if (!grow_one_stripe(conf))
1283 return 1;
1284 return 0;
1285}
1286
1287/**
1288 * scribble_len - return the required size of the scribble region
1289 * @num - total number of disks in the array
1290 *
1291 * The size must be enough to contain:
1292 * 1/ a struct page pointer for each device in the array +2
1293 * 2/ room to convert each entry in (1) to its corresponding dma
1294 * (dma_map_page()) or page (page_address()) address.
1295 *
1296 * Note: the +2 is for the destination buffers of the ddf/raid6 case where we
1297 * calculate over all devices (not just the data blocks), using zeros in place
1298 * of the P and Q blocks.
1299 */
1300static size_t scribble_len(int num)
1301{
1302 size_t len;
1303
1304 len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2);
1305
1306 return len;
1307}
1308
1309static int resize_stripes(raid5_conf_t *conf, int newsize)
1310{
1311 /* Make all the stripes able to hold 'newsize' devices.
1312 * New slots in each stripe get 'page' set to a new page.
1313 *
1314 * This happens in stages:
1315 * 1/ create a new kmem_cache and allocate the required number of
1316 * stripe_heads.
1317 * 2/ gather all the old stripe_heads and tranfer the pages across
1318 * to the new stripe_heads. This will have the side effect of
1319 * freezing the array as once all stripe_heads have been collected,
1320 * no IO will be possible. Old stripe heads are freed once their
1321 * pages have been transferred over, and the old kmem_cache is
1322 * freed when all stripes are done.
1323 * 3/ reallocate conf->disks to be suitable bigger. If this fails,
1324 * we simple return a failre status - no need to clean anything up.
1325 * 4/ allocate new pages for the new slots in the new stripe_heads.
1326 * If this fails, we don't bother trying the shrink the
1327 * stripe_heads down again, we just leave them as they are.
1328 * As each stripe_head is processed the new one is released into
1329 * active service.
1330 *
1331 * Once step2 is started, we cannot afford to wait for a write,
1332 * so we use GFP_NOIO allocations.
1333 */
1334 struct stripe_head *osh, *nsh;
1335 LIST_HEAD(newstripes);
1336 struct disk_info *ndisks;
1337 unsigned long cpu;
1338 int err;
1339 struct kmem_cache *sc;
1340 int i;
1341
1342 if (newsize <= conf->pool_size)
1343 return 0; /* never bother to shrink */
1344
1345 err = md_allow_write(conf->mddev);
1346 if (err)
1347 return err;
1348
1349 /* Step 1 */
1350 sc = kmem_cache_create(conf->cache_name[1-conf->active_name],
1351 sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev),
1352 0, 0, NULL);
1353 if (!sc)
1354 return -ENOMEM;
1355
1356 for (i = conf->max_nr_stripes; i; i--) {
1357 nsh = kmem_cache_alloc(sc, GFP_KERNEL);
1358 if (!nsh)
1359 break;
1360
1361 memset(nsh, 0, sizeof(*nsh) + (newsize-1)*sizeof(struct r5dev));
1362
1363 nsh->raid_conf = conf;
1364 spin_lock_init(&nsh->lock);
1365 #ifdef CONFIG_MULTICORE_RAID456
1366 init_waitqueue_head(&nsh->ops.wait_for_ops);
1367 #endif
1368
1369 list_add(&nsh->lru, &newstripes);
1370 }
1371 if (i) {
1372 /* didn't get enough, give up */
1373 while (!list_empty(&newstripes)) {
1374 nsh = list_entry(newstripes.next, struct stripe_head, lru);
1375 list_del(&nsh->lru);
1376 kmem_cache_free(sc, nsh);
1377 }
1378 kmem_cache_destroy(sc);
1379 return -ENOMEM;
1380 }
1381 /* Step 2 - Must use GFP_NOIO now.
1382 * OK, we have enough stripes, start collecting inactive
1383 * stripes and copying them over
1384 */
1385 list_for_each_entry(nsh, &newstripes, lru) {
1386 spin_lock_irq(&conf->device_lock);
1387 wait_event_lock_irq(conf->wait_for_stripe,
1388 !list_empty(&conf->inactive_list),
1389 conf->device_lock,
1390 unplug_slaves(conf->mddev)
1391 );
1392 osh = get_free_stripe(conf);
1393 spin_unlock_irq(&conf->device_lock);
1394 atomic_set(&nsh->count, 1);
1395 for(i=0; i<conf->pool_size; i++)
1396 nsh->dev[i].page = osh->dev[i].page;
1397 for( ; i<newsize; i++)
1398 nsh->dev[i].page = NULL;
1399 kmem_cache_free(conf->slab_cache, osh);
1400 }
1401 kmem_cache_destroy(conf->slab_cache);
1402
1403 /* Step 3.
1404 * At this point, we are holding all the stripes so the array
1405 * is completely stalled, so now is a good time to resize
1406 * conf->disks and the scribble region
1407 */
1408 ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO);
1409 if (ndisks) {
1410 for (i=0; i<conf->raid_disks; i++)
1411 ndisks[i] = conf->disks[i];
1412 kfree(conf->disks);
1413 conf->disks = ndisks;
1414 } else
1415 err = -ENOMEM;
1416
1417 get_online_cpus();
1418 conf->scribble_len = scribble_len(newsize);
1419 for_each_present_cpu(cpu) {
1420 struct raid5_percpu *percpu;
1421 void *scribble;
1422
1423 percpu = per_cpu_ptr(conf->percpu, cpu);
1424 scribble = kmalloc(conf->scribble_len, GFP_NOIO);
1425
1426 if (scribble) {
1427 kfree(percpu->scribble);
1428 percpu->scribble = scribble;
1429 } else {
1430 err = -ENOMEM;
1431 break;
1432 }
1433 }
1434 put_online_cpus();
1435
1436 /* Step 4, return new stripes to service */
1437 while(!list_empty(&newstripes)) {
1438 nsh = list_entry(newstripes.next, struct stripe_head, lru);
1439 list_del_init(&nsh->lru);
1440
1441 for (i=conf->raid_disks; i < newsize; i++)
1442 if (nsh->dev[i].page == NULL) {
1443 struct page *p = alloc_page(GFP_NOIO);
1444 nsh->dev[i].page = p;
1445 if (!p)
1446 err = -ENOMEM;
1447 }
1448 release_stripe(nsh);
1449 }
1450 /* critical section pass, GFP_NOIO no longer needed */
1451
1452 conf->slab_cache = sc;
1453 conf->active_name = 1-conf->active_name;
1454 conf->pool_size = newsize;
1455 return err;
1456}
1457
1458static int drop_one_stripe(raid5_conf_t *conf)
1459{
1460 struct stripe_head *sh;
1461
1462 spin_lock_irq(&conf->device_lock);
1463 sh = get_free_stripe(conf);
1464 spin_unlock_irq(&conf->device_lock);
1465 if (!sh)
1466 return 0;
1467 BUG_ON(atomic_read(&sh->count));
1468 shrink_buffers(sh, conf->pool_size);
1469 kmem_cache_free(conf->slab_cache, sh);
1470 atomic_dec(&conf->active_stripes);
1471 return 1;
1472}
1473
1474static void shrink_stripes(raid5_conf_t *conf)
1475{
1476 while (drop_one_stripe(conf))
1477 ;
1478
1479 if (conf->slab_cache)
1480 kmem_cache_destroy(conf->slab_cache);
1481 conf->slab_cache = NULL;
1482}
1483
1484static void raid5_end_read_request(struct bio * bi, int error)
1485{
1486 struct stripe_head *sh = bi->bi_private;
1487 raid5_conf_t *conf = sh->raid_conf;
1488 int disks = sh->disks, i;
1489 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
1490 char b[BDEVNAME_SIZE];
1491 mdk_rdev_t *rdev;
1492
1493
1494 for (i=0 ; i<disks; i++)
1495 if (bi == &sh->dev[i].req)
1496 break;
1497
1498 pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n",
1499 (unsigned long long)sh->sector, i, atomic_read(&sh->count),
1500 uptodate);
1501 if (i == disks) {
1502 BUG();
1503 return;
1504 }
1505
1506 if (uptodate) {
1507 set_bit(R5_UPTODATE, &sh->dev[i].flags);
1508 if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
1509 rdev = conf->disks[i].rdev;
1510 printk_rl(KERN_INFO "raid5:%s: read error corrected"
1511 " (%lu sectors at %llu on %s)\n",
1512 mdname(conf->mddev), STRIPE_SECTORS,
1513 (unsigned long long)(sh->sector
1514 + rdev->data_offset),
1515 bdevname(rdev->bdev, b));
1516 clear_bit(R5_ReadError, &sh->dev[i].flags);
1517 clear_bit(R5_ReWrite, &sh->dev[i].flags);
1518 }
1519 if (atomic_read(&conf->disks[i].rdev->read_errors))
1520 atomic_set(&conf->disks[i].rdev->read_errors, 0);
1521 } else {
1522 const char *bdn = bdevname(conf->disks[i].rdev->bdev, b);
1523 int retry = 0;
1524 rdev = conf->disks[i].rdev;
1525
1526 clear_bit(R5_UPTODATE, &sh->dev[i].flags);
1527 atomic_inc(&rdev->read_errors);
1528 if (conf->mddev->degraded)
1529 printk_rl(KERN_WARNING
1530 "raid5:%s: read error not correctable "
1531 "(sector %llu on %s).\n",
1532 mdname(conf->mddev),
1533 (unsigned long long)(sh->sector
1534 + rdev->data_offset),
1535 bdn);
1536 else if (test_bit(R5_ReWrite, &sh->dev[i].flags))
1537 /* Oh, no!!! */
1538 printk_rl(KERN_WARNING
1539 "raid5:%s: read error NOT corrected!! "
1540 "(sector %llu on %s).\n",
1541 mdname(conf->mddev),
1542 (unsigned long long)(sh->sector
1543 + rdev->data_offset),
1544 bdn);
1545 else if (atomic_read(&rdev->read_errors)
1546 > conf->max_nr_stripes)
1547 printk(KERN_WARNING
1548 "raid5:%s: Too many read errors, failing device %s.\n",
1549 mdname(conf->mddev), bdn);
1550 else
1551 retry = 1;
1552 if (retry)
1553 set_bit(R5_ReadError, &sh->dev[i].flags);
1554 else {
1555 clear_bit(R5_ReadError, &sh->dev[i].flags);
1556 clear_bit(R5_ReWrite, &sh->dev[i].flags);
1557 md_error(conf->mddev, rdev);
1558 }
1559 }
1560 rdev_dec_pending(conf->disks[i].rdev, conf->mddev);
1561 clear_bit(R5_LOCKED, &sh->dev[i].flags);
1562 set_bit(STRIPE_HANDLE, &sh->state);
1563 release_stripe(sh);
1564}
1565
1566static void raid5_end_write_request(struct bio *bi, int error)
1567{
1568 struct stripe_head *sh = bi->bi_private;
1569 raid5_conf_t *conf = sh->raid_conf;
1570 int disks = sh->disks, i;
1571 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
1572
1573 for (i=0 ; i<disks; i++)
1574 if (bi == &sh->dev[i].req)
1575 break;
1576
1577 pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n",
1578 (unsigned long long)sh->sector, i, atomic_read(&sh->count),
1579 uptodate);
1580 if (i == disks) {
1581 BUG();
1582 return;
1583 }
1584
1585 if (!uptodate)
1586 md_error(conf->mddev, conf->disks[i].rdev);
1587
1588 rdev_dec_pending(conf->disks[i].rdev, conf->mddev);
1589
1590 clear_bit(R5_LOCKED, &sh->dev[i].flags);
1591 set_bit(STRIPE_HANDLE, &sh->state);
1592 release_stripe(sh);
1593}
1594
1595
1596static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous);
1597
1598static void raid5_build_block(struct stripe_head *sh, int i, int previous)
1599{
1600 struct r5dev *dev = &sh->dev[i];
1601
1602 bio_init(&dev->req);
1603 dev->req.bi_io_vec = &dev->vec;
1604 dev->req.bi_vcnt++;
1605 dev->req.bi_max_vecs++;
1606 dev->vec.bv_page = dev->page;
1607 dev->vec.bv_len = STRIPE_SIZE;
1608 dev->vec.bv_offset = 0;
1609
1610 dev->req.bi_sector = sh->sector;
1611 dev->req.bi_private = sh;
1612
1613 dev->flags = 0;
1614 dev->sector = compute_blocknr(sh, i, previous);
1615}
1616
1617static void error(mddev_t *mddev, mdk_rdev_t *rdev)
1618{
1619 char b[BDEVNAME_SIZE];
1620 raid5_conf_t *conf = (raid5_conf_t *) mddev->private;
1621 pr_debug("raid5: error called\n");
1622
1623 if (!test_bit(Faulty, &rdev->flags)) {
1624 set_bit(MD_CHANGE_DEVS, &mddev->flags);
1625 if (test_and_clear_bit(In_sync, &rdev->flags)) {
1626 unsigned long flags;
1627 spin_lock_irqsave(&conf->device_lock, flags);
1628 mddev->degraded++;
1629 spin_unlock_irqrestore(&conf->device_lock, flags);
1630 /*
1631 * if recovery was running, make sure it aborts.
1632 */
1633 set_bit(MD_RECOVERY_INTR, &mddev->recovery);
1634 }
1635 set_bit(Faulty, &rdev->flags);
1636 printk(KERN_ALERT
1637 "raid5: Disk failure on %s, disabling device.\n"
1638 "raid5: Operation continuing on %d devices.\n",
1639 bdevname(rdev->bdev,b), conf->raid_disks - mddev->degraded);
1640 }
1641}
1642
1643/*
1644 * Input: a 'big' sector number,
1645 * Output: index of the data and parity disk, and the sector # in them.
1646 */
1647static sector_t raid5_compute_sector(raid5_conf_t *conf, sector_t r_sector,
1648 int previous, int *dd_idx,
1649 struct stripe_head *sh)
1650{
1651 long stripe;
1652 unsigned long chunk_number;
1653 unsigned int chunk_offset;
1654 int pd_idx, qd_idx;
1655 int ddf_layout = 0;
1656 sector_t new_sector;
1657 int algorithm = previous ? conf->prev_algo
1658 : conf->algorithm;
1659 int sectors_per_chunk = previous ? conf->prev_chunk_sectors
1660 : conf->chunk_sectors;
1661 int raid_disks = previous ? conf->previous_raid_disks
1662 : conf->raid_disks;
1663 int data_disks = raid_disks - conf->max_degraded;
1664
1665 /* First compute the information on this sector */
1666
1667 /*
1668 * Compute the chunk number and the sector offset inside the chunk
1669 */
1670 chunk_offset = sector_div(r_sector, sectors_per_chunk);
1671 chunk_number = r_sector;
1672 BUG_ON(r_sector != chunk_number);
1673
1674 /*
1675 * Compute the stripe number
1676 */
1677 stripe = chunk_number / data_disks;
1678
1679 /*
1680 * Compute the data disk and parity disk indexes inside the stripe
1681 */
1682 *dd_idx = chunk_number % data_disks;
1683
1684 /*
1685 * Select the parity disk based on the user selected algorithm.
1686 */
1687 pd_idx = qd_idx = ~0;
1688 switch(conf->level) {
1689 case 4:
1690 pd_idx = data_disks;
1691 break;
1692 case 5:
1693 switch (algorithm) {
1694 case ALGORITHM_LEFT_ASYMMETRIC:
1695 pd_idx = data_disks - stripe % raid_disks;
1696 if (*dd_idx >= pd_idx)
1697 (*dd_idx)++;
1698 break;
1699 case ALGORITHM_RIGHT_ASYMMETRIC:
1700 pd_idx = stripe % raid_disks;
1701 if (*dd_idx >= pd_idx)
1702 (*dd_idx)++;
1703 break;
1704 case ALGORITHM_LEFT_SYMMETRIC:
1705 pd_idx = data_disks - stripe % raid_disks;
1706 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
1707 break;
1708 case ALGORITHM_RIGHT_SYMMETRIC:
1709 pd_idx = stripe % raid_disks;
1710 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
1711 break;
1712 case ALGORITHM_PARITY_0:
1713 pd_idx = 0;
1714 (*dd_idx)++;
1715 break;
1716 case ALGORITHM_PARITY_N:
1717 pd_idx = data_disks;
1718 break;
1719 default:
1720 printk(KERN_ERR "raid5: unsupported algorithm %d\n",
1721 algorithm);
1722 BUG();
1723 }
1724 break;
1725 case 6:
1726
1727 switch (algorithm) {
1728 case ALGORITHM_LEFT_ASYMMETRIC:
1729 pd_idx = raid_disks - 1 - (stripe % raid_disks);
1730 qd_idx = pd_idx + 1;
1731 if (pd_idx == raid_disks-1) {
1732 (*dd_idx)++; /* Q D D D P */
1733 qd_idx = 0;
1734 } else if (*dd_idx >= pd_idx)
1735 (*dd_idx) += 2; /* D D P Q D */
1736 break;
1737 case ALGORITHM_RIGHT_ASYMMETRIC:
1738 pd_idx = stripe % raid_disks;
1739 qd_idx = pd_idx + 1;
1740 if (pd_idx == raid_disks-1) {
1741 (*dd_idx)++; /* Q D D D P */
1742 qd_idx = 0;
1743 } else if (*dd_idx >= pd_idx)
1744 (*dd_idx) += 2; /* D D P Q D */
1745 break;
1746 case ALGORITHM_LEFT_SYMMETRIC:
1747 pd_idx = raid_disks - 1 - (stripe % raid_disks);
1748 qd_idx = (pd_idx + 1) % raid_disks;
1749 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
1750 break;
1751 case ALGORITHM_RIGHT_SYMMETRIC:
1752 pd_idx = stripe % raid_disks;
1753 qd_idx = (pd_idx + 1) % raid_disks;
1754 *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
1755 break;
1756
1757 case ALGORITHM_PARITY_0:
1758 pd_idx = 0;
1759 qd_idx = 1;
1760 (*dd_idx) += 2;
1761 break;
1762 case ALGORITHM_PARITY_N:
1763 pd_idx = data_disks;
1764 qd_idx = data_disks + 1;
1765 break;
1766
1767 case ALGORITHM_ROTATING_ZERO_RESTART:
1768 /* Exactly the same as RIGHT_ASYMMETRIC, but or
1769 * of blocks for computing Q is different.
1770 */
1771 pd_idx = stripe % raid_disks;
1772 qd_idx = pd_idx + 1;
1773 if (pd_idx == raid_disks-1) {
1774 (*dd_idx)++; /* Q D D D P */
1775 qd_idx = 0;
1776 } else if (*dd_idx >= pd_idx)
1777 (*dd_idx) += 2; /* D D P Q D */
1778 ddf_layout = 1;
1779 break;
1780
1781 case ALGORITHM_ROTATING_N_RESTART:
1782 /* Same a left_asymmetric, by first stripe is
1783 * D D D P Q rather than
1784 * Q D D D P
1785 */
1786 pd_idx = raid_disks - 1 - ((stripe + 1) % raid_disks);
1787 qd_idx = pd_idx + 1;
1788 if (pd_idx == raid_disks-1) {
1789 (*dd_idx)++; /* Q D D D P */
1790 qd_idx = 0;
1791 } else if (*dd_idx >= pd_idx)
1792 (*dd_idx) += 2; /* D D P Q D */
1793 ddf_layout = 1;
1794 break;
1795
1796 case ALGORITHM_ROTATING_N_CONTINUE:
1797 /* Same as left_symmetric but Q is before P */
1798 pd_idx = raid_disks - 1 - (stripe % raid_disks);
1799 qd_idx = (pd_idx + raid_disks - 1) % raid_disks;
1800 *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
1801 ddf_layout = 1;
1802 break;
1803
1804 case ALGORITHM_LEFT_ASYMMETRIC_6:
1805 /* RAID5 left_asymmetric, with Q on last device */
1806 pd_idx = data_disks - stripe % (raid_disks-1);
1807 if (*dd_idx >= pd_idx)
1808 (*dd_idx)++;
1809 qd_idx = raid_disks - 1;
1810 break;
1811
1812 case ALGORITHM_RIGHT_ASYMMETRIC_6:
1813 pd_idx = stripe % (raid_disks-1);
1814 if (*dd_idx >= pd_idx)
1815 (*dd_idx)++;
1816 qd_idx = raid_disks - 1;
1817 break;
1818
1819 case ALGORITHM_LEFT_SYMMETRIC_6:
1820 pd_idx = data_disks - stripe % (raid_disks-1);
1821 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
1822 qd_idx = raid_disks - 1;
1823 break;
1824
1825 case ALGORITHM_RIGHT_SYMMETRIC_6:
1826 pd_idx = stripe % (raid_disks-1);
1827 *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
1828 qd_idx = raid_disks - 1;
1829 break;
1830
1831 case ALGORITHM_PARITY_0_6:
1832 pd_idx = 0;
1833 (*dd_idx)++;
1834 qd_idx = raid_disks - 1;
1835 break;
1836
1837
1838 default:
1839 printk(KERN_CRIT "raid6: unsupported algorithm %d\n",
1840 algorithm);
1841 BUG();
1842 }
1843 break;
1844 }
1845
1846 if (sh) {
1847 sh->pd_idx = pd_idx;
1848 sh->qd_idx = qd_idx;
1849 sh->ddf_layout = ddf_layout;
1850 }
1851 /*
1852 * Finally, compute the new sector number
1853 */
1854 new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset;
1855 return new_sector;
1856}
1857
1858
1859static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous)
1860{
1861 raid5_conf_t *conf = sh->raid_conf;
1862 int raid_disks = sh->disks;
1863 int data_disks = raid_disks - conf->max_degraded;
1864 sector_t new_sector = sh->sector, check;
1865 int sectors_per_chunk = previous ? conf->prev_chunk_sectors
1866 : conf->chunk_sectors;
1867 int algorithm = previous ? conf->prev_algo
1868 : conf->algorithm;
1869 sector_t stripe;
1870 int chunk_offset;
1871 int chunk_number, dummy1, dd_idx = i;
1872 sector_t r_sector;
1873 struct stripe_head sh2;
1874
1875
1876 chunk_offset = sector_div(new_sector, sectors_per_chunk);
1877 stripe = new_sector;
1878 BUG_ON(new_sector != stripe);
1879
1880 if (i == sh->pd_idx)
1881 return 0;
1882 switch(conf->level) {
1883 case 4: break;
1884 case 5:
1885 switch (algorithm) {
1886 case ALGORITHM_LEFT_ASYMMETRIC:
1887 case ALGORITHM_RIGHT_ASYMMETRIC:
1888 if (i > sh->pd_idx)
1889 i--;
1890 break;
1891 case ALGORITHM_LEFT_SYMMETRIC:
1892 case ALGORITHM_RIGHT_SYMMETRIC:
1893 if (i < sh->pd_idx)
1894 i += raid_disks;
1895 i -= (sh->pd_idx + 1);
1896 break;
1897 case ALGORITHM_PARITY_0:
1898 i -= 1;
1899 break;
1900 case ALGORITHM_PARITY_N:
1901 break;
1902 default:
1903 printk(KERN_ERR "raid5: unsupported algorithm %d\n",
1904 algorithm);
1905 BUG();
1906 }
1907 break;
1908 case 6:
1909 if (i == sh->qd_idx)
1910 return 0; /* It is the Q disk */
1911 switch (algorithm) {
1912 case ALGORITHM_LEFT_ASYMMETRIC:
1913 case ALGORITHM_RIGHT_ASYMMETRIC:
1914 case ALGORITHM_ROTATING_ZERO_RESTART:
1915 case ALGORITHM_ROTATING_N_RESTART:
1916 if (sh->pd_idx == raid_disks-1)
1917 i--; /* Q D D D P */
1918 else if (i > sh->pd_idx)
1919 i -= 2; /* D D P Q D */
1920 break;
1921 case ALGORITHM_LEFT_SYMMETRIC:
1922 case ALGORITHM_RIGHT_SYMMETRIC:
1923 if (sh->pd_idx == raid_disks-1)
1924 i--; /* Q D D D P */
1925 else {
1926 /* D D P Q D */
1927 if (i < sh->pd_idx)
1928 i += raid_disks;
1929 i -= (sh->pd_idx + 2);
1930 }
1931 break;
1932 case ALGORITHM_PARITY_0:
1933 i -= 2;
1934 break;
1935 case ALGORITHM_PARITY_N:
1936 break;
1937 case ALGORITHM_ROTATING_N_CONTINUE:
1938 if (sh->pd_idx == 0)
1939 i--; /* P D D D Q */
1940 else if (i > sh->pd_idx)
1941 i -= 2; /* D D Q P D */
1942 break;
1943 case ALGORITHM_LEFT_ASYMMETRIC_6:
1944 case ALGORITHM_RIGHT_ASYMMETRIC_6:
1945 if (i > sh->pd_idx)
1946 i--;
1947 break;
1948 case ALGORITHM_LEFT_SYMMETRIC_6:
1949 case ALGORITHM_RIGHT_SYMMETRIC_6:
1950 if (i < sh->pd_idx)
1951 i += data_disks + 1;
1952 i -= (sh->pd_idx + 1);
1953 break;
1954 case ALGORITHM_PARITY_0_6:
1955 i -= 1;
1956 break;
1957 default:
1958 printk(KERN_CRIT "raid6: unsupported algorithm %d\n",
1959 algorithm);
1960 BUG();
1961 }
1962 break;
1963 }
1964
1965 chunk_number = stripe * data_disks + i;
1966 r_sector = (sector_t)chunk_number * sectors_per_chunk + chunk_offset;
1967
1968 check = raid5_compute_sector(conf, r_sector,
1969 previous, &dummy1, &sh2);
1970 if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx
1971 || sh2.qd_idx != sh->qd_idx) {
1972 printk(KERN_ERR "compute_blocknr: map not correct\n");
1973 return 0;
1974 }
1975 return r_sector;
1976}
1977
1978
1979static void
1980schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
1981 int rcw, int expand)
1982{
1983 int i, pd_idx = sh->pd_idx, disks = sh->disks;
1984 raid5_conf_t *conf = sh->raid_conf;
1985 int level = conf->level;
1986
1987 if (rcw) {
1988 /* if we are not expanding this is a proper write request, and
1989 * there will be bios with new data to be drained into the
1990 * stripe cache
1991 */
1992 if (!expand) {
1993 sh->reconstruct_state = reconstruct_state_drain_run;
1994 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
1995 } else
1996 sh->reconstruct_state = reconstruct_state_run;
1997
1998 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
1999
2000 for (i = disks; i--; ) {
2001 struct r5dev *dev = &sh->dev[i];
2002
2003 if (dev->towrite) {
2004 set_bit(R5_LOCKED, &dev->flags);
2005 set_bit(R5_Wantdrain, &dev->flags);
2006 if (!expand)
2007 clear_bit(R5_UPTODATE, &dev->flags);
2008 s->locked++;
2009 }
2010 }
2011 if (s->locked + conf->max_degraded == disks)
2012 if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))
2013 atomic_inc(&conf->pending_full_writes);
2014 } else {
2015 BUG_ON(level == 6);
2016 BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||
2017 test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));
2018
2019 sh->reconstruct_state = reconstruct_state_prexor_drain_run;
2020 set_bit(STRIPE_OP_PREXOR, &s->ops_request);
2021 set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
2022 set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
2023
2024 for (i = disks; i--; ) {
2025 struct r5dev *dev = &sh->dev[i];
2026 if (i == pd_idx)
2027 continue;
2028
2029 if (dev->towrite &&
2030 (test_bit(R5_UPTODATE, &dev->flags) ||
2031 test_bit(R5_Wantcompute, &dev->flags))) {
2032 set_bit(R5_Wantdrain, &dev->flags);
2033 set_bit(R5_LOCKED, &dev->flags);
2034 clear_bit(R5_UPTODATE, &dev->flags);
2035 s->locked++;
2036 }
2037 }
2038 }
2039
2040 /* keep the parity disk(s) locked while asynchronous operations
2041 * are in flight
2042 */
2043 set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);
2044 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
2045 s->locked++;
2046
2047 if (level == 6) {
2048 int qd_idx = sh->qd_idx;
2049 struct r5dev *dev = &sh->dev[qd_idx];
2050
2051 set_bit(R5_LOCKED, &dev->flags);
2052 clear_bit(R5_UPTODATE, &dev->flags);
2053 s->locked++;
2054 }
2055
2056 pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n",
2057 __func__, (unsigned long long)sh->sector,
2058 s->locked, s->ops_request);
2059}
2060
2061/*
2062 * Each stripe/dev can have one or more bion attached.
2063 * toread/towrite point to the first in a chain.
2064 * The bi_next chain must be in order.
2065 */
2066static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, int forwrite)
2067{
2068 struct bio **bip;
2069 raid5_conf_t *conf = sh->raid_conf;
2070 int firstwrite=0;
2071
2072 pr_debug("adding bh b#%llu to stripe s#%llu\n",
2073 (unsigned long long)bi->bi_sector,
2074 (unsigned long long)sh->sector);
2075
2076
2077 spin_lock(&sh->lock);
2078 spin_lock_irq(&conf->device_lock);
2079 if (forwrite) {
2080 bip = &sh->dev[dd_idx].towrite;
2081 if (*bip == NULL && sh->dev[dd_idx].written == NULL)
2082 firstwrite = 1;
2083 } else
2084 bip = &sh->dev[dd_idx].toread;
2085 while (*bip && (*bip)->bi_sector < bi->bi_sector) {
2086 if ((*bip)->bi_sector + ((*bip)->bi_size >> 9) > bi->bi_sector)
2087 goto overlap;
2088 bip = & (*bip)->bi_next;
2089 }
2090 if (*bip && (*bip)->bi_sector < bi->bi_sector + ((bi->bi_size)>>9))
2091 goto overlap;
2092
2093 BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next);
2094 if (*bip)
2095 bi->bi_next = *bip;
2096 *bip = bi;
2097 bi->bi_phys_segments++;
2098 spin_unlock_irq(&conf->device_lock);
2099 spin_unlock(&sh->lock);
2100
2101 pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n",
2102 (unsigned long long)bi->bi_sector,
2103 (unsigned long long)sh->sector, dd_idx);
2104
2105 if (conf->mddev->bitmap && firstwrite) {
2106 bitmap_startwrite(conf->mddev->bitmap, sh->sector,
2107 STRIPE_SECTORS, 0);
2108 sh->bm_seq = conf->seq_flush+1;
2109 set_bit(STRIPE_BIT_DELAY, &sh->state);
2110 }
2111
2112 if (forwrite) {
2113 /* check if page is covered */
2114 sector_t sector = sh->dev[dd_idx].sector;
2115 for (bi=sh->dev[dd_idx].towrite;
2116 sector < sh->dev[dd_idx].sector + STRIPE_SECTORS &&
2117 bi && bi->bi_sector <= sector;
2118 bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) {
2119 if (bi->bi_sector + (bi->bi_size>>9) >= sector)
2120 sector = bi->bi_sector + (bi->bi_size>>9);
2121 }
2122 if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS)
2123 set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags);
2124 }
2125 return 1;
2126
2127 overlap:
2128 set_bit(R5_Overlap, &sh->dev[dd_idx].flags);
2129 spin_unlock_irq(&conf->device_lock);
2130 spin_unlock(&sh->lock);
2131 return 0;
2132}
2133
2134static void end_reshape(raid5_conf_t *conf);
2135
2136static void stripe_set_idx(sector_t stripe, raid5_conf_t *conf, int previous,
2137 struct stripe_head *sh)
2138{
2139 int sectors_per_chunk =
2140 previous ? conf->prev_chunk_sectors : conf->chunk_sectors;
2141 int dd_idx;
2142 int chunk_offset = sector_div(stripe, sectors_per_chunk);
2143 int disks = previous ? conf->previous_raid_disks : conf->raid_disks;
2144
2145 raid5_compute_sector(conf,
2146 stripe * (disks - conf->max_degraded)
2147 *sectors_per_chunk + chunk_offset,
2148 previous,
2149 &dd_idx, sh);
2150}
2151
2152static void
2153handle_failed_stripe(raid5_conf_t *conf, struct stripe_head *sh,
2154 struct stripe_head_state *s, int disks,
2155 struct bio **return_bi)
2156{
2157 int i;
2158 for (i = disks; i--; ) {
2159 struct bio *bi;
2160 int bitmap_end = 0;
2161
2162 if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
2163 mdk_rdev_t *rdev;
2164 rcu_read_lock();
2165 rdev = rcu_dereference(conf->disks[i].rdev);
2166 if (rdev && test_bit(In_sync, &rdev->flags))
2167 /* multiple read failures in one stripe */
2168 md_error(conf->mddev, rdev);
2169 rcu_read_unlock();
2170 }
2171 spin_lock_irq(&conf->device_lock);
2172 /* fail all writes first */
2173 bi = sh->dev[i].towrite;
2174 sh->dev[i].towrite = NULL;
2175 if (bi) {
2176 s->to_write--;
2177 bitmap_end = 1;
2178 }
2179
2180 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
2181 wake_up(&conf->wait_for_overlap);
2182
2183 while (bi && bi->bi_sector <
2184 sh->dev[i].sector + STRIPE_SECTORS) {
2185 struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
2186 clear_bit(BIO_UPTODATE, &bi->bi_flags);
2187 if (!raid5_dec_bi_phys_segments(bi)) {
2188 md_write_end(conf->mddev);
2189 bi->bi_next = *return_bi;
2190 *return_bi = bi;
2191 }
2192 bi = nextbi;
2193 }
2194 /* and fail all 'written' */
2195 bi = sh->dev[i].written;
2196 sh->dev[i].written = NULL;
2197 if (bi) bitmap_end = 1;
2198 while (bi && bi->bi_sector <
2199 sh->dev[i].sector + STRIPE_SECTORS) {
2200 struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector);
2201 clear_bit(BIO_UPTODATE, &bi->bi_flags);
2202 if (!raid5_dec_bi_phys_segments(bi)) {
2203 md_write_end(conf->mddev);
2204 bi->bi_next = *return_bi;
2205 *return_bi = bi;
2206 }
2207 bi = bi2;
2208 }
2209
2210 /* fail any reads if this device is non-operational and
2211 * the data has not reached the cache yet.
2212 */
2213 if (!test_bit(R5_Wantfill, &sh->dev[i].flags) &&
2214 (!test_bit(R5_Insync, &sh->dev[i].flags) ||
2215 test_bit(R5_ReadError, &sh->dev[i].flags))) {
2216 bi = sh->dev[i].toread;
2217 sh->dev[i].toread = NULL;
2218 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
2219 wake_up(&conf->wait_for_overlap);
2220 if (bi) s->to_read--;
2221 while (bi && bi->bi_sector <
2222 sh->dev[i].sector + STRIPE_SECTORS) {
2223 struct bio *nextbi =
2224 r5_next_bio(bi, sh->dev[i].sector);
2225 clear_bit(BIO_UPTODATE, &bi->bi_flags);
2226 if (!raid5_dec_bi_phys_segments(bi)) {
2227 bi->bi_next = *return_bi;
2228 *return_bi = bi;
2229 }
2230 bi = nextbi;
2231 }
2232 }
2233 spin_unlock_irq(&conf->device_lock);
2234 if (bitmap_end)
2235 bitmap_endwrite(conf->mddev->bitmap, sh->sector,
2236 STRIPE_SECTORS, 0, 0);
2237 }
2238
2239 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
2240 if (atomic_dec_and_test(&conf->pending_full_writes))
2241 md_wakeup_thread(conf->mddev->thread);
2242}
2243
2244/* fetch_block5 - checks the given member device to see if its data needs
2245 * to be read or computed to satisfy a request.
2246 *
2247 * Returns 1 when no more member devices need to be checked, otherwise returns
2248 * 0 to tell the loop in handle_stripe_fill5 to continue
2249 */
2250static int fetch_block5(struct stripe_head *sh, struct stripe_head_state *s,
2251 int disk_idx, int disks)
2252{
2253 struct r5dev *dev = &sh->dev[disk_idx];
2254 struct r5dev *failed_dev = &sh->dev[s->failed_num];
2255
2256 /* is the data in this block needed, and can we get it? */
2257 if (!test_bit(R5_LOCKED, &dev->flags) &&
2258 !test_bit(R5_UPTODATE, &dev->flags) &&
2259 (dev->toread ||
2260 (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) ||
2261 s->syncing || s->expanding ||
2262 (s->failed &&
2263 (failed_dev->toread ||
2264 (failed_dev->towrite &&
2265 !test_bit(R5_OVERWRITE, &failed_dev->flags)))))) {
2266 /* We would like to get this block, possibly by computing it,
2267 * otherwise read it if the backing disk is insync
2268 */
2269 if ((s->uptodate == disks - 1) &&
2270 (s->failed && disk_idx == s->failed_num)) {
2271 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
2272 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
2273 set_bit(R5_Wantcompute, &dev->flags);
2274 sh->ops.target = disk_idx;
2275 sh->ops.target2 = -1;
2276 s->req_compute = 1;
2277 /* Careful: from this point on 'uptodate' is in the eye
2278 * of raid_run_ops which services 'compute' operations
2279 * before writes. R5_Wantcompute flags a block that will
2280 * be R5_UPTODATE by the time it is needed for a
2281 * subsequent operation.
2282 */
2283 s->uptodate++;
2284 return 1; /* uptodate + compute == disks */
2285 } else if (test_bit(R5_Insync, &dev->flags)) {
2286 set_bit(R5_LOCKED, &dev->flags);
2287 set_bit(R5_Wantread, &dev->flags);
2288 s->locked++;
2289 pr_debug("Reading block %d (sync=%d)\n", disk_idx,
2290 s->syncing);
2291 }
2292 }
2293
2294 return 0;
2295}
2296
2297/**
2298 * handle_stripe_fill5 - read or compute data to satisfy pending requests.
2299 */
2300static void handle_stripe_fill5(struct stripe_head *sh,
2301 struct stripe_head_state *s, int disks)
2302{
2303 int i;
2304
2305 /* look for blocks to read/compute, skip this if a compute
2306 * is already in flight, or if the stripe contents are in the
2307 * midst of changing due to a write
2308 */
2309 if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&
2310 !sh->reconstruct_state)
2311 for (i = disks; i--; )
2312 if (fetch_block5(sh, s, i, disks))
2313 break;
2314 set_bit(STRIPE_HANDLE, &sh->state);
2315}
2316
2317/* fetch_block6 - checks the given member device to see if its data needs
2318 * to be read or computed to satisfy a request.
2319 *
2320 * Returns 1 when no more member devices need to be checked, otherwise returns
2321 * 0 to tell the loop in handle_stripe_fill6 to continue
2322 */
2323static int fetch_block6(struct stripe_head *sh, struct stripe_head_state *s,
2324 struct r6_state *r6s, int disk_idx, int disks)
2325{
2326 struct r5dev *dev = &sh->dev[disk_idx];
2327 struct r5dev *fdev[2] = { &sh->dev[r6s->failed_num[0]],
2328 &sh->dev[r6s->failed_num[1]] };
2329
2330 if (!test_bit(R5_LOCKED, &dev->flags) &&
2331 !test_bit(R5_UPTODATE, &dev->flags) &&
2332 (dev->toread ||
2333 (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) ||
2334 s->syncing || s->expanding ||
2335 (s->failed >= 1 &&
2336 (fdev[0]->toread || s->to_write)) ||
2337 (s->failed >= 2 &&
2338 (fdev[1]->toread || s->to_write)))) {
2339 /* we would like to get this block, possibly by computing it,
2340 * otherwise read it if the backing disk is insync
2341 */
2342 BUG_ON(test_bit(R5_Wantcompute, &dev->flags));
2343 BUG_ON(test_bit(R5_Wantread, &dev->flags));
2344 if ((s->uptodate == disks - 1) &&
2345 (s->failed && (disk_idx == r6s->failed_num[0] ||
2346 disk_idx == r6s->failed_num[1]))) {
2347 /* have disk failed, and we're requested to fetch it;
2348 * do compute it
2349 */
2350 pr_debug("Computing stripe %llu block %d\n",
2351 (unsigned long long)sh->sector, disk_idx);
2352 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
2353 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
2354 set_bit(R5_Wantcompute, &dev->flags);
2355 sh->ops.target = disk_idx;
2356 sh->ops.target2 = -1; /* no 2nd target */
2357 s->req_compute = 1;
2358 s->uptodate++;
2359 return 1;
2360 } else if (s->uptodate == disks-2 && s->failed >= 2) {
2361 /* Computing 2-failure is *very* expensive; only
2362 * do it if failed >= 2
2363 */
2364 int other;
2365 for (other = disks; other--; ) {
2366 if (other == disk_idx)
2367 continue;
2368 if (!test_bit(R5_UPTODATE,
2369 &sh->dev[other].flags))
2370 break;
2371 }
2372 BUG_ON(other < 0);
2373 pr_debug("Computing stripe %llu blocks %d,%d\n",
2374 (unsigned long long)sh->sector,
2375 disk_idx, other);
2376 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
2377 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
2378 set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);
2379 set_bit(R5_Wantcompute, &sh->dev[other].flags);
2380 sh->ops.target = disk_idx;
2381 sh->ops.target2 = other;
2382 s->uptodate += 2;
2383 s->req_compute = 1;
2384 return 1;
2385 } else if (test_bit(R5_Insync, &dev->flags)) {
2386 set_bit(R5_LOCKED, &dev->flags);
2387 set_bit(R5_Wantread, &dev->flags);
2388 s->locked++;
2389 pr_debug("Reading block %d (sync=%d)\n",
2390 disk_idx, s->syncing);
2391 }
2392 }
2393
2394 return 0;
2395}
2396
2397/**
2398 * handle_stripe_fill6 - read or compute data to satisfy pending requests.
2399 */
2400static void handle_stripe_fill6(struct stripe_head *sh,
2401 struct stripe_head_state *s, struct r6_state *r6s,
2402 int disks)
2403{
2404 int i;
2405
2406 /* look for blocks to read/compute, skip this if a compute
2407 * is already in flight, or if the stripe contents are in the
2408 * midst of changing due to a write
2409 */
2410 if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&
2411 !sh->reconstruct_state)
2412 for (i = disks; i--; )
2413 if (fetch_block6(sh, s, r6s, i, disks))
2414 break;
2415 set_bit(STRIPE_HANDLE, &sh->state);
2416}
2417
2418
2419/* handle_stripe_clean_event
2420 * any written block on an uptodate or failed drive can be returned.
2421 * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but
2422 * never LOCKED, so we don't need to test 'failed' directly.
2423 */
2424static void handle_stripe_clean_event(raid5_conf_t *conf,
2425 struct stripe_head *sh, int disks, struct bio **return_bi)
2426{
2427 int i;
2428 struct r5dev *dev;
2429
2430 for (i = disks; i--; )
2431 if (sh->dev[i].written) {
2432 dev = &sh->dev[i];
2433 if (!test_bit(R5_LOCKED, &dev->flags) &&
2434 test_bit(R5_UPTODATE, &dev->flags)) {
2435 /* We can return any write requests */
2436 struct bio *wbi, *wbi2;
2437 int bitmap_end = 0;
2438 pr_debug("Return write for disc %d\n", i);
2439 spin_lock_irq(&conf->device_lock);
2440 wbi = dev->written;
2441 dev->written = NULL;
2442 while (wbi && wbi->bi_sector <
2443 dev->sector + STRIPE_SECTORS) {
2444 wbi2 = r5_next_bio(wbi, dev->sector);
2445 if (!raid5_dec_bi_phys_segments(wbi)) {
2446 md_write_end(conf->mddev);
2447 wbi->bi_next = *return_bi;
2448 *return_bi = wbi;
2449 }
2450 wbi = wbi2;
2451 }
2452 if (dev->towrite == NULL)
2453 bitmap_end = 1;
2454 spin_unlock_irq(&conf->device_lock);
2455 if (bitmap_end)
2456 bitmap_endwrite(conf->mddev->bitmap,
2457 sh->sector,
2458 STRIPE_SECTORS,
2459 !test_bit(STRIPE_DEGRADED, &sh->state),
2460 0);
2461 }
2462 }
2463
2464 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
2465 if (atomic_dec_and_test(&conf->pending_full_writes))
2466 md_wakeup_thread(conf->mddev->thread);
2467}
2468
2469static void handle_stripe_dirtying5(raid5_conf_t *conf,
2470 struct stripe_head *sh, struct stripe_head_state *s, int disks)
2471{
2472 int rmw = 0, rcw = 0, i;
2473 for (i = disks; i--; ) {
2474 /* would I have to read this buffer for read_modify_write */
2475 struct r5dev *dev = &sh->dev[i];
2476 if ((dev->towrite || i == sh->pd_idx) &&
2477 !test_bit(R5_LOCKED, &dev->flags) &&
2478 !(test_bit(R5_UPTODATE, &dev->flags) ||
2479 test_bit(R5_Wantcompute, &dev->flags))) {
2480 if (test_bit(R5_Insync, &dev->flags))
2481 rmw++;
2482 else
2483 rmw += 2*disks; /* cannot read it */
2484 }
2485 /* Would I have to read this buffer for reconstruct_write */
2486 if (!test_bit(R5_OVERWRITE, &dev->flags) && i != sh->pd_idx &&
2487 !test_bit(R5_LOCKED, &dev->flags) &&
2488 !(test_bit(R5_UPTODATE, &dev->flags) ||
2489 test_bit(R5_Wantcompute, &dev->flags))) {
2490 if (test_bit(R5_Insync, &dev->flags)) rcw++;
2491 else
2492 rcw += 2*disks;
2493 }
2494 }
2495 pr_debug("for sector %llu, rmw=%d rcw=%d\n",
2496 (unsigned long long)sh->sector, rmw, rcw);
2497 set_bit(STRIPE_HANDLE, &sh->state);
2498 if (rmw < rcw && rmw > 0)
2499 /* prefer read-modify-write, but need to get some data */
2500 for (i = disks; i--; ) {
2501 struct r5dev *dev = &sh->dev[i];
2502 if ((dev->towrite || i == sh->pd_idx) &&
2503 !test_bit(R5_LOCKED, &dev->flags) &&
2504 !(test_bit(R5_UPTODATE, &dev->flags) ||
2505 test_bit(R5_Wantcompute, &dev->flags)) &&
2506 test_bit(R5_Insync, &dev->flags)) {
2507 if (
2508 test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
2509 pr_debug("Read_old block "
2510 "%d for r-m-w\n", i);
2511 set_bit(R5_LOCKED, &dev->flags);
2512 set_bit(R5_Wantread, &dev->flags);
2513 s->locked++;
2514 } else {
2515 set_bit(STRIPE_DELAYED, &sh->state);
2516 set_bit(STRIPE_HANDLE, &sh->state);
2517 }
2518 }
2519 }
2520 if (rcw <= rmw && rcw > 0)
2521 /* want reconstruct write, but need to get some data */
2522 for (i = disks; i--; ) {
2523 struct r5dev *dev = &sh->dev[i];
2524 if (!test_bit(R5_OVERWRITE, &dev->flags) &&
2525 i != sh->pd_idx &&
2526 !test_bit(R5_LOCKED, &dev->flags) &&
2527 !(test_bit(R5_UPTODATE, &dev->flags) ||
2528 test_bit(R5_Wantcompute, &dev->flags)) &&
2529 test_bit(R5_Insync, &dev->flags)) {
2530 if (
2531 test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
2532 pr_debug("Read_old block "
2533 "%d for Reconstruct\n", i);
2534 set_bit(R5_LOCKED, &dev->flags);
2535 set_bit(R5_Wantread, &dev->flags);
2536 s->locked++;
2537 } else {
2538 set_bit(STRIPE_DELAYED, &sh->state);
2539 set_bit(STRIPE_HANDLE, &sh->state);
2540 }
2541 }
2542 }
2543 /* now if nothing is locked, and if we have enough data,
2544 * we can start a write request
2545 */
2546 /* since handle_stripe can be called at any time we need to handle the
2547 * case where a compute block operation has been submitted and then a
2548 * subsequent call wants to start a write request. raid_run_ops only
2549 * handles the case where compute block and reconstruct are requested
2550 * simultaneously. If this is not the case then new writes need to be
2551 * held off until the compute completes.
2552 */
2553 if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
2554 (s->locked == 0 && (rcw == 0 || rmw == 0) &&
2555 !test_bit(STRIPE_BIT_DELAY, &sh->state)))
2556 schedule_reconstruction(sh, s, rcw == 0, 0);
2557}
2558
2559static void handle_stripe_dirtying6(raid5_conf_t *conf,
2560 struct stripe_head *sh, struct stripe_head_state *s,
2561 struct r6_state *r6s, int disks)
2562{
2563 int rcw = 0, pd_idx = sh->pd_idx, i;
2564 int qd_idx = sh->qd_idx;
2565
2566 set_bit(STRIPE_HANDLE, &sh->state);
2567 for (i = disks; i--; ) {
2568 struct r5dev *dev = &sh->dev[i];
2569 /* check if we haven't enough data */
2570 if (!test_bit(R5_OVERWRITE, &dev->flags) &&
2571 i != pd_idx && i != qd_idx &&
2572 !test_bit(R5_LOCKED, &dev->flags) &&
2573 !(test_bit(R5_UPTODATE, &dev->flags) ||
2574 test_bit(R5_Wantcompute, &dev->flags))) {
2575 rcw++;
2576 if (!test_bit(R5_Insync, &dev->flags))
2577 continue; /* it's a failed drive */
2578
2579 if (
2580 test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
2581 pr_debug("Read_old stripe %llu "
2582 "block %d for Reconstruct\n",
2583 (unsigned long long)sh->sector, i);
2584 set_bit(R5_LOCKED, &dev->flags);
2585 set_bit(R5_Wantread, &dev->flags);
2586 s->locked++;
2587 } else {
2588 pr_debug("Request delayed stripe %llu "
2589 "block %d for Reconstruct\n",
2590 (unsigned long long)sh->sector, i);
2591 set_bit(STRIPE_DELAYED, &sh->state);
2592 set_bit(STRIPE_HANDLE, &sh->state);
2593 }
2594 }
2595 }
2596 /* now if nothing is locked, and if we have enough data, we can start a
2597 * write request
2598 */
2599 if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
2600 s->locked == 0 && rcw == 0 &&
2601 !test_bit(STRIPE_BIT_DELAY, &sh->state)) {
2602 schedule_reconstruction(sh, s, 1, 0);
2603 }
2604}
2605
2606static void handle_parity_checks5(raid5_conf_t *conf, struct stripe_head *sh,
2607 struct stripe_head_state *s, int disks)
2608{
2609 struct r5dev *dev = NULL;
2610
2611 set_bit(STRIPE_HANDLE, &sh->state);
2612
2613 switch (sh->check_state) {
2614 case check_state_idle:
2615 /* start a new check operation if there are no failures */
2616 if (s->failed == 0) {
2617 BUG_ON(s->uptodate != disks);
2618 sh->check_state = check_state_run;
2619 set_bit(STRIPE_OP_CHECK, &s->ops_request);
2620 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
2621 s->uptodate--;
2622 break;
2623 }
2624 dev = &sh->dev[s->failed_num];
2625 /* fall through */
2626 case check_state_compute_result:
2627 sh->check_state = check_state_idle;
2628 if (!dev)
2629 dev = &sh->dev[sh->pd_idx];
2630
2631 /* check that a write has not made the stripe insync */
2632 if (test_bit(STRIPE_INSYNC, &sh->state))
2633 break;
2634
2635 /* either failed parity check, or recovery is happening */
2636 BUG_ON(!test_bit(R5_UPTODATE, &dev->flags));
2637 BUG_ON(s->uptodate != disks);
2638
2639 set_bit(R5_LOCKED, &dev->flags);
2640 s->locked++;
2641 set_bit(R5_Wantwrite, &dev->flags);
2642
2643 clear_bit(STRIPE_DEGRADED, &sh->state);
2644 set_bit(STRIPE_INSYNC, &sh->state);
2645 break;
2646 case check_state_run:
2647 break; /* we will be called again upon completion */
2648 case check_state_check_result:
2649 sh->check_state = check_state_idle;
2650
2651 /* if a failure occurred during the check operation, leave
2652 * STRIPE_INSYNC not set and let the stripe be handled again
2653 */
2654 if (s->failed)
2655 break;
2656
2657 /* handle a successful check operation, if parity is correct
2658 * we are done. Otherwise update the mismatch count and repair
2659 * parity if !MD_RECOVERY_CHECK
2660 */
2661 if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)
2662 /* parity is correct (on disc,
2663 * not in buffer any more)
2664 */
2665 set_bit(STRIPE_INSYNC, &sh->state);
2666 else {
2667 conf->mddev->resync_mismatches += STRIPE_SECTORS;
2668 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
2669 /* don't try to repair!! */
2670 set_bit(STRIPE_INSYNC, &sh->state);
2671 else {
2672 sh->check_state = check_state_compute_run;
2673 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
2674 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
2675 set_bit(R5_Wantcompute,
2676 &sh->dev[sh->pd_idx].flags);
2677 sh->ops.target = sh->pd_idx;
2678 sh->ops.target2 = -1;
2679 s->uptodate++;
2680 }
2681 }
2682 break;
2683 case check_state_compute_run:
2684 break;
2685 default:
2686 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
2687 __func__, sh->check_state,
2688 (unsigned long long) sh->sector);
2689 BUG();
2690 }
2691}
2692
2693
2694static void handle_parity_checks6(raid5_conf_t *conf, struct stripe_head *sh,
2695 struct stripe_head_state *s,
2696 struct r6_state *r6s, int disks)
2697{
2698 int pd_idx = sh->pd_idx;
2699 int qd_idx = sh->qd_idx;
2700 struct r5dev *dev;
2701
2702 set_bit(STRIPE_HANDLE, &sh->state);
2703
2704 BUG_ON(s->failed > 2);
2705
2706 /* Want to check and possibly repair P and Q.
2707 * However there could be one 'failed' device, in which
2708 * case we can only check one of them, possibly using the
2709 * other to generate missing data
2710 */
2711
2712 switch (sh->check_state) {
2713 case check_state_idle:
2714 /* start a new check operation if there are < 2 failures */
2715 if (s->failed == r6s->q_failed) {
2716 /* The only possible failed device holds Q, so it
2717 * makes sense to check P (If anything else were failed,
2718 * we would have used P to recreate it).
2719 */
2720 sh->check_state = check_state_run;
2721 }
2722 if (!r6s->q_failed && s->failed < 2) {
2723 /* Q is not failed, and we didn't use it to generate
2724 * anything, so it makes sense to check it
2725 */
2726 if (sh->check_state == check_state_run)
2727 sh->check_state = check_state_run_pq;
2728 else
2729 sh->check_state = check_state_run_q;
2730 }
2731
2732 /* discard potentially stale zero_sum_result */
2733 sh->ops.zero_sum_result = 0;
2734
2735 if (sh->check_state == check_state_run) {
2736 /* async_xor_zero_sum destroys the contents of P */
2737 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
2738 s->uptodate--;
2739 }
2740 if (sh->check_state >= check_state_run &&
2741 sh->check_state <= check_state_run_pq) {
2742 /* async_syndrome_zero_sum preserves P and Q, so
2743 * no need to mark them !uptodate here
2744 */
2745 set_bit(STRIPE_OP_CHECK, &s->ops_request);
2746 break;
2747 }
2748
2749 /* we have 2-disk failure */
2750 BUG_ON(s->failed != 2);
2751 /* fall through */
2752 case check_state_compute_result:
2753 sh->check_state = check_state_idle;
2754
2755 /* check that a write has not made the stripe insync */
2756 if (test_bit(STRIPE_INSYNC, &sh->state))
2757 break;
2758
2759 /* now write out any block on a failed drive,
2760 * or P or Q if they were recomputed
2761 */
2762 BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */
2763 if (s->failed == 2) {
2764 dev = &sh->dev[r6s->failed_num[1]];
2765 s->locked++;
2766 set_bit(R5_LOCKED, &dev->flags);
2767 set_bit(R5_Wantwrite, &dev->flags);
2768 }
2769 if (s->failed >= 1) {
2770 dev = &sh->dev[r6s->failed_num[0]];
2771 s->locked++;
2772 set_bit(R5_LOCKED, &dev->flags);
2773 set_bit(R5_Wantwrite, &dev->flags);
2774 }
2775 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
2776 dev = &sh->dev[pd_idx];
2777 s->locked++;
2778 set_bit(R5_LOCKED, &dev->flags);
2779 set_bit(R5_Wantwrite, &dev->flags);
2780 }
2781 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
2782 dev = &sh->dev[qd_idx];
2783 s->locked++;
2784 set_bit(R5_LOCKED, &dev->flags);
2785 set_bit(R5_Wantwrite, &dev->flags);
2786 }
2787 clear_bit(STRIPE_DEGRADED, &sh->state);
2788
2789 set_bit(STRIPE_INSYNC, &sh->state);
2790 break;
2791 case check_state_run:
2792 case check_state_run_q:
2793 case check_state_run_pq:
2794 break; /* we will be called again upon completion */
2795 case check_state_check_result:
2796 sh->check_state = check_state_idle;
2797
2798 /* handle a successful check operation, if parity is correct
2799 * we are done. Otherwise update the mismatch count and repair
2800 * parity if !MD_RECOVERY_CHECK
2801 */
2802 if (sh->ops.zero_sum_result == 0) {
2803 /* both parities are correct */
2804 if (!s->failed)
2805 set_bit(STRIPE_INSYNC, &sh->state);
2806 else {
2807 /* in contrast to the raid5 case we can validate
2808 * parity, but still have a failure to write
2809 * back
2810 */
2811 sh->check_state = check_state_compute_result;
2812 /* Returning at this point means that we may go
2813 * off and bring p and/or q uptodate again so
2814 * we make sure to check zero_sum_result again
2815 * to verify if p or q need writeback
2816 */
2817 }
2818 } else {
2819 conf->mddev->resync_mismatches += STRIPE_SECTORS;
2820 if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
2821 /* don't try to repair!! */
2822 set_bit(STRIPE_INSYNC, &sh->state);
2823 else {
2824 int *target = &sh->ops.target;
2825
2826 sh->ops.target = -1;
2827 sh->ops.target2 = -1;
2828 sh->check_state = check_state_compute_run;
2829 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
2830 set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
2831 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
2832 set_bit(R5_Wantcompute,
2833 &sh->dev[pd_idx].flags);
2834 *target = pd_idx;
2835 target = &sh->ops.target2;
2836 s->uptodate++;
2837 }
2838 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
2839 set_bit(R5_Wantcompute,
2840 &sh->dev[qd_idx].flags);
2841 *target = qd_idx;
2842 s->uptodate++;
2843 }
2844 }
2845 }
2846 break;
2847 case check_state_compute_run:
2848 break;
2849 default:
2850 printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
2851 __func__, sh->check_state,
2852 (unsigned long long) sh->sector);
2853 BUG();
2854 }
2855}
2856
2857static void handle_stripe_expansion(raid5_conf_t *conf, struct stripe_head *sh,
2858 struct r6_state *r6s)
2859{
2860 int i;
2861
2862 /* We have read all the blocks in this stripe and now we need to
2863 * copy some of them into a target stripe for expand.
2864 */
2865 struct dma_async_tx_descriptor *tx = NULL;
2866 clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);
2867 for (i = 0; i < sh->disks; i++)
2868 if (i != sh->pd_idx && i != sh->qd_idx) {
2869 int dd_idx, j;
2870 struct stripe_head *sh2;
2871 struct async_submit_ctl submit;
2872
2873 sector_t bn = compute_blocknr(sh, i, 1);
2874 sector_t s = raid5_compute_sector(conf, bn, 0,
2875 &dd_idx, NULL);
2876 sh2 = get_active_stripe(conf, s, 0, 1, 1);
2877 if (sh2 == NULL)
2878 /* so far only the early blocks of this stripe
2879 * have been requested. When later blocks
2880 * get requested, we will try again
2881 */
2882 continue;
2883 if (!test_bit(STRIPE_EXPANDING, &sh2->state) ||
2884 test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) {
2885 /* must have already done this block */
2886 release_stripe(sh2);
2887 continue;
2888 }
2889
2890 /* place all the copies on one channel */
2891 init_async_submit(&submit, 0, tx, NULL, NULL, NULL);
2892 tx = async_memcpy(sh2->dev[dd_idx].page,
2893 sh->dev[i].page, 0, 0, STRIPE_SIZE,
2894 &submit);
2895
2896 set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);
2897 set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags);
2898 for (j = 0; j < conf->raid_disks; j++)
2899 if (j != sh2->pd_idx &&
2900 (!r6s || j != sh2->qd_idx) &&
2901 !test_bit(R5_Expanded, &sh2->dev[j].flags))
2902 break;
2903 if (j == conf->raid_disks) {
2904 set_bit(STRIPE_EXPAND_READY, &sh2->state);
2905 set_bit(STRIPE_HANDLE, &sh2->state);
2906 }
2907 release_stripe(sh2);
2908
2909 }
2910 /* done submitting copies, wait for them to complete */
2911 if (tx) {
2912 async_tx_ack(tx);
2913 dma_wait_for_async_tx(tx);
2914 }
2915}
2916
2917
2918/*
2919 * handle_stripe - do things to a stripe.
2920 *
2921 * We lock the stripe and then examine the state of various bits
2922 * to see what needs to be done.
2923 * Possible results:
2924 * return some read request which now have data
2925 * return some write requests which are safely on disc
2926 * schedule a read on some buffers
2927 * schedule a write of some buffers
2928 * return confirmation of parity correctness
2929 *
2930 * buffers are taken off read_list or write_list, and bh_cache buffers
2931 * get BH_Lock set before the stripe lock is released.
2932 *
2933 */
2934
2935static void handle_stripe5(struct stripe_head *sh)
2936{
2937 raid5_conf_t *conf = sh->raid_conf;
2938 int disks = sh->disks, i;
2939 struct bio *return_bi = NULL;
2940 struct stripe_head_state s;
2941 struct r5dev *dev;
2942 mdk_rdev_t *blocked_rdev = NULL;
2943 int prexor;
2944
2945 memset(&s, 0, sizeof(s));
2946 pr_debug("handling stripe %llu, state=%#lx cnt=%d, pd_idx=%d check:%d "
2947 "reconstruct:%d\n", (unsigned long long)sh->sector, sh->state,
2948 atomic_read(&sh->count), sh->pd_idx, sh->check_state,
2949 sh->reconstruct_state);
2950
2951 spin_lock(&sh->lock);
2952 clear_bit(STRIPE_HANDLE, &sh->state);
2953 clear_bit(STRIPE_DELAYED, &sh->state);
2954
2955 s.syncing = test_bit(STRIPE_SYNCING, &sh->state);
2956 s.expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state);
2957 s.expanded = test_bit(STRIPE_EXPAND_READY, &sh->state);
2958
2959 /* Now to look around and see what can be done */
2960 rcu_read_lock();
2961 for (i=disks; i--; ) {
2962 mdk_rdev_t *rdev;
2963
2964 dev = &sh->dev[i];
2965 clear_bit(R5_Insync, &dev->flags);
2966
2967 pr_debug("check %d: state 0x%lx toread %p read %p write %p "
2968 "written %p\n", i, dev->flags, dev->toread, dev->read,
2969 dev->towrite, dev->written);
2970
2971 /* maybe we can request a biofill operation
2972 *
2973 * new wantfill requests are only permitted while
2974 * ops_complete_biofill is guaranteed to be inactive
2975 */
2976 if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread &&
2977 !test_bit(STRIPE_BIOFILL_RUN, &sh->state))
2978 set_bit(R5_Wantfill, &dev->flags);
2979
2980 /* now count some things */
2981 if (test_bit(R5_LOCKED, &dev->flags)) s.locked++;
2982 if (test_bit(R5_UPTODATE, &dev->flags)) s.uptodate++;
2983 if (test_bit(R5_Wantcompute, &dev->flags)) s.compute++;
2984
2985 if (test_bit(R5_Wantfill, &dev->flags))
2986 s.to_fill++;
2987 else if (dev->toread)
2988 s.to_read++;
2989 if (dev->towrite) {
2990 s.to_write++;
2991 if (!test_bit(R5_OVERWRITE, &dev->flags))
2992 s.non_overwrite++;
2993 }
2994 if (dev->written)
2995 s.written++;
2996 rdev = rcu_dereference(conf->disks[i].rdev);
2997 if (blocked_rdev == NULL &&
2998 rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
2999 blocked_rdev = rdev;
3000 atomic_inc(&rdev->nr_pending);
3001 }
3002 if (!rdev || !test_bit(In_sync, &rdev->flags)) {
3003 /* The ReadError flag will just be confusing now */
3004 clear_bit(R5_ReadError, &dev->flags);
3005 clear_bit(R5_ReWrite, &dev->flags);
3006 }
3007 if (!rdev || !test_bit(In_sync, &rdev->flags)
3008 || test_bit(R5_ReadError, &dev->flags)) {
3009 s.failed++;
3010 s.failed_num = i;
3011 } else
3012 set_bit(R5_Insync, &dev->flags);
3013 }
3014 rcu_read_unlock();
3015
3016 if (unlikely(blocked_rdev)) {
3017 if (s.syncing || s.expanding || s.expanded ||
3018 s.to_write || s.written) {
3019 set_bit(STRIPE_HANDLE, &sh->state);
3020 goto unlock;
3021 }
3022 /* There is nothing for the blocked_rdev to block */
3023 rdev_dec_pending(blocked_rdev, conf->mddev);
3024 blocked_rdev = NULL;
3025 }
3026
3027 if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {
3028 set_bit(STRIPE_OP_BIOFILL, &s.ops_request);
3029 set_bit(STRIPE_BIOFILL_RUN, &sh->state);
3030 }
3031
3032 pr_debug("locked=%d uptodate=%d to_read=%d"
3033 " to_write=%d failed=%d failed_num=%d\n",
3034 s.locked, s.uptodate, s.to_read, s.to_write,
3035 s.failed, s.failed_num);
3036 /* check if the array has lost two devices and, if so, some requests might
3037 * need to be failed
3038 */
3039 if (s.failed > 1 && s.to_read+s.to_write+s.written)
3040 handle_failed_stripe(conf, sh, &s, disks, &return_bi);
3041 if (s.failed > 1 && s.syncing) {
3042 md_done_sync(conf->mddev, STRIPE_SECTORS,0);
3043 clear_bit(STRIPE_SYNCING, &sh->state);
3044 s.syncing = 0;
3045 }
3046
3047 /* might be able to return some write requests if the parity block
3048 * is safe, or on a failed drive
3049 */
3050 dev = &sh->dev[sh->pd_idx];
3051 if ( s.written &&
3052 ((test_bit(R5_Insync, &dev->flags) &&
3053 !test_bit(R5_LOCKED, &dev->flags) &&
3054 test_bit(R5_UPTODATE, &dev->flags)) ||
3055 (s.failed == 1 && s.failed_num == sh->pd_idx)))
3056 handle_stripe_clean_event(conf, sh, disks, &return_bi);
3057
3058 /* Now we might consider reading some blocks, either to check/generate
3059 * parity, or to satisfy requests
3060 * or to load a block that is being partially written.
3061 */
3062 if (s.to_read || s.non_overwrite ||
3063 (s.syncing && (s.uptodate + s.compute < disks)) || s.expanding)
3064 handle_stripe_fill5(sh, &s, disks);
3065
3066 /* Now we check to see if any write operations have recently
3067 * completed
3068 */
3069 prexor = 0;
3070 if (sh->reconstruct_state == reconstruct_state_prexor_drain_result)
3071 prexor = 1;
3072 if (sh->reconstruct_state == reconstruct_state_drain_result ||
3073 sh->reconstruct_state == reconstruct_state_prexor_drain_result) {
3074 sh->reconstruct_state = reconstruct_state_idle;
3075
3076 /* All the 'written' buffers and the parity block are ready to
3077 * be written back to disk
3078 */
3079 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags));
3080 for (i = disks; i--; ) {
3081 dev = &sh->dev[i];
3082 if (test_bit(R5_LOCKED, &dev->flags) &&
3083 (i == sh->pd_idx || dev->written)) {
3084 pr_debug("Writing block %d\n", i);
3085 set_bit(R5_Wantwrite, &dev->flags);
3086 if (prexor)
3087 continue;
3088 if (!test_bit(R5_Insync, &dev->flags) ||
3089 (i == sh->pd_idx && s.failed == 0))
3090 set_bit(STRIPE_INSYNC, &sh->state);
3091 }
3092 }
3093 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
3094 atomic_dec(&conf->preread_active_stripes);
3095 if (atomic_read(&conf->preread_active_stripes) <
3096 IO_THRESHOLD)
3097 md_wakeup_thread(conf->mddev->thread);
3098 }
3099 }
3100
3101 /* Now to consider new write requests and what else, if anything
3102 * should be read. We do not handle new writes when:
3103 * 1/ A 'write' operation (copy+xor) is already in flight.
3104 * 2/ A 'check' operation is in flight, as it may clobber the parity
3105 * block.
3106 */
3107 if (s.to_write && !sh->reconstruct_state && !sh->check_state)
3108 handle_stripe_dirtying5(conf, sh, &s, disks);
3109
3110 /* maybe we need to check and possibly fix the parity for this stripe
3111 * Any reads will already have been scheduled, so we just see if enough
3112 * data is available. The parity check is held off while parity
3113 * dependent operations are in flight.
3114 */
3115 if (sh->check_state ||
3116 (s.syncing && s.locked == 0 &&
3117 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
3118 !test_bit(STRIPE_INSYNC, &sh->state)))
3119 handle_parity_checks5(conf, sh, &s, disks);
3120
3121 if (s.syncing && s.locked == 0 && test_bit(STRIPE_INSYNC, &sh->state)) {
3122 md_done_sync(conf->mddev, STRIPE_SECTORS,1);
3123 clear_bit(STRIPE_SYNCING, &sh->state);
3124 }
3125
3126 /* If the failed drive is just a ReadError, then we might need to progress
3127 * the repair/check process
3128 */
3129 if (s.failed == 1 && !conf->mddev->ro &&
3130 test_bit(R5_ReadError, &sh->dev[s.failed_num].flags)
3131 && !test_bit(R5_LOCKED, &sh->dev[s.failed_num].flags)
3132 && test_bit(R5_UPTODATE, &sh->dev[s.failed_num].flags)
3133 ) {
3134 dev = &sh->dev[s.failed_num];
3135 if (!test_bit(R5_ReWrite, &dev->flags)) {
3136 set_bit(R5_Wantwrite, &dev->flags);
3137 set_bit(R5_ReWrite, &dev->flags);
3138 set_bit(R5_LOCKED, &dev->flags);
3139 s.locked++;
3140 } else {
3141 /* let's read it back */
3142 set_bit(R5_Wantread, &dev->flags);
3143 set_bit(R5_LOCKED, &dev->flags);
3144 s.locked++;
3145 }
3146 }
3147
3148 /* Finish reconstruct operations initiated by the expansion process */
3149 if (sh->reconstruct_state == reconstruct_state_result) {
3150 struct stripe_head *sh2
3151 = get_active_stripe(conf, sh->sector, 1, 1, 1);
3152 if (sh2 && test_bit(STRIPE_EXPAND_SOURCE, &sh2->state)) {
3153 /* sh cannot be written until sh2 has been read.
3154 * so arrange for sh to be delayed a little
3155 */
3156 set_bit(STRIPE_DELAYED, &sh->state);
3157 set_bit(STRIPE_HANDLE, &sh->state);
3158 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE,
3159 &sh2->state))
3160 atomic_inc(&conf->preread_active_stripes);
3161 release_stripe(sh2);
3162 goto unlock;
3163 }
3164 if (sh2)
3165 release_stripe(sh2);
3166
3167 sh->reconstruct_state = reconstruct_state_idle;
3168 clear_bit(STRIPE_EXPANDING, &sh->state);
3169 for (i = conf->raid_disks; i--; ) {
3170 set_bit(R5_Wantwrite, &sh->dev[i].flags);
3171 set_bit(R5_LOCKED, &sh->dev[i].flags);
3172 s.locked++;
3173 }
3174 }
3175
3176 if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&
3177 !sh->reconstruct_state) {
3178 /* Need to write out all blocks after computing parity */
3179 sh->disks = conf->raid_disks;
3180 stripe_set_idx(sh->sector, conf, 0, sh);
3181 schedule_reconstruction(sh, &s, 1, 1);
3182 } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
3183 clear_bit(STRIPE_EXPAND_READY, &sh->state);
3184 atomic_dec(&conf->reshape_stripes);
3185 wake_up(&conf->wait_for_overlap);
3186 md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
3187 }
3188
3189 if (s.expanding && s.locked == 0 &&
3190 !test_bit(STRIPE_COMPUTE_RUN, &sh->state))
3191 handle_stripe_expansion(conf, sh, NULL);
3192
3193 unlock:
3194 spin_unlock(&sh->lock);
3195
3196 /* wait for this device to become unblocked */
3197 if (unlikely(blocked_rdev))
3198 md_wait_for_blocked_rdev(blocked_rdev, conf->mddev);
3199
3200 if (s.ops_request)
3201 raid_run_ops(sh, s.ops_request);
3202
3203 ops_run_io(sh, &s);
3204
3205 return_io(return_bi);
3206}
3207
3208static void handle_stripe6(struct stripe_head *sh)
3209{
3210 raid5_conf_t *conf = sh->raid_conf;
3211 int disks = sh->disks;
3212 struct bio *return_bi = NULL;
3213 int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx;
3214 struct stripe_head_state s;
3215 struct r6_state r6s;
3216 struct r5dev *dev, *pdev, *qdev;
3217 mdk_rdev_t *blocked_rdev = NULL;
3218
3219 pr_debug("handling stripe %llu, state=%#lx cnt=%d, "
3220 "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n",
3221 (unsigned long long)sh->sector, sh->state,
3222 atomic_read(&sh->count), pd_idx, qd_idx,
3223 sh->check_state, sh->reconstruct_state);
3224 memset(&s, 0, sizeof(s));
3225
3226 spin_lock(&sh->lock);
3227 clear_bit(STRIPE_HANDLE, &sh->state);
3228 clear_bit(STRIPE_DELAYED, &sh->state);
3229
3230 s.syncing = test_bit(STRIPE_SYNCING, &sh->state);
3231 s.expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state);
3232 s.expanded = test_bit(STRIPE_EXPAND_READY, &sh->state);
3233 /* Now to look around and see what can be done */
3234
3235 rcu_read_lock();
3236 for (i=disks; i--; ) {
3237 mdk_rdev_t *rdev;
3238 dev = &sh->dev[i];
3239 clear_bit(R5_Insync, &dev->flags);
3240
3241 pr_debug("check %d: state 0x%lx read %p write %p written %p\n",
3242 i, dev->flags, dev->toread, dev->towrite, dev->written);
3243 /* maybe we can reply to a read
3244 *
3245 * new wantfill requests are only permitted while
3246 * ops_complete_biofill is guaranteed to be inactive
3247 */
3248 if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread &&
3249 !test_bit(STRIPE_BIOFILL_RUN, &sh->state))
3250 set_bit(R5_Wantfill, &dev->flags);
3251
3252 /* now count some things */
3253 if (test_bit(R5_LOCKED, &dev->flags)) s.locked++;
3254 if (test_bit(R5_UPTODATE, &dev->flags)) s.uptodate++;
3255 if (test_bit(R5_Wantcompute, &dev->flags)) {
3256 s.compute++;
3257 BUG_ON(s.compute > 2);
3258 }
3259
3260 if (test_bit(R5_Wantfill, &dev->flags)) {
3261 s.to_fill++;
3262 } else if (dev->toread)
3263 s.to_read++;
3264 if (dev->towrite) {
3265 s.to_write++;
3266 if (!test_bit(R5_OVERWRITE, &dev->flags))
3267 s.non_overwrite++;
3268 }
3269 if (dev->written)
3270 s.written++;
3271 rdev = rcu_dereference(conf->disks[i].rdev);
3272 if (blocked_rdev == NULL &&
3273 rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
3274 blocked_rdev = rdev;
3275 atomic_inc(&rdev->nr_pending);
3276 }
3277 if (!rdev || !test_bit(In_sync, &rdev->flags)) {
3278 /* The ReadError flag will just be confusing now */
3279 clear_bit(R5_ReadError, &dev->flags);
3280 clear_bit(R5_ReWrite, &dev->flags);
3281 }
3282 if (!rdev || !test_bit(In_sync, &rdev->flags)
3283 || test_bit(R5_ReadError, &dev->flags)) {
3284 if (s.failed < 2)
3285 r6s.failed_num[s.failed] = i;
3286 s.failed++;
3287 } else
3288 set_bit(R5_Insync, &dev->flags);
3289 }
3290 rcu_read_unlock();
3291
3292 if (unlikely(blocked_rdev)) {
3293 if (s.syncing || s.expanding || s.expanded ||
3294 s.to_write || s.written) {
3295 set_bit(STRIPE_HANDLE, &sh->state);
3296 goto unlock;
3297 }
3298 /* There is nothing for the blocked_rdev to block */
3299 rdev_dec_pending(blocked_rdev, conf->mddev);
3300 blocked_rdev = NULL;
3301 }
3302
3303 if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {
3304 set_bit(STRIPE_OP_BIOFILL, &s.ops_request);
3305 set_bit(STRIPE_BIOFILL_RUN, &sh->state);
3306 }
3307
3308 pr_debug("locked=%d uptodate=%d to_read=%d"
3309 " to_write=%d failed=%d failed_num=%d,%d\n",
3310 s.locked, s.uptodate, s.to_read, s.to_write, s.failed,
3311 r6s.failed_num[0], r6s.failed_num[1]);
3312 /* check if the array has lost >2 devices and, if so, some requests
3313 * might need to be failed
3314 */
3315 if (s.failed > 2 && s.to_read+s.to_write+s.written)
3316 handle_failed_stripe(conf, sh, &s, disks, &return_bi);
3317 if (s.failed > 2 && s.syncing) {
3318 md_done_sync(conf->mddev, STRIPE_SECTORS,0);
3319 clear_bit(STRIPE_SYNCING, &sh->state);
3320 s.syncing = 0;
3321 }
3322
3323 /*
3324 * might be able to return some write requests if the parity blocks
3325 * are safe, or on a failed drive
3326 */
3327 pdev = &sh->dev[pd_idx];
3328 r6s.p_failed = (s.failed >= 1 && r6s.failed_num[0] == pd_idx)
3329 || (s.failed >= 2 && r6s.failed_num[1] == pd_idx);
3330 qdev = &sh->dev[qd_idx];
3331 r6s.q_failed = (s.failed >= 1 && r6s.failed_num[0] == qd_idx)
3332 || (s.failed >= 2 && r6s.failed_num[1] == qd_idx);
3333
3334 if ( s.written &&
3335 ( r6s.p_failed || ((test_bit(R5_Insync, &pdev->flags)
3336 && !test_bit(R5_LOCKED, &pdev->flags)
3337 && test_bit(R5_UPTODATE, &pdev->flags)))) &&
3338 ( r6s.q_failed || ((test_bit(R5_Insync, &qdev->flags)
3339 && !test_bit(R5_LOCKED, &qdev->flags)
3340 && test_bit(R5_UPTODATE, &qdev->flags)))))
3341 handle_stripe_clean_event(conf, sh, disks, &return_bi);
3342
3343 /* Now we might consider reading some blocks, either to check/generate
3344 * parity, or to satisfy requests
3345 * or to load a block that is being partially written.
3346 */
3347 if (s.to_read || s.non_overwrite || (s.to_write && s.failed) ||
3348 (s.syncing && (s.uptodate + s.compute < disks)) || s.expanding)
3349 handle_stripe_fill6(sh, &s, &r6s, disks);
3350
3351 /* Now we check to see if any write operations have recently
3352 * completed
3353 */
3354 if (sh->reconstruct_state == reconstruct_state_drain_result) {
3355 int qd_idx = sh->qd_idx;
3356
3357 sh->reconstruct_state = reconstruct_state_idle;
3358 /* All the 'written' buffers and the parity blocks are ready to
3359 * be written back to disk
3360 */
3361 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags));
3362 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags));
3363 for (i = disks; i--; ) {
3364 dev = &sh->dev[i];
3365 if (test_bit(R5_LOCKED, &dev->flags) &&
3366 (i == sh->pd_idx || i == qd_idx ||
3367 dev->written)) {
3368 pr_debug("Writing block %d\n", i);
3369 BUG_ON(!test_bit(R5_UPTODATE, &dev->flags));
3370 set_bit(R5_Wantwrite, &dev->flags);
3371 if (!test_bit(R5_Insync, &dev->flags) ||
3372 ((i == sh->pd_idx || i == qd_idx) &&
3373 s.failed == 0))
3374 set_bit(STRIPE_INSYNC, &sh->state);
3375 }
3376 }
3377 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
3378 atomic_dec(&conf->preread_active_stripes);
3379 if (atomic_read(&conf->preread_active_stripes) <
3380 IO_THRESHOLD)
3381 md_wakeup_thread(conf->mddev->thread);
3382 }
3383 }
3384
3385 /* Now to consider new write requests and what else, if anything
3386 * should be read. We do not handle new writes when:
3387 * 1/ A 'write' operation (copy+gen_syndrome) is already in flight.
3388 * 2/ A 'check' operation is in flight, as it may clobber the parity
3389 * block.
3390 */
3391 if (s.to_write && !sh->reconstruct_state && !sh->check_state)
3392 handle_stripe_dirtying6(conf, sh, &s, &r6s, disks);
3393
3394 /* maybe we need to check and possibly fix the parity for this stripe
3395 * Any reads will already have been scheduled, so we just see if enough
3396 * data is available. The parity check is held off while parity
3397 * dependent operations are in flight.
3398 */
3399 if (sh->check_state ||
3400 (s.syncing && s.locked == 0 &&
3401 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
3402 !test_bit(STRIPE_INSYNC, &sh->state)))
3403 handle_parity_checks6(conf, sh, &s, &r6s, disks);
3404
3405 if (s.syncing && s.locked == 0 && test_bit(STRIPE_INSYNC, &sh->state)) {
3406 md_done_sync(conf->mddev, STRIPE_SECTORS,1);
3407 clear_bit(STRIPE_SYNCING, &sh->state);
3408 }
3409
3410 /* If the failed drives are just a ReadError, then we might need
3411 * to progress the repair/check process
3412 */
3413 if (s.failed <= 2 && !conf->mddev->ro)
3414 for (i = 0; i < s.failed; i++) {
3415 dev = &sh->dev[r6s.failed_num[i]];
3416 if (test_bit(R5_ReadError, &dev->flags)
3417 && !test_bit(R5_LOCKED, &dev->flags)
3418 && test_bit(R5_UPTODATE, &dev->flags)
3419 ) {
3420 if (!test_bit(R5_ReWrite, &dev->flags)) {
3421 set_bit(R5_Wantwrite, &dev->flags);
3422 set_bit(R5_ReWrite, &dev->flags);
3423 set_bit(R5_LOCKED, &dev->flags);
3424 s.locked++;
3425 } else {
3426 /* let's read it back */
3427 set_bit(R5_Wantread, &dev->flags);
3428 set_bit(R5_LOCKED, &dev->flags);
3429 s.locked++;
3430 }
3431 }
3432 }
3433
3434 /* Finish reconstruct operations initiated by the expansion process */
3435 if (sh->reconstruct_state == reconstruct_state_result) {
3436 sh->reconstruct_state = reconstruct_state_idle;
3437 clear_bit(STRIPE_EXPANDING, &sh->state);
3438 for (i = conf->raid_disks; i--; ) {
3439 set_bit(R5_Wantwrite, &sh->dev[i].flags);
3440 set_bit(R5_LOCKED, &sh->dev[i].flags);
3441 s.locked++;
3442 }
3443 }
3444
3445 if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&
3446 !sh->reconstruct_state) {
3447 struct stripe_head *sh2
3448 = get_active_stripe(conf, sh->sector, 1, 1, 1);
3449 if (sh2 && test_bit(STRIPE_EXPAND_SOURCE, &sh2->state)) {
3450 /* sh cannot be written until sh2 has been read.
3451 * so arrange for sh to be delayed a little
3452 */
3453 set_bit(STRIPE_DELAYED, &sh->state);
3454 set_bit(STRIPE_HANDLE, &sh->state);
3455 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE,
3456 &sh2->state))
3457 atomic_inc(&conf->preread_active_stripes);
3458 release_stripe(sh2);
3459 goto unlock;
3460 }
3461 if (sh2)
3462 release_stripe(sh2);
3463
3464 /* Need to write out all blocks after computing P&Q */
3465 sh->disks = conf->raid_disks;
3466 stripe_set_idx(sh->sector, conf, 0, sh);
3467 schedule_reconstruction(sh, &s, 1, 1);
3468 } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
3469 clear_bit(STRIPE_EXPAND_READY, &sh->state);
3470 atomic_dec(&conf->reshape_stripes);
3471 wake_up(&conf->wait_for_overlap);
3472 md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
3473 }
3474
3475 if (s.expanding && s.locked == 0 &&
3476 !test_bit(STRIPE_COMPUTE_RUN, &sh->state))
3477 handle_stripe_expansion(conf, sh, &r6s);
3478
3479 unlock:
3480 spin_unlock(&sh->lock);
3481
3482 /* wait for this device to become unblocked */
3483 if (unlikely(blocked_rdev))
3484 md_wait_for_blocked_rdev(blocked_rdev, conf->mddev);
3485
3486 if (s.ops_request)
3487 raid_run_ops(sh, s.ops_request);
3488
3489 ops_run_io(sh, &s);
3490
3491 return_io(return_bi);
3492}
3493
3494static void handle_stripe(struct stripe_head *sh)
3495{
3496 if (sh->raid_conf->level == 6)
3497 handle_stripe6(sh);
3498 else
3499 handle_stripe5(sh);
3500}
3501
3502static void raid5_activate_delayed(raid5_conf_t *conf)
3503{
3504 if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) {
3505 while (!list_empty(&conf->delayed_list)) {
3506 struct list_head *l = conf->delayed_list.next;
3507 struct stripe_head *sh;
3508 sh = list_entry(l, struct stripe_head, lru);
3509 list_del_init(l);
3510 clear_bit(STRIPE_DELAYED, &sh->state);
3511 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
3512 atomic_inc(&conf->preread_active_stripes);
3513 list_add_tail(&sh->lru, &conf->hold_list);
3514 }
3515 } else
3516 blk_plug_device(conf->mddev->queue);
3517}
3518
3519static void activate_bit_delay(raid5_conf_t *conf)
3520{
3521 /* device_lock is held */
3522 struct list_head head;
3523 list_add(&head, &conf->bitmap_list);
3524 list_del_init(&conf->bitmap_list);
3525 while (!list_empty(&head)) {
3526 struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru);
3527 list_del_init(&sh->lru);
3528 atomic_inc(&sh->count);
3529 __release_stripe(conf, sh);
3530 }
3531}
3532
3533static void unplug_slaves(mddev_t *mddev)
3534{
3535 raid5_conf_t *conf = mddev->private;
3536 int i;
3537
3538 rcu_read_lock();
3539 for (i = 0; i < conf->raid_disks; i++) {
3540 mdk_rdev_t *rdev = rcu_dereference(conf->disks[i].rdev);
3541 if (rdev && !test_bit(Faulty, &rdev->flags) && atomic_read(&rdev->nr_pending)) {
3542 struct request_queue *r_queue = bdev_get_queue(rdev->bdev);
3543
3544 atomic_inc(&rdev->nr_pending);
3545 rcu_read_unlock();
3546
3547 blk_unplug(r_queue);
3548
3549 rdev_dec_pending(rdev, mddev);
3550 rcu_read_lock();
3551 }
3552 }
3553 rcu_read_unlock();
3554}
3555
3556static void raid5_unplug_device(struct request_queue *q)
3557{
3558 mddev_t *mddev = q->queuedata;
3559 raid5_conf_t *conf = mddev->private;
3560 unsigned long flags;
3561
3562 spin_lock_irqsave(&conf->device_lock, flags);
3563
3564 if (blk_remove_plug(q)) {
3565 conf->seq_flush++;
3566 raid5_activate_delayed(conf);
3567 }
3568 md_wakeup_thread(mddev->thread);
3569
3570 spin_unlock_irqrestore(&conf->device_lock, flags);
3571
3572 unplug_slaves(mddev);
3573}
3574
3575static int raid5_congested(void *data, int bits)
3576{
3577 mddev_t *mddev = data;
3578 raid5_conf_t *conf = mddev->private;
3579
3580 /* No difference between reads and writes. Just check
3581 * how busy the stripe_cache is
3582 */
3583
3584 if (mddev_congested(mddev, bits))
3585 return 1;
3586 if (conf->inactive_blocked)
3587 return 1;
3588 if (conf->quiesce)
3589 return 1;
3590 if (list_empty_careful(&conf->inactive_list))
3591 return 1;
3592
3593 return 0;
3594}
3595
3596/* We want read requests to align with chunks where possible,
3597 * but write requests don't need to.
3598 */
3599static int raid5_mergeable_bvec(struct request_queue *q,
3600 struct bvec_merge_data *bvm,
3601 struct bio_vec *biovec)
3602{
3603 mddev_t *mddev = q->queuedata;
3604 sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
3605 int max;
3606 unsigned int chunk_sectors = mddev->chunk_sectors;
3607 unsigned int bio_sectors = bvm->bi_size >> 9;
3608
3609 if ((bvm->bi_rw & 1) == WRITE)
3610 return biovec->bv_len; /* always allow writes to be mergeable */
3611
3612 if (mddev->new_chunk_sectors < mddev->chunk_sectors)
3613 chunk_sectors = mddev->new_chunk_sectors;
3614 max = (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9;
3615 if (max < 0) max = 0;
3616 if (max <= biovec->bv_len && bio_sectors == 0)
3617 return biovec->bv_len;
3618 else
3619 return max;
3620}
3621
3622
3623static int in_chunk_boundary(mddev_t *mddev, struct bio *bio)
3624{
3625 sector_t sector = bio->bi_sector + get_start_sect(bio->bi_bdev);
3626 unsigned int chunk_sectors = mddev->chunk_sectors;
3627 unsigned int bio_sectors = bio->bi_size >> 9;
3628
3629 if (mddev->new_chunk_sectors < mddev->chunk_sectors)
3630 chunk_sectors = mddev->new_chunk_sectors;
3631 return chunk_sectors >=
3632 ((sector & (chunk_sectors - 1)) + bio_sectors);
3633}
3634
3635/*
3636 * add bio to the retry LIFO ( in O(1) ... we are in interrupt )
3637 * later sampled by raid5d.
3638 */
3639static void add_bio_to_retry(struct bio *bi,raid5_conf_t *conf)
3640{
3641 unsigned long flags;
3642
3643 spin_lock_irqsave(&conf->device_lock, flags);
3644
3645 bi->bi_next = conf->retry_read_aligned_list;
3646 conf->retry_read_aligned_list = bi;
3647
3648 spin_unlock_irqrestore(&conf->device_lock, flags);
3649 md_wakeup_thread(conf->mddev->thread);
3650}
3651
3652
3653static struct bio *remove_bio_from_retry(raid5_conf_t *conf)
3654{
3655 struct bio *bi;
3656
3657 bi = conf->retry_read_aligned;
3658 if (bi) {
3659 conf->retry_read_aligned = NULL;
3660 return bi;
3661 }
3662 bi = conf->retry_read_aligned_list;
3663 if(bi) {
3664 conf->retry_read_aligned_list = bi->bi_next;
3665 bi->bi_next = NULL;
3666 /*
3667 * this sets the active strip count to 1 and the processed
3668 * strip count to zero (upper 8 bits)
3669 */
3670 bi->bi_phys_segments = 1; /* biased count of active stripes */
3671 }
3672
3673 return bi;
3674}
3675
3676
3677/*
3678 * The "raid5_align_endio" should check if the read succeeded and if it
3679 * did, call bio_endio on the original bio (having bio_put the new bio
3680 * first).
3681 * If the read failed..
3682 */
3683static void raid5_align_endio(struct bio *bi, int error)
3684{
3685 struct bio* raid_bi = bi->bi_private;
3686 mddev_t *mddev;
3687 raid5_conf_t *conf;
3688 int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
3689 mdk_rdev_t *rdev;
3690
3691 bio_put(bi);
3692
3693 mddev = raid_bi->bi_bdev->bd_disk->queue->queuedata;
3694 conf = mddev->private;
3695 rdev = (void*)raid_bi->bi_next;
3696 raid_bi->bi_next = NULL;
3697
3698 rdev_dec_pending(rdev, conf->mddev);
3699
3700 if (!error && uptodate) {
3701 bio_endio(raid_bi, 0);
3702 if (atomic_dec_and_test(&conf->active_aligned_reads))
3703 wake_up(&conf->wait_for_stripe);
3704 return;
3705 }
3706
3707
3708 pr_debug("raid5_align_endio : io error...handing IO for a retry\n");
3709
3710 add_bio_to_retry(raid_bi, conf);
3711}
3712
3713static int bio_fits_rdev(struct bio *bi)
3714{
3715 struct request_queue *q = bdev_get_queue(bi->bi_bdev);
3716
3717 if ((bi->bi_size>>9) > queue_max_sectors(q))
3718 return 0;
3719 blk_recount_segments(q, bi);
3720 if (bi->bi_phys_segments > queue_max_phys_segments(q))
3721 return 0;
3722
3723 if (q->merge_bvec_fn)
3724 /* it's too hard to apply the merge_bvec_fn at this stage,
3725 * just just give up
3726 */
3727 return 0;
3728
3729 return 1;
3730}
3731
3732
3733static int chunk_aligned_read(struct request_queue *q, struct bio * raid_bio)
3734{
3735 mddev_t *mddev = q->queuedata;
3736 raid5_conf_t *conf = mddev->private;
3737 unsigned int dd_idx;
3738 struct bio* align_bi;
3739 mdk_rdev_t *rdev;
3740
3741 if (!in_chunk_boundary(mddev, raid_bio)) {
3742 pr_debug("chunk_aligned_read : non aligned\n");
3743 return 0;
3744 }
3745 /*
3746 * use bio_clone to make a copy of the bio
3747 */
3748 align_bi = bio_clone(raid_bio, GFP_NOIO);
3749 if (!align_bi)
3750 return 0;
3751 /*
3752 * set bi_end_io to a new function, and set bi_private to the
3753 * original bio.
3754 */
3755 align_bi->bi_end_io = raid5_align_endio;
3756 align_bi->bi_private = raid_bio;
3757 /*
3758 * compute position
3759 */
3760 align_bi->bi_sector = raid5_compute_sector(conf, raid_bio->bi_sector,
3761 0,
3762 &dd_idx, NULL);
3763
3764 rcu_read_lock();
3765 rdev = rcu_dereference(conf->disks[dd_idx].rdev);
3766 if (rdev && test_bit(In_sync, &rdev->flags)) {
3767 atomic_inc(&rdev->nr_pending);
3768 rcu_read_unlock();
3769 raid_bio->bi_next = (void*)rdev;
3770 align_bi->bi_bdev = rdev->bdev;
3771 align_bi->bi_flags &= ~(1 << BIO_SEG_VALID);
3772 align_bi->bi_sector += rdev->data_offset;
3773
3774 if (!bio_fits_rdev(align_bi)) {
3775 /* too big in some way */
3776 bio_put(align_bi);
3777 rdev_dec_pending(rdev, mddev);
3778 return 0;
3779 }
3780
3781 spin_lock_irq(&conf->device_lock);
3782 wait_event_lock_irq(conf->wait_for_stripe,
3783 conf->quiesce == 0,
3784 conf->device_lock, /* nothing */);
3785 atomic_inc(&conf->active_aligned_reads);
3786 spin_unlock_irq(&conf->device_lock);
3787
3788 generic_make_request(align_bi);
3789 return 1;
3790 } else {
3791 rcu_read_unlock();
3792 bio_put(align_bi);
3793 return 0;
3794 }
3795}
3796
3797/* __get_priority_stripe - get the next stripe to process
3798 *
3799 * Full stripe writes are allowed to pass preread active stripes up until
3800 * the bypass_threshold is exceeded. In general the bypass_count
3801 * increments when the handle_list is handled before the hold_list; however, it
3802 * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a
3803 * stripe with in flight i/o. The bypass_count will be reset when the
3804 * head of the hold_list has changed, i.e. the head was promoted to the
3805 * handle_list.
3806 */
3807static struct stripe_head *__get_priority_stripe(raid5_conf_t *conf)
3808{
3809 struct stripe_head *sh;
3810
3811 pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n",
3812 __func__,
3813 list_empty(&conf->handle_list) ? "empty" : "busy",
3814 list_empty(&conf->hold_list) ? "empty" : "busy",
3815 atomic_read(&conf->pending_full_writes), conf->bypass_count);
3816
3817 if (!list_empty(&conf->handle_list)) {
3818 sh = list_entry(conf->handle_list.next, typeof(*sh), lru);
3819
3820 if (list_empty(&conf->hold_list))
3821 conf->bypass_count = 0;
3822 else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) {
3823 if (conf->hold_list.next == conf->last_hold)
3824 conf->bypass_count++;
3825 else {
3826 conf->last_hold = conf->hold_list.next;
3827 conf->bypass_count -= conf->bypass_threshold;
3828 if (conf->bypass_count < 0)
3829 conf->bypass_count = 0;
3830 }
3831 }
3832 } else if (!list_empty(&conf->hold_list) &&
3833 ((conf->bypass_threshold &&
3834 conf->bypass_count > conf->bypass_threshold) ||
3835 atomic_read(&conf->pending_full_writes) == 0)) {
3836 sh = list_entry(conf->hold_list.next,
3837 typeof(*sh), lru);
3838 conf->bypass_count -= conf->bypass_threshold;
3839 if (conf->bypass_count < 0)
3840 conf->bypass_count = 0;
3841 } else
3842 return NULL;
3843
3844 list_del_init(&sh->lru);
3845 atomic_inc(&sh->count);
3846 BUG_ON(atomic_read(&sh->count) != 1);
3847 return sh;
3848}
3849
3850static int make_request(struct request_queue *q, struct bio * bi)
3851{
3852 mddev_t *mddev = q->queuedata;
3853 raid5_conf_t *conf = mddev->private;
3854 int dd_idx;
3855 sector_t new_sector;
3856 sector_t logical_sector, last_sector;
3857 struct stripe_head *sh;
3858 const int rw = bio_data_dir(bi);
3859 int cpu, remaining;
3860
3861 if (unlikely(bio_rw_flagged(bi, BIO_RW_BARRIER))) {
3862 bio_endio(bi, -EOPNOTSUPP);
3863 return 0;
3864 }
3865
3866 md_write_start(mddev, bi);
3867
3868 cpu = part_stat_lock();
3869 part_stat_inc(cpu, &mddev->gendisk->part0, ios[rw]);
3870 part_stat_add(cpu, &mddev->gendisk->part0, sectors[rw],
3871 bio_sectors(bi));
3872 part_stat_unlock();
3873
3874 if (rw == READ &&
3875 mddev->reshape_position == MaxSector &&
3876 chunk_aligned_read(q,bi))
3877 return 0;
3878
3879 logical_sector = bi->bi_sector & ~((sector_t)STRIPE_SECTORS-1);
3880 last_sector = bi->bi_sector + (bi->bi_size>>9);
3881 bi->bi_next = NULL;
3882 bi->bi_phys_segments = 1; /* over-loaded to count active stripes */
3883
3884 for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) {
3885 DEFINE_WAIT(w);
3886 int disks, data_disks;
3887 int previous;
3888
3889 retry:
3890 previous = 0;
3891 disks = conf->raid_disks;
3892 prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
3893 if (unlikely(conf->reshape_progress != MaxSector)) {
3894 /* spinlock is needed as reshape_progress may be
3895 * 64bit on a 32bit platform, and so it might be
3896 * possible to see a half-updated value
3897 * Ofcourse reshape_progress could change after
3898 * the lock is dropped, so once we get a reference
3899 * to the stripe that we think it is, we will have
3900 * to check again.
3901 */
3902 spin_lock_irq(&conf->device_lock);
3903 if (mddev->delta_disks < 0
3904 ? logical_sector < conf->reshape_progress
3905 : logical_sector >= conf->reshape_progress) {
3906 disks = conf->previous_raid_disks;
3907 previous = 1;
3908 } else {
3909 if (mddev->delta_disks < 0
3910 ? logical_sector < conf->reshape_safe
3911 : logical_sector >= conf->reshape_safe) {
3912 spin_unlock_irq(&conf->device_lock);
3913 schedule();
3914 goto retry;
3915 }
3916 }
3917 spin_unlock_irq(&conf->device_lock);
3918 }
3919 data_disks = disks - conf->max_degraded;
3920
3921 new_sector = raid5_compute_sector(conf, logical_sector,
3922 previous,
3923 &dd_idx, NULL);
3924 pr_debug("raid5: make_request, sector %llu logical %llu\n",
3925 (unsigned long long)new_sector,
3926 (unsigned long long)logical_sector);
3927
3928 sh = get_active_stripe(conf, new_sector, previous,
3929 (bi->bi_rw&RWA_MASK), 0);
3930 if (sh) {
3931 if (unlikely(previous)) {
3932 /* expansion might have moved on while waiting for a
3933 * stripe, so we must do the range check again.
3934 * Expansion could still move past after this
3935 * test, but as we are holding a reference to
3936 * 'sh', we know that if that happens,
3937 * STRIPE_EXPANDING will get set and the expansion
3938 * won't proceed until we finish with the stripe.
3939 */
3940 int must_retry = 0;
3941 spin_lock_irq(&conf->device_lock);
3942 if (mddev->delta_disks < 0
3943 ? logical_sector >= conf->reshape_progress
3944 : logical_sector < conf->reshape_progress)
3945 /* mismatch, need to try again */
3946 must_retry = 1;
3947 spin_unlock_irq(&conf->device_lock);
3948 if (must_retry) {
3949 release_stripe(sh);
3950 schedule();
3951 goto retry;
3952 }
3953 }
3954
3955 if (bio_data_dir(bi) == WRITE &&
3956 logical_sector >= mddev->suspend_lo &&
3957 logical_sector < mddev->suspend_hi) {
3958 release_stripe(sh);
3959 /* As the suspend_* range is controlled by
3960 * userspace, we want an interruptible
3961 * wait.
3962 */
3963 flush_signals(current);
3964 prepare_to_wait(&conf->wait_for_overlap,
3965 &w, TASK_INTERRUPTIBLE);
3966 if (logical_sector >= mddev->suspend_lo &&
3967 logical_sector < mddev->suspend_hi)
3968 schedule();
3969 goto retry;
3970 }
3971
3972 if (test_bit(STRIPE_EXPANDING, &sh->state) ||
3973 !add_stripe_bio(sh, bi, dd_idx, (bi->bi_rw&RW_MASK))) {
3974 /* Stripe is busy expanding or
3975 * add failed due to overlap. Flush everything
3976 * and wait a while
3977 */
3978 raid5_unplug_device(mddev->queue);
3979 release_stripe(sh);
3980 schedule();
3981 goto retry;
3982 }
3983 finish_wait(&conf->wait_for_overlap, &w);
3984 set_bit(STRIPE_HANDLE, &sh->state);
3985 clear_bit(STRIPE_DELAYED, &sh->state);
3986 release_stripe(sh);
3987 } else {
3988 /* cannot get stripe for read-ahead, just give-up */
3989 clear_bit(BIO_UPTODATE, &bi->bi_flags);
3990 finish_wait(&conf->wait_for_overlap, &w);
3991 break;
3992 }
3993
3994 }
3995 spin_lock_irq(&conf->device_lock);
3996 remaining = raid5_dec_bi_phys_segments(bi);
3997 spin_unlock_irq(&conf->device_lock);
3998 if (remaining == 0) {
3999
4000 if ( rw == WRITE )
4001 md_write_end(mddev);
4002
4003 bio_endio(bi, 0);
4004 }
4005 return 0;
4006}
4007
4008static sector_t raid5_size(mddev_t *mddev, sector_t sectors, int raid_disks);
4009
4010static sector_t reshape_request(mddev_t *mddev, sector_t sector_nr, int *skipped)
4011{
4012 /* reshaping is quite different to recovery/resync so it is
4013 * handled quite separately ... here.
4014 *
4015 * On each call to sync_request, we gather one chunk worth of
4016 * destination stripes and flag them as expanding.
4017 * Then we find all the source stripes and request reads.
4018 * As the reads complete, handle_stripe will copy the data
4019 * into the destination stripe and release that stripe.
4020 */
4021 raid5_conf_t *conf = (raid5_conf_t *) mddev->private;
4022 struct stripe_head *sh;
4023 sector_t first_sector, last_sector;
4024 int raid_disks = conf->previous_raid_disks;
4025 int data_disks = raid_disks - conf->max_degraded;
4026 int new_data_disks = conf->raid_disks - conf->max_degraded;
4027 int i;
4028 int dd_idx;
4029 sector_t writepos, readpos, safepos;
4030 sector_t stripe_addr;
4031 int reshape_sectors;
4032 struct list_head stripes;
4033
4034 if (sector_nr == 0) {
4035 /* If restarting in the middle, skip the initial sectors */
4036 if (mddev->delta_disks < 0 &&
4037 conf->reshape_progress < raid5_size(mddev, 0, 0)) {
4038 sector_nr = raid5_size(mddev, 0, 0)
4039 - conf->reshape_progress;
4040 } else if (mddev->delta_disks >= 0 &&
4041 conf->reshape_progress > 0)
4042 sector_nr = conf->reshape_progress;
4043 sector_div(sector_nr, new_data_disks);
4044 if (sector_nr) {
4045 *skipped = 1;
4046 return sector_nr;
4047 }
4048 }
4049
4050 /* We need to process a full chunk at a time.
4051 * If old and new chunk sizes differ, we need to process the
4052 * largest of these
4053 */
4054 if (mddev->new_chunk_sectors > mddev->chunk_sectors)
4055 reshape_sectors = mddev->new_chunk_sectors;
4056 else
4057 reshape_sectors = mddev->chunk_sectors;
4058
4059 /* we update the metadata when there is more than 3Meg
4060 * in the block range (that is rather arbitrary, should
4061 * probably be time based) or when the data about to be
4062 * copied would over-write the source of the data at
4063 * the front of the range.
4064 * i.e. one new_stripe along from reshape_progress new_maps
4065 * to after where reshape_safe old_maps to
4066 */
4067 writepos = conf->reshape_progress;
4068 sector_div(writepos, new_data_disks);
4069 readpos = conf->reshape_progress;
4070 sector_div(readpos, data_disks);
4071 safepos = conf->reshape_safe;
4072 sector_div(safepos, data_disks);
4073 if (mddev->delta_disks < 0) {
4074 writepos -= min_t(sector_t, reshape_sectors, writepos);
4075 readpos += reshape_sectors;
4076 safepos += reshape_sectors;
4077 } else {
4078 writepos += reshape_sectors;
4079 readpos -= min_t(sector_t, reshape_sectors, readpos);
4080 safepos -= min_t(sector_t, reshape_sectors, safepos);
4081 }
4082
4083 /* 'writepos' is the most advanced device address we might write.
4084 * 'readpos' is the least advanced device address we might read.
4085 * 'safepos' is the least address recorded in the metadata as having
4086 * been reshaped.
4087 * If 'readpos' is behind 'writepos', then there is no way that we can
4088 * ensure safety in the face of a crash - that must be done by userspace
4089 * making a backup of the data. So in that case there is no particular
4090 * rush to update metadata.
4091 * Otherwise if 'safepos' is behind 'writepos', then we really need to
4092 * update the metadata to advance 'safepos' to match 'readpos' so that
4093 * we can be safe in the event of a crash.
4094 * So we insist on updating metadata if safepos is behind writepos and
4095 * readpos is beyond writepos.
4096 * In any case, update the metadata every 10 seconds.
4097 * Maybe that number should be configurable, but I'm not sure it is
4098 * worth it.... maybe it could be a multiple of safemode_delay???
4099 */
4100 if ((mddev->delta_disks < 0
4101 ? (safepos > writepos && readpos < writepos)
4102 : (safepos < writepos && readpos > writepos)) ||
4103 time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
4104 /* Cannot proceed until we've updated the superblock... */
4105 wait_event(conf->wait_for_overlap,
4106 atomic_read(&conf->reshape_stripes)==0);
4107 mddev->reshape_position = conf->reshape_progress;
4108 mddev->curr_resync_completed = mddev->curr_resync;
4109 conf->reshape_checkpoint = jiffies;
4110 set_bit(MD_CHANGE_DEVS, &mddev->flags);
4111 md_wakeup_thread(mddev->thread);
4112 wait_event(mddev->sb_wait, mddev->flags == 0 ||
4113 kthread_should_stop());
4114 spin_lock_irq(&conf->device_lock);
4115 conf->reshape_safe = mddev->reshape_position;
4116 spin_unlock_irq(&conf->device_lock);
4117 wake_up(&conf->wait_for_overlap);
4118 sysfs_notify(&mddev->kobj, NULL, "sync_completed");
4119 }
4120
4121 if (mddev->delta_disks < 0) {
4122 BUG_ON(conf->reshape_progress == 0);
4123 stripe_addr = writepos;
4124 BUG_ON((mddev->dev_sectors &
4125 ~((sector_t)reshape_sectors - 1))
4126 - reshape_sectors - stripe_addr
4127 != sector_nr);
4128 } else {
4129 BUG_ON(writepos != sector_nr + reshape_sectors);
4130 stripe_addr = sector_nr;
4131 }
4132 INIT_LIST_HEAD(&stripes);
4133 for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) {
4134 int j;
4135 int skipped_disk = 0;
4136 sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1);
4137 set_bit(STRIPE_EXPANDING, &sh->state);
4138 atomic_inc(&conf->reshape_stripes);
4139 /* If any of this stripe is beyond the end of the old
4140 * array, then we need to zero those blocks
4141 */
4142 for (j=sh->disks; j--;) {
4143 sector_t s;
4144 if (j == sh->pd_idx)
4145 continue;
4146 if (conf->level == 6 &&
4147 j == sh->qd_idx)
4148 continue;
4149 s = compute_blocknr(sh, j, 0);
4150 if (s < raid5_size(mddev, 0, 0)) {
4151 skipped_disk = 1;
4152 continue;
4153 }
4154 memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE);
4155 set_bit(R5_Expanded, &sh->dev[j].flags);
4156 set_bit(R5_UPTODATE, &sh->dev[j].flags);
4157 }
4158 if (!skipped_disk) {
4159 set_bit(STRIPE_EXPAND_READY, &sh->state);
4160 set_bit(STRIPE_HANDLE, &sh->state);
4161 }
4162 list_add(&sh->lru, &stripes);
4163 }
4164 spin_lock_irq(&conf->device_lock);
4165 if (mddev->delta_disks < 0)
4166 conf->reshape_progress -= reshape_sectors * new_data_disks;
4167 else
4168 conf->reshape_progress += reshape_sectors * new_data_disks;
4169 spin_unlock_irq(&conf->device_lock);
4170 /* Ok, those stripe are ready. We can start scheduling
4171 * reads on the source stripes.
4172 * The source stripes are determined by mapping the first and last
4173 * block on the destination stripes.
4174 */
4175 first_sector =
4176 raid5_compute_sector(conf, stripe_addr*(new_data_disks),
4177 1, &dd_idx, NULL);
4178 last_sector =
4179 raid5_compute_sector(conf, ((stripe_addr+reshape_sectors)
4180 * new_data_disks - 1),
4181 1, &dd_idx, NULL);
4182 if (last_sector >= mddev->dev_sectors)
4183 last_sector = mddev->dev_sectors - 1;
4184 while (first_sector <= last_sector) {
4185 sh = get_active_stripe(conf, first_sector, 1, 0, 1);
4186 set_bit(STRIPE_EXPAND_SOURCE, &sh->state);
4187 set_bit(STRIPE_HANDLE, &sh->state);
4188 release_stripe(sh);
4189 first_sector += STRIPE_SECTORS;
4190 }
4191 /* Now that the sources are clearly marked, we can release
4192 * the destination stripes
4193 */
4194 while (!list_empty(&stripes)) {
4195 sh = list_entry(stripes.next, struct stripe_head, lru);
4196 list_del_init(&sh->lru);
4197 release_stripe(sh);
4198 }
4199 /* If this takes us to the resync_max point where we have to pause,
4200 * then we need to write out the superblock.
4201 */
4202 sector_nr += reshape_sectors;
4203 if ((sector_nr - mddev->curr_resync_completed) * 2
4204 >= mddev->resync_max - mddev->curr_resync_completed) {
4205 /* Cannot proceed until we've updated the superblock... */
4206 wait_event(conf->wait_for_overlap,
4207 atomic_read(&conf->reshape_stripes) == 0);
4208 mddev->reshape_position = conf->reshape_progress;
4209 mddev->curr_resync_completed = mddev->curr_resync + reshape_sectors;
4210 conf->reshape_checkpoint = jiffies;
4211 set_bit(MD_CHANGE_DEVS, &mddev->flags);
4212 md_wakeup_thread(mddev->thread);
4213 wait_event(mddev->sb_wait,
4214 !test_bit(MD_CHANGE_DEVS, &mddev->flags)
4215 || kthread_should_stop());
4216 spin_lock_irq(&conf->device_lock);
4217 conf->reshape_safe = mddev->reshape_position;
4218 spin_unlock_irq(&conf->device_lock);
4219 wake_up(&conf->wait_for_overlap);
4220 sysfs_notify(&mddev->kobj, NULL, "sync_completed");
4221 }
4222 return reshape_sectors;
4223}
4224
4225/* FIXME go_faster isn't used */
4226static inline sector_t sync_request(mddev_t *mddev, sector_t sector_nr, int *skipped, int go_faster)
4227{
4228 raid5_conf_t *conf = (raid5_conf_t *) mddev->private;
4229 struct stripe_head *sh;
4230 sector_t max_sector = mddev->dev_sectors;
4231 int sync_blocks;
4232 int still_degraded = 0;
4233 int i;
4234
4235 if (sector_nr >= max_sector) {
4236 /* just being told to finish up .. nothing much to do */
4237 unplug_slaves(mddev);
4238
4239 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
4240 end_reshape(conf);
4241 return 0;
4242 }
4243
4244 if (mddev->curr_resync < max_sector) /* aborted */
4245 bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
4246 &sync_blocks, 1);
4247 else /* completed sync */
4248 conf->fullsync = 0;
4249 bitmap_close_sync(mddev->bitmap);
4250
4251 return 0;
4252 }
4253
4254 /* Allow raid5_quiesce to complete */
4255 wait_event(conf->wait_for_overlap, conf->quiesce != 2);
4256
4257 if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
4258 return reshape_request(mddev, sector_nr, skipped);
4259
4260 /* No need to check resync_max as we never do more than one
4261 * stripe, and as resync_max will always be on a chunk boundary,
4262 * if the check in md_do_sync didn't fire, there is no chance
4263 * of overstepping resync_max here
4264 */
4265
4266 /* if there is too many failed drives and we are trying
4267 * to resync, then assert that we are finished, because there is
4268 * nothing we can do.
4269 */
4270 if (mddev->degraded >= conf->max_degraded &&
4271 test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
4272 sector_t rv = mddev->dev_sectors - sector_nr;
4273 *skipped = 1;
4274 return rv;
4275 }
4276 if (!bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
4277 !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
4278 !conf->fullsync && sync_blocks >= STRIPE_SECTORS) {
4279 /* we can skip this block, and probably more */
4280 sync_blocks /= STRIPE_SECTORS;
4281 *skipped = 1;
4282 return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */
4283 }
4284
4285
4286 bitmap_cond_end_sync(mddev->bitmap, sector_nr);
4287
4288 sh = get_active_stripe(conf, sector_nr, 0, 1, 0);
4289 if (sh == NULL) {
4290 sh = get_active_stripe(conf, sector_nr, 0, 0, 0);
4291 /* make sure we don't swamp the stripe cache if someone else
4292 * is trying to get access
4293 */
4294 schedule_timeout_uninterruptible(1);
4295 }
4296 /* Need to check if array will still be degraded after recovery/resync
4297 * We don't need to check the 'failed' flag as when that gets set,
4298 * recovery aborts.
4299 */
4300 for (i = 0; i < conf->raid_disks; i++)
4301 if (conf->disks[i].rdev == NULL)
4302 still_degraded = 1;
4303
4304 bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded);
4305
4306 spin_lock(&sh->lock);
4307 set_bit(STRIPE_SYNCING, &sh->state);
4308 clear_bit(STRIPE_INSYNC, &sh->state);
4309 spin_unlock(&sh->lock);
4310
4311 handle_stripe(sh);
4312 release_stripe(sh);
4313
4314 return STRIPE_SECTORS;
4315}
4316
4317static int retry_aligned_read(raid5_conf_t *conf, struct bio *raid_bio)
4318{
4319 /* We may not be able to submit a whole bio at once as there
4320 * may not be enough stripe_heads available.
4321 * We cannot pre-allocate enough stripe_heads as we may need
4322 * more than exist in the cache (if we allow ever large chunks).
4323 * So we do one stripe head at a time and record in
4324 * ->bi_hw_segments how many have been done.
4325 *
4326 * We *know* that this entire raid_bio is in one chunk, so
4327 * it will be only one 'dd_idx' and only need one call to raid5_compute_sector.
4328 */
4329 struct stripe_head *sh;
4330 int dd_idx;
4331 sector_t sector, logical_sector, last_sector;
4332 int scnt = 0;
4333 int remaining;
4334 int handled = 0;
4335
4336 logical_sector = raid_bio->bi_sector & ~((sector_t)STRIPE_SECTORS-1);
4337 sector = raid5_compute_sector(conf, logical_sector,
4338 0, &dd_idx, NULL);
4339 last_sector = raid_bio->bi_sector + (raid_bio->bi_size>>9);
4340
4341 for (; logical_sector < last_sector;
4342 logical_sector += STRIPE_SECTORS,
4343 sector += STRIPE_SECTORS,
4344 scnt++) {
4345
4346 if (scnt < raid5_bi_hw_segments(raid_bio))
4347 /* already done this stripe */
4348 continue;
4349
4350 sh = get_active_stripe(conf, sector, 0, 1, 0);
4351
4352 if (!sh) {
4353 /* failed to get a stripe - must wait */
4354 raid5_set_bi_hw_segments(raid_bio, scnt);
4355 conf->retry_read_aligned = raid_bio;
4356 return handled;
4357 }
4358
4359 set_bit(R5_ReadError, &sh->dev[dd_idx].flags);
4360 if (!add_stripe_bio(sh, raid_bio, dd_idx, 0)) {
4361 release_stripe(sh);
4362 raid5_set_bi_hw_segments(raid_bio, scnt);
4363 conf->retry_read_aligned = raid_bio;
4364 return handled;
4365 }
4366
4367 handle_stripe(sh);
4368 release_stripe(sh);
4369 handled++;
4370 }
4371 spin_lock_irq(&conf->device_lock);
4372 remaining = raid5_dec_bi_phys_segments(raid_bio);
4373 spin_unlock_irq(&conf->device_lock);
4374 if (remaining == 0)
4375 bio_endio(raid_bio, 0);
4376 if (atomic_dec_and_test(&conf->active_aligned_reads))
4377 wake_up(&conf->wait_for_stripe);
4378 return handled;
4379}
4380
4381
4382/*
4383 * This is our raid5 kernel thread.
4384 *
4385 * We scan the hash table for stripes which can be handled now.
4386 * During the scan, completed stripes are saved for us by the interrupt
4387 * handler, so that they will not have to wait for our next wakeup.
4388 */
4389static void raid5d(mddev_t *mddev)
4390{
4391 struct stripe_head *sh;
4392 raid5_conf_t *conf = mddev->private;
4393 int handled;
4394
4395 pr_debug("+++ raid5d active\n");
4396
4397 md_check_recovery(mddev);
4398
4399 handled = 0;
4400 spin_lock_irq(&conf->device_lock);
4401 while (1) {
4402 struct bio *bio;
4403
4404 if (conf->seq_flush != conf->seq_write) {
4405 int seq = conf->seq_flush;
4406 spin_unlock_irq(&conf->device_lock);
4407 bitmap_unplug(mddev->bitmap);
4408 spin_lock_irq(&conf->device_lock);
4409 conf->seq_write = seq;
4410 activate_bit_delay(conf);
4411 }
4412
4413 while ((bio = remove_bio_from_retry(conf))) {
4414 int ok;
4415 spin_unlock_irq(&conf->device_lock);
4416 ok = retry_aligned_read(conf, bio);
4417 spin_lock_irq(&conf->device_lock);
4418 if (!ok)
4419 break;
4420 handled++;
4421 }
4422
4423 sh = __get_priority_stripe(conf);
4424
4425 if (!sh)
4426 break;
4427 spin_unlock_irq(&conf->device_lock);
4428
4429 handled++;
4430 handle_stripe(sh);
4431 release_stripe(sh);
4432 cond_resched();
4433
4434 spin_lock_irq(&conf->device_lock);
4435 }
4436 pr_debug("%d stripes handled\n", handled);
4437
4438 spin_unlock_irq(&conf->device_lock);
4439
4440 async_tx_issue_pending_all();
4441 unplug_slaves(mddev);
4442
4443 pr_debug("--- raid5d inactive\n");
4444}
4445
4446static ssize_t
4447raid5_show_stripe_cache_size(mddev_t *mddev, char *page)
4448{
4449 raid5_conf_t *conf = mddev->private;
4450 if (conf)
4451 return sprintf(page, "%d\n", conf->max_nr_stripes);
4452 else
4453 return 0;
4454}
4455
4456static ssize_t
4457raid5_store_stripe_cache_size(mddev_t *mddev, const char *page, size_t len)
4458{
4459 raid5_conf_t *conf = mddev->private;
4460 unsigned long new;
4461 int err;
4462
4463 if (len >= PAGE_SIZE)
4464 return -EINVAL;
4465 if (!conf)
4466 return -ENODEV;
4467
4468 if (strict_strtoul(page, 10, &new))
4469 return -EINVAL;
4470 if (new <= 16 || new > 32768)
4471 return -EINVAL;
4472 while (new < conf->max_nr_stripes) {
4473 if (drop_one_stripe(conf))
4474 conf->max_nr_stripes--;
4475 else
4476 break;
4477 }
4478 err = md_allow_write(mddev);
4479 if (err)
4480 return err;
4481 while (new > conf->max_nr_stripes) {
4482 if (grow_one_stripe(conf))
4483 conf->max_nr_stripes++;
4484 else break;
4485 }
4486 return len;
4487}
4488
4489static struct md_sysfs_entry
4490raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR,
4491 raid5_show_stripe_cache_size,
4492 raid5_store_stripe_cache_size);
4493
4494static ssize_t
4495raid5_show_preread_threshold(mddev_t *mddev, char *page)
4496{
4497 raid5_conf_t *conf = mddev->private;
4498 if (conf)
4499 return sprintf(page, "%d\n", conf->bypass_threshold);
4500 else
4501 return 0;
4502}
4503
4504static ssize_t
4505raid5_store_preread_threshold(mddev_t *mddev, const char *page, size_t len)
4506{
4507 raid5_conf_t *conf = mddev->private;
4508 unsigned long new;
4509 if (len >= PAGE_SIZE)
4510 return -EINVAL;
4511 if (!conf)
4512 return -ENODEV;
4513
4514 if (strict_strtoul(page, 10, &new))
4515 return -EINVAL;
4516 if (new > conf->max_nr_stripes)
4517 return -EINVAL;
4518 conf->bypass_threshold = new;
4519 return len;
4520}
4521
4522static struct md_sysfs_entry
4523raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold,
4524 S_IRUGO | S_IWUSR,
4525 raid5_show_preread_threshold,
4526 raid5_store_preread_threshold);
4527
4528static ssize_t
4529stripe_cache_active_show(mddev_t *mddev, char *page)
4530{
4531 raid5_conf_t *conf = mddev->private;
4532 if (conf)
4533 return sprintf(page, "%d\n", atomic_read(&conf->active_stripes));
4534 else
4535 return 0;
4536}
4537
4538static struct md_sysfs_entry
4539raid5_stripecache_active = __ATTR_RO(stripe_cache_active);
4540
4541static struct attribute *raid5_attrs[] = {
4542 &raid5_stripecache_size.attr,
4543 &raid5_stripecache_active.attr,
4544 &raid5_preread_bypass_threshold.attr,
4545 NULL,
4546};
4547static struct attribute_group raid5_attrs_group = {
4548 .name = NULL,
4549 .attrs = raid5_attrs,
4550};
4551
4552static sector_t
4553raid5_size(mddev_t *mddev, sector_t sectors, int raid_disks)
4554{
4555 raid5_conf_t *conf = mddev->private;
4556
4557 if (!sectors)
4558 sectors = mddev->dev_sectors;
4559 if (!raid_disks) {
4560 /* size is defined by the smallest of previous and new size */
4561 if (conf->raid_disks < conf->previous_raid_disks)
4562 raid_disks = conf->raid_disks;
4563 else
4564 raid_disks = conf->previous_raid_disks;
4565 }
4566
4567 sectors &= ~((sector_t)mddev->chunk_sectors - 1);
4568 sectors &= ~((sector_t)mddev->new_chunk_sectors - 1);
4569 return sectors * (raid_disks - conf->max_degraded);
4570}
4571
4572static void raid5_free_percpu(raid5_conf_t *conf)
4573{
4574 struct raid5_percpu *percpu;
4575 unsigned long cpu;
4576
4577 if (!conf->percpu)
4578 return;
4579
4580 get_online_cpus();
4581 for_each_possible_cpu(cpu) {
4582 percpu = per_cpu_ptr(conf->percpu, cpu);
4583 safe_put_page(percpu->spare_page);
4584 kfree(percpu->scribble);
4585 }
4586#ifdef CONFIG_HOTPLUG_CPU
4587 unregister_cpu_notifier(&conf->cpu_notify);
4588#endif
4589 put_online_cpus();
4590
4591 free_percpu(conf->percpu);
4592}
4593
4594static void free_conf(raid5_conf_t *conf)
4595{
4596 shrink_stripes(conf);
4597 raid5_free_percpu(conf);
4598 kfree(conf->disks);
4599 kfree(conf->stripe_hashtbl);
4600 kfree(conf);
4601}
4602
4603#ifdef CONFIG_HOTPLUG_CPU
4604static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action,
4605 void *hcpu)
4606{
4607 raid5_conf_t *conf = container_of(nfb, raid5_conf_t, cpu_notify);
4608 long cpu = (long)hcpu;
4609 struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu);
4610
4611 switch (action) {
4612 case CPU_UP_PREPARE:
4613 case CPU_UP_PREPARE_FROZEN:
4614 if (conf->level == 6 && !percpu->spare_page)
4615 percpu->spare_page = alloc_page(GFP_KERNEL);
4616 if (!percpu->scribble)
4617 percpu->scribble = kmalloc(conf->scribble_len, GFP_KERNEL);
4618
4619 if (!percpu->scribble ||
4620 (conf->level == 6 && !percpu->spare_page)) {
4621 safe_put_page(percpu->spare_page);
4622 kfree(percpu->scribble);
4623 pr_err("%s: failed memory allocation for cpu%ld\n",
4624 __func__, cpu);
4625 return NOTIFY_BAD;
4626 }
4627 break;
4628 case CPU_DEAD:
4629 case CPU_DEAD_FROZEN:
4630 safe_put_page(percpu->spare_page);
4631 kfree(percpu->scribble);
4632 percpu->spare_page = NULL;
4633 percpu->scribble = NULL;
4634 break;
4635 default:
4636 break;
4637 }
4638 return NOTIFY_OK;
4639}
4640#endif
4641
4642static int raid5_alloc_percpu(raid5_conf_t *conf)
4643{
4644 unsigned long cpu;
4645 struct page *spare_page;
4646 struct raid5_percpu *allcpus;
4647 void *scribble;
4648 int err;
4649
4650 allcpus = alloc_percpu(struct raid5_percpu);
4651 if (!allcpus)
4652 return -ENOMEM;
4653 conf->percpu = allcpus;
4654
4655 get_online_cpus();
4656 err = 0;
4657 for_each_present_cpu(cpu) {
4658 if (conf->level == 6) {
4659 spare_page = alloc_page(GFP_KERNEL);
4660 if (!spare_page) {
4661 err = -ENOMEM;
4662 break;
4663 }
4664 per_cpu_ptr(conf->percpu, cpu)->spare_page = spare_page;
4665 }
4666 scribble = kmalloc(scribble_len(conf->raid_disks), GFP_KERNEL);
4667 if (!scribble) {
4668 err = -ENOMEM;
4669 break;
4670 }
4671 per_cpu_ptr(conf->percpu, cpu)->scribble = scribble;
4672 }
4673#ifdef CONFIG_HOTPLUG_CPU
4674 conf->cpu_notify.notifier_call = raid456_cpu_notify;
4675 conf->cpu_notify.priority = 0;
4676 if (err == 0)
4677 err = register_cpu_notifier(&conf->cpu_notify);
4678#endif
4679 put_online_cpus();
4680
4681 return err;
4682}
4683
4684static raid5_conf_t *setup_conf(mddev_t *mddev)
4685{
4686 raid5_conf_t *conf;
4687 int raid_disk, memory;
4688 mdk_rdev_t *rdev;
4689 struct disk_info *disk;
4690
4691 if (mddev->new_level != 5
4692 && mddev->new_level != 4
4693 && mddev->new_level != 6) {
4694 printk(KERN_ERR "raid5: %s: raid level not set to 4/5/6 (%d)\n",
4695 mdname(mddev), mddev->new_level);
4696 return ERR_PTR(-EIO);
4697 }
4698 if ((mddev->new_level == 5
4699 && !algorithm_valid_raid5(mddev->new_layout)) ||
4700 (mddev->new_level == 6
4701 && !algorithm_valid_raid6(mddev->new_layout))) {
4702 printk(KERN_ERR "raid5: %s: layout %d not supported\n",
4703 mdname(mddev), mddev->new_layout);
4704 return ERR_PTR(-EIO);
4705 }
4706 if (mddev->new_level == 6 && mddev->raid_disks < 4) {
4707 printk(KERN_ERR "raid6: not enough configured devices for %s (%d, minimum 4)\n",
4708 mdname(mddev), mddev->raid_disks);
4709 return ERR_PTR(-EINVAL);
4710 }
4711
4712 if (!mddev->new_chunk_sectors ||
4713 (mddev->new_chunk_sectors << 9) % PAGE_SIZE ||
4714 !is_power_of_2(mddev->new_chunk_sectors)) {
4715 printk(KERN_ERR "raid5: invalid chunk size %d for %s\n",
4716 mddev->new_chunk_sectors << 9, mdname(mddev));
4717 return ERR_PTR(-EINVAL);
4718 }
4719
4720 conf = kzalloc(sizeof(raid5_conf_t), GFP_KERNEL);
4721 if (conf == NULL)
4722 goto abort;
4723 spin_lock_init(&conf->device_lock);
4724 init_waitqueue_head(&conf->wait_for_stripe);
4725 init_waitqueue_head(&conf->wait_for_overlap);
4726 INIT_LIST_HEAD(&conf->handle_list);
4727 INIT_LIST_HEAD(&conf->hold_list);
4728 INIT_LIST_HEAD(&conf->delayed_list);
4729 INIT_LIST_HEAD(&conf->bitmap_list);
4730 INIT_LIST_HEAD(&conf->inactive_list);
4731 atomic_set(&conf->active_stripes, 0);
4732 atomic_set(&conf->preread_active_stripes, 0);
4733 atomic_set(&conf->active_aligned_reads, 0);
4734 conf->bypass_threshold = BYPASS_THRESHOLD;
4735
4736 conf->raid_disks = mddev->raid_disks;
4737 conf->scribble_len = scribble_len(conf->raid_disks);
4738 if (mddev->reshape_position == MaxSector)
4739 conf->previous_raid_disks = mddev->raid_disks;
4740 else
4741 conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks;
4742
4743 conf->disks = kzalloc(conf->raid_disks * sizeof(struct disk_info),
4744 GFP_KERNEL);
4745 if (!conf->disks)
4746 goto abort;
4747
4748 conf->mddev = mddev;
4749
4750 if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL)
4751 goto abort;
4752
4753 conf->level = mddev->new_level;
4754 if (raid5_alloc_percpu(conf) != 0)
4755 goto abort;
4756
4757 pr_debug("raid5: run(%s) called.\n", mdname(mddev));
4758
4759 list_for_each_entry(rdev, &mddev->disks, same_set) {
4760 raid_disk = rdev->raid_disk;
4761 if (raid_disk >= conf->raid_disks
4762 || raid_disk < 0)
4763 continue;
4764 disk = conf->disks + raid_disk;
4765
4766 disk->rdev = rdev;
4767
4768 if (test_bit(In_sync, &rdev->flags)) {
4769 char b[BDEVNAME_SIZE];
4770 printk(KERN_INFO "raid5: device %s operational as raid"
4771 " disk %d\n", bdevname(rdev->bdev,b),
4772 raid_disk);
4773 } else
4774 /* Cannot rely on bitmap to complete recovery */
4775 conf->fullsync = 1;
4776 }
4777
4778 conf->chunk_sectors = mddev->new_chunk_sectors;
4779 conf->level = mddev->new_level;
4780 if (conf->level == 6)
4781 conf->max_degraded = 2;
4782 else
4783 conf->max_degraded = 1;
4784 conf->algorithm = mddev->new_layout;
4785 conf->max_nr_stripes = NR_STRIPES;
4786 conf->reshape_progress = mddev->reshape_position;
4787 if (conf->reshape_progress != MaxSector) {
4788 conf->prev_chunk_sectors = mddev->chunk_sectors;
4789 conf->prev_algo = mddev->layout;
4790 }
4791
4792 memory = conf->max_nr_stripes * (sizeof(struct stripe_head) +
4793 conf->raid_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024;
4794 if (grow_stripes(conf, conf->max_nr_stripes)) {
4795 printk(KERN_ERR
4796 "raid5: couldn't allocate %dkB for buffers\n", memory);
4797 goto abort;
4798 } else
4799 printk(KERN_INFO "raid5: allocated %dkB for %s\n",
4800 memory, mdname(mddev));
4801
4802 conf->thread = md_register_thread(raid5d, mddev, NULL);
4803 if (!conf->thread) {
4804 printk(KERN_ERR
4805 "raid5: couldn't allocate thread for %s\n",
4806 mdname(mddev));
4807 goto abort;
4808 }
4809
4810 return conf;
4811
4812 abort:
4813 if (conf) {
4814 free_conf(conf);
4815 return ERR_PTR(-EIO);
4816 } else
4817 return ERR_PTR(-ENOMEM);
4818}
4819
4820static int run(mddev_t *mddev)
4821{
4822 raid5_conf_t *conf;
4823 int working_disks = 0, chunk_size;
4824 mdk_rdev_t *rdev;
4825
4826 if (mddev->recovery_cp != MaxSector)
4827 printk(KERN_NOTICE "raid5: %s is not clean"
4828 " -- starting background reconstruction\n",
4829 mdname(mddev));
4830 if (mddev->reshape_position != MaxSector) {
4831 /* Check that we can continue the reshape.
4832 * Currently only disks can change, it must
4833 * increase, and we must be past the point where
4834 * a stripe over-writes itself
4835 */
4836 sector_t here_new, here_old;
4837 int old_disks;
4838 int max_degraded = (mddev->level == 6 ? 2 : 1);
4839
4840 if (mddev->new_level != mddev->level) {
4841 printk(KERN_ERR "raid5: %s: unsupported reshape "
4842 "required - aborting.\n",
4843 mdname(mddev));
4844 return -EINVAL;
4845 }
4846 old_disks = mddev->raid_disks - mddev->delta_disks;
4847 /* reshape_position must be on a new-stripe boundary, and one
4848 * further up in new geometry must map after here in old
4849 * geometry.
4850 */
4851 here_new = mddev->reshape_position;
4852 if (sector_div(here_new, mddev->new_chunk_sectors *
4853 (mddev->raid_disks - max_degraded))) {
4854 printk(KERN_ERR "raid5: reshape_position not "
4855 "on a stripe boundary\n");
4856 return -EINVAL;
4857 }
4858 /* here_new is the stripe we will write to */
4859 here_old = mddev->reshape_position;
4860 sector_div(here_old, mddev->chunk_sectors *
4861 (old_disks-max_degraded));
4862 /* here_old is the first stripe that we might need to read
4863 * from */
4864 if (mddev->delta_disks == 0) {
4865 /* We cannot be sure it is safe to start an in-place
4866 * reshape. It is only safe if user-space if monitoring
4867 * and taking constant backups.
4868 * mdadm always starts a situation like this in
4869 * readonly mode so it can take control before
4870 * allowing any writes. So just check for that.
4871 */
4872 if ((here_new * mddev->new_chunk_sectors !=
4873 here_old * mddev->chunk_sectors) ||
4874 mddev->ro == 0) {
4875 printk(KERN_ERR "raid5: in-place reshape must be started"
4876 " in read-only mode - aborting\n");
4877 return -EINVAL;
4878 }
4879 } else if (mddev->delta_disks < 0
4880 ? (here_new * mddev->new_chunk_sectors <=
4881 here_old * mddev->chunk_sectors)
4882 : (here_new * mddev->new_chunk_sectors >=
4883 here_old * mddev->chunk_sectors)) {
4884 /* Reading from the same stripe as writing to - bad */
4885 printk(KERN_ERR "raid5: reshape_position too early for "
4886 "auto-recovery - aborting.\n");
4887 return -EINVAL;
4888 }
4889 printk(KERN_INFO "raid5: reshape will continue\n");
4890 /* OK, we should be able to continue; */
4891 } else {
4892 BUG_ON(mddev->level != mddev->new_level);
4893 BUG_ON(mddev->layout != mddev->new_layout);
4894 BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors);
4895 BUG_ON(mddev->delta_disks != 0);
4896 }
4897
4898 if (mddev->private == NULL)
4899 conf = setup_conf(mddev);
4900 else
4901 conf = mddev->private;
4902
4903 if (IS_ERR(conf))
4904 return PTR_ERR(conf);
4905
4906 mddev->thread = conf->thread;
4907 conf->thread = NULL;
4908 mddev->private = conf;
4909
4910 /*
4911 * 0 for a fully functional array, 1 or 2 for a degraded array.
4912 */
4913 list_for_each_entry(rdev, &mddev->disks, same_set)
4914 if (rdev->raid_disk >= 0 &&
4915 test_bit(In_sync, &rdev->flags))
4916 working_disks++;
4917
4918 mddev->degraded = conf->raid_disks - working_disks;
4919
4920 if (mddev->degraded > conf->max_degraded) {
4921 printk(KERN_ERR "raid5: not enough operational devices for %s"
4922 " (%d/%d failed)\n",
4923 mdname(mddev), mddev->degraded, conf->raid_disks);
4924 goto abort;
4925 }
4926
4927 /* device size must be a multiple of chunk size */
4928 mddev->dev_sectors &= ~(mddev->chunk_sectors - 1);
4929 mddev->resync_max_sectors = mddev->dev_sectors;
4930
4931 if (mddev->degraded > 0 &&
4932 mddev->recovery_cp != MaxSector) {
4933 if (mddev->ok_start_degraded)
4934 printk(KERN_WARNING
4935 "raid5: starting dirty degraded array: %s"
4936 "- data corruption possible.\n",
4937 mdname(mddev));
4938 else {
4939 printk(KERN_ERR
4940 "raid5: cannot start dirty degraded array for %s\n",
4941 mdname(mddev));
4942 goto abort;
4943 }
4944 }
4945
4946 if (mddev->degraded == 0)
4947 printk("raid5: raid level %d set %s active with %d out of %d"
4948 " devices, algorithm %d\n", conf->level, mdname(mddev),
4949 mddev->raid_disks-mddev->degraded, mddev->raid_disks,
4950 mddev->new_layout);
4951 else
4952 printk(KERN_ALERT "raid5: raid level %d set %s active with %d"
4953 " out of %d devices, algorithm %d\n", conf->level,
4954 mdname(mddev), mddev->raid_disks - mddev->degraded,
4955 mddev->raid_disks, mddev->new_layout);
4956
4957 print_raid5_conf(conf);
4958
4959 if (conf->reshape_progress != MaxSector) {
4960 printk("...ok start reshape thread\n");
4961 conf->reshape_safe = conf->reshape_progress;
4962 atomic_set(&conf->reshape_stripes, 0);
4963 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
4964 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
4965 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
4966 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
4967 mddev->sync_thread = md_register_thread(md_do_sync, mddev,
4968 "reshape");
4969 }
4970
4971 /* read-ahead size must cover two whole stripes, which is
4972 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
4973 */
4974 {
4975 int data_disks = conf->previous_raid_disks - conf->max_degraded;
4976 int stripe = data_disks *
4977 ((mddev->chunk_sectors << 9) / PAGE_SIZE);
4978 if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
4979 mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
4980 }
4981
4982 /* Ok, everything is just fine now */
4983 if (sysfs_create_group(&mddev->kobj, &raid5_attrs_group))
4984 printk(KERN_WARNING
4985 "raid5: failed to create sysfs attributes for %s\n",
4986 mdname(mddev));
4987
4988 mddev->queue->queue_lock = &conf->device_lock;
4989
4990 mddev->queue->unplug_fn = raid5_unplug_device;
4991 mddev->queue->backing_dev_info.congested_data = mddev;
4992 mddev->queue->backing_dev_info.congested_fn = raid5_congested;
4993
4994 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
4995
4996 blk_queue_merge_bvec(mddev->queue, raid5_mergeable_bvec);
4997 chunk_size = mddev->chunk_sectors << 9;
4998 blk_queue_io_min(mddev->queue, chunk_size);
4999 blk_queue_io_opt(mddev->queue, chunk_size *
5000 (conf->raid_disks - conf->max_degraded));
5001
5002 list_for_each_entry(rdev, &mddev->disks, same_set)
5003 disk_stack_limits(mddev->gendisk, rdev->bdev,
5004 rdev->data_offset << 9);
5005
5006 return 0;
5007abort:
5008 md_unregister_thread(mddev->thread);
5009 mddev->thread = NULL;
5010 if (conf) {
5011 print_raid5_conf(conf);
5012 free_conf(conf);
5013 }
5014 mddev->private = NULL;
5015 printk(KERN_ALERT "raid5: failed to run raid set %s\n", mdname(mddev));
5016 return -EIO;
5017}
5018
5019
5020
5021static int stop(mddev_t *mddev)
5022{
5023 raid5_conf_t *conf = (raid5_conf_t *) mddev->private;
5024
5025 md_unregister_thread(mddev->thread);
5026 mddev->thread = NULL;
5027 mddev->queue->backing_dev_info.congested_fn = NULL;
5028 blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/
5029 sysfs_remove_group(&mddev->kobj, &raid5_attrs_group);
5030 free_conf(conf);
5031 mddev->private = NULL;
5032 return 0;
5033}
5034
5035#ifdef DEBUG
5036static void print_sh(struct seq_file *seq, struct stripe_head *sh)
5037{
5038 int i;
5039
5040 seq_printf(seq, "sh %llu, pd_idx %d, state %ld.\n",
5041 (unsigned long long)sh->sector, sh->pd_idx, sh->state);
5042 seq_printf(seq, "sh %llu, count %d.\n",
5043 (unsigned long long)sh->sector, atomic_read(&sh->count));
5044 seq_printf(seq, "sh %llu, ", (unsigned long long)sh->sector);
5045 for (i = 0; i < sh->disks; i++) {
5046 seq_printf(seq, "(cache%d: %p %ld) ",
5047 i, sh->dev[i].page, sh->dev[i].flags);
5048 }
5049 seq_printf(seq, "\n");
5050}
5051
5052static void printall(struct seq_file *seq, raid5_conf_t *conf)
5053{
5054 struct stripe_head *sh;
5055 struct hlist_node *hn;
5056 int i;
5057
5058 spin_lock_irq(&conf->device_lock);
5059 for (i = 0; i < NR_HASH; i++) {
5060 hlist_for_each_entry(sh, hn, &conf->stripe_hashtbl[i], hash) {
5061 if (sh->raid_conf != conf)
5062 continue;
5063 print_sh(seq, sh);
5064 }
5065 }
5066 spin_unlock_irq(&conf->device_lock);
5067}
5068#endif
5069
5070static void status(struct seq_file *seq, mddev_t *mddev)
5071{
5072 raid5_conf_t *conf = (raid5_conf_t *) mddev->private;
5073 int i;
5074
5075 seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level,
5076 mddev->chunk_sectors / 2, mddev->layout);
5077 seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded);
5078 for (i = 0; i < conf->raid_disks; i++)
5079 seq_printf (seq, "%s",
5080 conf->disks[i].rdev &&
5081 test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_");
5082 seq_printf (seq, "]");
5083#ifdef DEBUG
5084 seq_printf (seq, "\n");
5085 printall(seq, conf);
5086#endif
5087}
5088
5089static void print_raid5_conf (raid5_conf_t *conf)
5090{
5091 int i;
5092 struct disk_info *tmp;
5093
5094 printk("RAID5 conf printout:\n");
5095 if (!conf) {
5096 printk("(conf==NULL)\n");
5097 return;
5098 }
5099 printk(" --- rd:%d wd:%d\n", conf->raid_disks,
5100 conf->raid_disks - conf->mddev->degraded);
5101
5102 for (i = 0; i < conf->raid_disks; i++) {
5103 char b[BDEVNAME_SIZE];
5104 tmp = conf->disks + i;
5105 if (tmp->rdev)
5106 printk(" disk %d, o:%d, dev:%s\n",
5107 i, !test_bit(Faulty, &tmp->rdev->flags),
5108 bdevname(tmp->rdev->bdev,b));
5109 }
5110}
5111
5112static int raid5_spare_active(mddev_t *mddev)
5113{
5114 int i;
5115 raid5_conf_t *conf = mddev->private;
5116 struct disk_info *tmp;
5117
5118 for (i = 0; i < conf->raid_disks; i++) {
5119 tmp = conf->disks + i;
5120 if (tmp->rdev
5121 && !test_bit(Faulty, &tmp->rdev->flags)
5122 && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
5123 unsigned long flags;
5124 spin_lock_irqsave(&conf->device_lock, flags);
5125 mddev->degraded--;
5126 spin_unlock_irqrestore(&conf->device_lock, flags);
5127 }
5128 }
5129 print_raid5_conf(conf);
5130 return 0;
5131}
5132
5133static int raid5_remove_disk(mddev_t *mddev, int number)
5134{
5135 raid5_conf_t *conf = mddev->private;
5136 int err = 0;
5137 mdk_rdev_t *rdev;
5138 struct disk_info *p = conf->disks + number;
5139
5140 print_raid5_conf(conf);
5141 rdev = p->rdev;
5142 if (rdev) {
5143 if (number >= conf->raid_disks &&
5144 conf->reshape_progress == MaxSector)
5145 clear_bit(In_sync, &rdev->flags);
5146
5147 if (test_bit(In_sync, &rdev->flags) ||
5148 atomic_read(&rdev->nr_pending)) {
5149 err = -EBUSY;
5150 goto abort;
5151 }
5152 /* Only remove non-faulty devices if recovery
5153 * isn't possible.
5154 */
5155 if (!test_bit(Faulty, &rdev->flags) &&
5156 mddev->degraded <= conf->max_degraded &&
5157 number < conf->raid_disks) {
5158 err = -EBUSY;
5159 goto abort;
5160 }
5161 p->rdev = NULL;
5162 synchronize_rcu();
5163 if (atomic_read(&rdev->nr_pending)) {
5164 /* lost the race, try later */
5165 err = -EBUSY;
5166 p->rdev = rdev;
5167 }
5168 }
5169abort:
5170
5171 print_raid5_conf(conf);
5172 return err;
5173}
5174
5175static int raid5_add_disk(mddev_t *mddev, mdk_rdev_t *rdev)
5176{
5177 raid5_conf_t *conf = mddev->private;
5178 int err = -EEXIST;
5179 int disk;
5180 struct disk_info *p;
5181 int first = 0;
5182 int last = conf->raid_disks - 1;
5183
5184 if (mddev->degraded > conf->max_degraded)
5185 /* no point adding a device */
5186 return -EINVAL;
5187
5188 if (rdev->raid_disk >= 0)
5189 first = last = rdev->raid_disk;
5190
5191 /*
5192 * find the disk ... but prefer rdev->saved_raid_disk
5193 * if possible.
5194 */
5195 if (rdev->saved_raid_disk >= 0 &&
5196 rdev->saved_raid_disk >= first &&
5197 conf->disks[rdev->saved_raid_disk].rdev == NULL)
5198 disk = rdev->saved_raid_disk;
5199 else
5200 disk = first;
5201 for ( ; disk <= last ; disk++)
5202 if ((p=conf->disks + disk)->rdev == NULL) {
5203 clear_bit(In_sync, &rdev->flags);
5204 rdev->raid_disk = disk;
5205 err = 0;
5206 if (rdev->saved_raid_disk != disk)
5207 conf->fullsync = 1;
5208 rcu_assign_pointer(p->rdev, rdev);
5209 break;
5210 }
5211 print_raid5_conf(conf);
5212 return err;
5213}
5214
5215static int raid5_resize(mddev_t *mddev, sector_t sectors)
5216{
5217 /* no resync is happening, and there is enough space
5218 * on all devices, so we can resize.
5219 * We need to make sure resync covers any new space.
5220 * If the array is shrinking we should possibly wait until
5221 * any io in the removed space completes, but it hardly seems
5222 * worth it.
5223 */
5224 sectors &= ~((sector_t)mddev->chunk_sectors - 1);
5225 md_set_array_sectors(mddev, raid5_size(mddev, sectors,
5226 mddev->raid_disks));
5227 if (mddev->array_sectors >
5228 raid5_size(mddev, sectors, mddev->raid_disks))
5229 return -EINVAL;
5230 set_capacity(mddev->gendisk, mddev->array_sectors);
5231 mddev->changed = 1;
5232 revalidate_disk(mddev->gendisk);
5233 if (sectors > mddev->dev_sectors && mddev->recovery_cp == MaxSector) {
5234 mddev->recovery_cp = mddev->dev_sectors;
5235 set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
5236 }
5237 mddev->dev_sectors = sectors;
5238 mddev->resync_max_sectors = sectors;
5239 return 0;
5240}
5241
5242static int check_stripe_cache(mddev_t *mddev)
5243{
5244 /* Can only proceed if there are plenty of stripe_heads.
5245 * We need a minimum of one full stripe,, and for sensible progress
5246 * it is best to have about 4 times that.
5247 * If we require 4 times, then the default 256 4K stripe_heads will
5248 * allow for chunk sizes up to 256K, which is probably OK.
5249 * If the chunk size is greater, user-space should request more
5250 * stripe_heads first.
5251 */
5252 raid5_conf_t *conf = mddev->private;
5253 if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4
5254 > conf->max_nr_stripes ||
5255 ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4
5256 > conf->max_nr_stripes) {
5257 printk(KERN_WARNING "raid5: reshape: not enough stripes. Needed %lu\n",
5258 ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)
5259 / STRIPE_SIZE)*4);
5260 return 0;
5261 }
5262 return 1;
5263}
5264
5265static int check_reshape(mddev_t *mddev)
5266{
5267 raid5_conf_t *conf = mddev->private;
5268
5269 if (mddev->delta_disks == 0 &&
5270 mddev->new_layout == mddev->layout &&
5271 mddev->new_chunk_sectors == mddev->chunk_sectors)
5272 return 0; /* nothing to do */
5273 if (mddev->bitmap)
5274 /* Cannot grow a bitmap yet */
5275 return -EBUSY;
5276 if (mddev->degraded > conf->max_degraded)
5277 return -EINVAL;
5278 if (mddev->delta_disks < 0) {
5279 /* We might be able to shrink, but the devices must
5280 * be made bigger first.
5281 * For raid6, 4 is the minimum size.
5282 * Otherwise 2 is the minimum
5283 */
5284 int min = 2;
5285 if (mddev->level == 6)
5286 min = 4;
5287 if (mddev->raid_disks + mddev->delta_disks < min)
5288 return -EINVAL;
5289 }
5290
5291 if (!check_stripe_cache(mddev))
5292 return -ENOSPC;
5293
5294 return resize_stripes(conf, conf->raid_disks + mddev->delta_disks);
5295}
5296
5297static int raid5_start_reshape(mddev_t *mddev)
5298{
5299 raid5_conf_t *conf = mddev->private;
5300 mdk_rdev_t *rdev;
5301 int spares = 0;
5302 int added_devices = 0;
5303 unsigned long flags;
5304
5305 if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
5306 return -EBUSY;
5307
5308 if (!check_stripe_cache(mddev))
5309 return -ENOSPC;
5310
5311 list_for_each_entry(rdev, &mddev->disks, same_set)
5312 if (rdev->raid_disk < 0 &&
5313 !test_bit(Faulty, &rdev->flags))
5314 spares++;
5315
5316 if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded)
5317 /* Not enough devices even to make a degraded array
5318 * of that size
5319 */
5320 return -EINVAL;
5321
5322 /* Refuse to reduce size of the array. Any reductions in
5323 * array size must be through explicit setting of array_size
5324 * attribute.
5325 */
5326 if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks)
5327 < mddev->array_sectors) {
5328 printk(KERN_ERR "md: %s: array size must be reduced "
5329 "before number of disks\n", mdname(mddev));
5330 return -EINVAL;
5331 }
5332
5333 atomic_set(&conf->reshape_stripes, 0);
5334 spin_lock_irq(&conf->device_lock);
5335 conf->previous_raid_disks = conf->raid_disks;
5336 conf->raid_disks += mddev->delta_disks;
5337 conf->prev_chunk_sectors = conf->chunk_sectors;
5338 conf->chunk_sectors = mddev->new_chunk_sectors;
5339 conf->prev_algo = conf->algorithm;
5340 conf->algorithm = mddev->new_layout;
5341 if (mddev->delta_disks < 0)
5342 conf->reshape_progress = raid5_size(mddev, 0, 0);
5343 else
5344 conf->reshape_progress = 0;
5345 conf->reshape_safe = conf->reshape_progress;
5346 conf->generation++;
5347 spin_unlock_irq(&conf->device_lock);
5348
5349 /* Add some new drives, as many as will fit.
5350 * We know there are enough to make the newly sized array work.
5351 */
5352 list_for_each_entry(rdev, &mddev->disks, same_set)
5353 if (rdev->raid_disk < 0 &&
5354 !test_bit(Faulty, &rdev->flags)) {
5355 if (raid5_add_disk(mddev, rdev) == 0) {
5356 char nm[20];
5357 set_bit(In_sync, &rdev->flags);
5358 added_devices++;
5359 rdev->recovery_offset = 0;
5360 sprintf(nm, "rd%d", rdev->raid_disk);
5361 if (sysfs_create_link(&mddev->kobj,
5362 &rdev->kobj, nm))
5363 printk(KERN_WARNING
5364 "raid5: failed to create "
5365 " link %s for %s\n",
5366 nm, mdname(mddev));
5367 } else
5368 break;
5369 }
5370
5371 if (mddev->delta_disks > 0) {
5372 spin_lock_irqsave(&conf->device_lock, flags);
5373 mddev->degraded = (conf->raid_disks - conf->previous_raid_disks)
5374 - added_devices;
5375 spin_unlock_irqrestore(&conf->device_lock, flags);
5376 }
5377 mddev->raid_disks = conf->raid_disks;
5378 mddev->reshape_position = conf->reshape_progress;
5379 set_bit(MD_CHANGE_DEVS, &mddev->flags);
5380
5381 clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
5382 clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
5383 set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
5384 set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
5385 mddev->sync_thread = md_register_thread(md_do_sync, mddev,
5386 "reshape");
5387 if (!mddev->sync_thread) {
5388 mddev->recovery = 0;
5389 spin_lock_irq(&conf->device_lock);
5390 mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks;
5391 conf->reshape_progress = MaxSector;
5392 spin_unlock_irq(&conf->device_lock);
5393 return -EAGAIN;
5394 }
5395 conf->reshape_checkpoint = jiffies;
5396 md_wakeup_thread(mddev->sync_thread);
5397 md_new_event(mddev);
5398 return 0;
5399}
5400
5401/* This is called from the reshape thread and should make any
5402 * changes needed in 'conf'
5403 */
5404static void end_reshape(raid5_conf_t *conf)
5405{
5406
5407 if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) {
5408
5409 spin_lock_irq(&conf->device_lock);
5410 conf->previous_raid_disks = conf->raid_disks;
5411 conf->reshape_progress = MaxSector;
5412 spin_unlock_irq(&conf->device_lock);
5413 wake_up(&conf->wait_for_overlap);
5414
5415 /* read-ahead size must cover two whole stripes, which is
5416 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
5417 */
5418 {
5419 int data_disks = conf->raid_disks - conf->max_degraded;
5420 int stripe = data_disks * ((conf->chunk_sectors << 9)
5421 / PAGE_SIZE);
5422 if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
5423 conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
5424 }
5425 }
5426}
5427
5428/* This is called from the raid5d thread with mddev_lock held.
5429 * It makes config changes to the device.
5430 */
5431static void raid5_finish_reshape(mddev_t *mddev)
5432{
5433 raid5_conf_t *conf = mddev->private;
5434
5435 if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
5436
5437 if (mddev->delta_disks > 0) {
5438 md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
5439 set_capacity(mddev->gendisk, mddev->array_sectors);
5440 mddev->changed = 1;
5441 revalidate_disk(mddev->gendisk);
5442 } else {
5443 int d;
5444 mddev->degraded = conf->raid_disks;
5445 for (d = 0; d < conf->raid_disks ; d++)
5446 if (conf->disks[d].rdev &&
5447 test_bit(In_sync,
5448 &conf->disks[d].rdev->flags))
5449 mddev->degraded--;
5450 for (d = conf->raid_disks ;
5451 d < conf->raid_disks - mddev->delta_disks;
5452 d++) {
5453 mdk_rdev_t *rdev = conf->disks[d].rdev;
5454 if (rdev && raid5_remove_disk(mddev, d) == 0) {
5455 char nm[20];
5456 sprintf(nm, "rd%d", rdev->raid_disk);
5457 sysfs_remove_link(&mddev->kobj, nm);
5458 rdev->raid_disk = -1;
5459 }
5460 }
5461 }
5462 mddev->layout = conf->algorithm;
5463 mddev->chunk_sectors = conf->chunk_sectors;
5464 mddev->reshape_position = MaxSector;
5465 mddev->delta_disks = 0;
5466 }
5467}
5468
5469static void raid5_quiesce(mddev_t *mddev, int state)
5470{
5471 raid5_conf_t *conf = mddev->private;
5472
5473 switch(state) {
5474 case 2: /* resume for a suspend */
5475 wake_up(&conf->wait_for_overlap);
5476 break;
5477
5478 case 1: /* stop all writes */
5479 spin_lock_irq(&conf->device_lock);
5480 /* '2' tells resync/reshape to pause so that all
5481 * active stripes can drain
5482 */
5483 conf->quiesce = 2;
5484 wait_event_lock_irq(conf->wait_for_stripe,
5485 atomic_read(&conf->active_stripes) == 0 &&
5486 atomic_read(&conf->active_aligned_reads) == 0,
5487 conf->device_lock, /* nothing */);
5488 conf->quiesce = 1;
5489 spin_unlock_irq(&conf->device_lock);
5490 /* allow reshape to continue */
5491 wake_up(&conf->wait_for_overlap);
5492 break;
5493
5494 case 0: /* re-enable writes */
5495 spin_lock_irq(&conf->device_lock);
5496 conf->quiesce = 0;
5497 wake_up(&conf->wait_for_stripe);
5498 wake_up(&conf->wait_for_overlap);
5499 spin_unlock_irq(&conf->device_lock);
5500 break;
5501 }
5502}
5503
5504
5505static void *raid5_takeover_raid1(mddev_t *mddev)
5506{
5507 int chunksect;
5508
5509 if (mddev->raid_disks != 2 ||
5510 mddev->degraded > 1)
5511 return ERR_PTR(-EINVAL);
5512
5513 /* Should check if there are write-behind devices? */
5514
5515 chunksect = 64*2; /* 64K by default */
5516
5517 /* The array must be an exact multiple of chunksize */
5518 while (chunksect && (mddev->array_sectors & (chunksect-1)))
5519 chunksect >>= 1;
5520
5521 if ((chunksect<<9) < STRIPE_SIZE)
5522 /* array size does not allow a suitable chunk size */
5523 return ERR_PTR(-EINVAL);
5524
5525 mddev->new_level = 5;
5526 mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC;
5527 mddev->new_chunk_sectors = chunksect;
5528
5529 return setup_conf(mddev);
5530}
5531
5532static void *raid5_takeover_raid6(mddev_t *mddev)
5533{
5534 int new_layout;
5535
5536 switch (mddev->layout) {
5537 case ALGORITHM_LEFT_ASYMMETRIC_6:
5538 new_layout = ALGORITHM_LEFT_ASYMMETRIC;
5539 break;
5540 case ALGORITHM_RIGHT_ASYMMETRIC_6:
5541 new_layout = ALGORITHM_RIGHT_ASYMMETRIC;
5542 break;
5543 case ALGORITHM_LEFT_SYMMETRIC_6:
5544 new_layout = ALGORITHM_LEFT_SYMMETRIC;
5545 break;
5546 case ALGORITHM_RIGHT_SYMMETRIC_6:
5547 new_layout = ALGORITHM_RIGHT_SYMMETRIC;
5548 break;
5549 case ALGORITHM_PARITY_0_6:
5550 new_layout = ALGORITHM_PARITY_0;
5551 break;
5552 case ALGORITHM_PARITY_N:
5553 new_layout = ALGORITHM_PARITY_N;
5554 break;
5555 default:
5556 return ERR_PTR(-EINVAL);
5557 }
5558 mddev->new_level = 5;
5559 mddev->new_layout = new_layout;
5560 mddev->delta_disks = -1;
5561 mddev->raid_disks -= 1;
5562 return setup_conf(mddev);
5563}
5564
5565
5566static int raid5_check_reshape(mddev_t *mddev)
5567{
5568 /* For a 2-drive array, the layout and chunk size can be changed
5569 * immediately as not restriping is needed.
5570 * For larger arrays we record the new value - after validation
5571 * to be used by a reshape pass.
5572 */
5573 raid5_conf_t *conf = mddev->private;
5574 int new_chunk = mddev->new_chunk_sectors;
5575
5576 if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout))
5577 return -EINVAL;
5578 if (new_chunk > 0) {
5579 if (!is_power_of_2(new_chunk))
5580 return -EINVAL;
5581 if (new_chunk < (PAGE_SIZE>>9))
5582 return -EINVAL;
5583 if (mddev->array_sectors & (new_chunk-1))
5584 /* not factor of array size */
5585 return -EINVAL;
5586 }
5587
5588 /* They look valid */
5589
5590 if (mddev->raid_disks == 2) {
5591 /* can make the change immediately */
5592 if (mddev->new_layout >= 0) {
5593 conf->algorithm = mddev->new_layout;
5594 mddev->layout = mddev->new_layout;
5595 }
5596 if (new_chunk > 0) {
5597 conf->chunk_sectors = new_chunk ;
5598 mddev->chunk_sectors = new_chunk;
5599 }
5600 set_bit(MD_CHANGE_DEVS, &mddev->flags);
5601 md_wakeup_thread(mddev->thread);
5602 }
5603 return check_reshape(mddev);
5604}
5605
5606static int raid6_check_reshape(mddev_t *mddev)
5607{
5608 int new_chunk = mddev->new_chunk_sectors;
5609
5610 if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout))
5611 return -EINVAL;
5612 if (new_chunk > 0) {
5613 if (!is_power_of_2(new_chunk))
5614 return -EINVAL;
5615 if (new_chunk < (PAGE_SIZE >> 9))
5616 return -EINVAL;
5617 if (mddev->array_sectors & (new_chunk-1))
5618 /* not factor of array size */
5619 return -EINVAL;
5620 }
5621
5622 /* They look valid */
5623 return check_reshape(mddev);
5624}
5625
5626static void *raid5_takeover(mddev_t *mddev)
5627{
5628 /* raid5 can take over:
5629 * raid0 - if all devices are the same - make it a raid4 layout
5630 * raid1 - if there are two drives. We need to know the chunk size
5631 * raid4 - trivial - just use a raid4 layout.
5632 * raid6 - Providing it is a *_6 layout
5633 */
5634
5635 if (mddev->level == 1)
5636 return raid5_takeover_raid1(mddev);
5637 if (mddev->level == 4) {
5638 mddev->new_layout = ALGORITHM_PARITY_N;
5639 mddev->new_level = 5;
5640 return setup_conf(mddev);
5641 }
5642 if (mddev->level == 6)
5643 return raid5_takeover_raid6(mddev);
5644
5645 return ERR_PTR(-EINVAL);
5646}
5647
5648
5649static struct mdk_personality raid5_personality;
5650
5651static void *raid6_takeover(mddev_t *mddev)
5652{
5653 /* Currently can only take over a raid5. We map the
5654 * personality to an equivalent raid6 personality
5655 * with the Q block at the end.
5656 */
5657 int new_layout;
5658
5659 if (mddev->pers != &raid5_personality)
5660 return ERR_PTR(-EINVAL);
5661 if (mddev->degraded > 1)
5662 return ERR_PTR(-EINVAL);
5663 if (mddev->raid_disks > 253)
5664 return ERR_PTR(-EINVAL);
5665 if (mddev->raid_disks < 3)
5666 return ERR_PTR(-EINVAL);
5667
5668 switch (mddev->layout) {
5669 case ALGORITHM_LEFT_ASYMMETRIC:
5670 new_layout = ALGORITHM_LEFT_ASYMMETRIC_6;
5671 break;
5672 case ALGORITHM_RIGHT_ASYMMETRIC:
5673 new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6;
5674 break;
5675 case ALGORITHM_LEFT_SYMMETRIC:
5676 new_layout = ALGORITHM_LEFT_SYMMETRIC_6;
5677 break;
5678 case ALGORITHM_RIGHT_SYMMETRIC:
5679 new_layout = ALGORITHM_RIGHT_SYMMETRIC_6;
5680 break;
5681 case ALGORITHM_PARITY_0:
5682 new_layout = ALGORITHM_PARITY_0_6;
5683 break;
5684 case ALGORITHM_PARITY_N:
5685 new_layout = ALGORITHM_PARITY_N;
5686 break;
5687 default:
5688 return ERR_PTR(-EINVAL);
5689 }
5690 mddev->new_level = 6;
5691 mddev->new_layout = new_layout;
5692 mddev->delta_disks = 1;
5693 mddev->raid_disks += 1;
5694 return setup_conf(mddev);
5695}
5696
5697
5698static struct mdk_personality raid6_personality =
5699{
5700 .name = "raid6",
5701 .level = 6,
5702 .owner = THIS_MODULE,
5703 .make_request = make_request,
5704 .run = run,
5705 .stop = stop,
5706 .status = status,
5707 .error_handler = error,
5708 .hot_add_disk = raid5_add_disk,
5709 .hot_remove_disk= raid5_remove_disk,
5710 .spare_active = raid5_spare_active,
5711 .sync_request = sync_request,
5712 .resize = raid5_resize,
5713 .size = raid5_size,
5714 .check_reshape = raid6_check_reshape,
5715 .start_reshape = raid5_start_reshape,
5716 .finish_reshape = raid5_finish_reshape,
5717 .quiesce = raid5_quiesce,
5718 .takeover = raid6_takeover,
5719};
5720static struct mdk_personality raid5_personality =
5721{
5722 .name = "raid5",
5723 .level = 5,
5724 .owner = THIS_MODULE,
5725 .make_request = make_request,
5726 .run = run,
5727 .stop = stop,
5728 .status = status,
5729 .error_handler = error,
5730 .hot_add_disk = raid5_add_disk,
5731 .hot_remove_disk= raid5_remove_disk,
5732 .spare_active = raid5_spare_active,
5733 .sync_request = sync_request,
5734 .resize = raid5_resize,
5735 .size = raid5_size,
5736 .check_reshape = raid5_check_reshape,
5737 .start_reshape = raid5_start_reshape,
5738 .finish_reshape = raid5_finish_reshape,
5739 .quiesce = raid5_quiesce,
5740 .takeover = raid5_takeover,
5741};
5742
5743static struct mdk_personality raid4_personality =
5744{
5745 .name = "raid4",
5746 .level = 4,
5747 .owner = THIS_MODULE,
5748 .make_request = make_request,
5749 .run = run,
5750 .stop = stop,
5751 .status = status,
5752 .error_handler = error,
5753 .hot_add_disk = raid5_add_disk,
5754 .hot_remove_disk= raid5_remove_disk,
5755 .spare_active = raid5_spare_active,
5756 .sync_request = sync_request,
5757 .resize = raid5_resize,
5758 .size = raid5_size,
5759 .check_reshape = raid5_check_reshape,
5760 .start_reshape = raid5_start_reshape,
5761 .finish_reshape = raid5_finish_reshape,
5762 .quiesce = raid5_quiesce,
5763};
5764
5765static int __init raid5_init(void)
5766{
5767 register_md_personality(&raid6_personality);
5768 register_md_personality(&raid5_personality);
5769 register_md_personality(&raid4_personality);
5770 return 0;
5771}
5772
5773static void raid5_exit(void)
5774{
5775 unregister_md_personality(&raid6_personality);
5776 unregister_md_personality(&raid5_personality);
5777 unregister_md_personality(&raid4_personality);
5778}
5779
5780module_init(raid5_init);
5781module_exit(raid5_exit);
5782MODULE_LICENSE("GPL");
5783MODULE_ALIAS("md-personality-4"); /* RAID5 */
5784MODULE_ALIAS("md-raid5");
5785MODULE_ALIAS("md-raid4");
5786MODULE_ALIAS("md-level-5");
5787MODULE_ALIAS("md-level-4");
5788MODULE_ALIAS("md-personality-8"); /* RAID6 */
5789MODULE_ALIAS("md-raid6");
5790MODULE_ALIAS("md-level-6");
5791
5792/* This used to be two separate modules, they were: */
5793MODULE_ALIAS("raid5");
5794MODULE_ALIAS("raid6");
Ubuntu Artful kernel mirror