2 * bcache setup/teardown code, and some metadata io - read a superblock and
3 * figure out what to do with it.
5 * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
6 * Copyright 2012 Google, Inc.
14 #include "writeback.h"
16 #include <linux/blkdev.h>
17 #include <linux/buffer_head.h>
18 #include <linux/debugfs.h>
19 #include <linux/genhd.h>
20 #include <linux/idr.h>
21 #include <linux/kthread.h>
22 #include <linux/module.h>
23 #include <linux/random.h>
24 #include <linux/reboot.h>
25 #include <linux/sysfs.h>
27 MODULE_LICENSE("GPL");
28 MODULE_AUTHOR("Kent Overstreet <kent.overstreet@gmail.com>");
30 static const char bcache_magic
[] = {
31 0xc6, 0x85, 0x73, 0xf6, 0x4e, 0x1a, 0x45, 0xca,
32 0x82, 0x65, 0xf5, 0x7f, 0x48, 0xba, 0x6d, 0x81
35 static const char invalid_uuid
[] = {
36 0xa0, 0x3e, 0xf8, 0xed, 0x3e, 0xe1, 0xb8, 0x78,
37 0xc8, 0x50, 0xfc, 0x5e, 0xcb, 0x16, 0xcd, 0x99
40 static struct kobject
*bcache_kobj
;
41 struct mutex bch_register_lock
;
42 LIST_HEAD(bch_cache_sets
);
43 static LIST_HEAD(uncached_devices
);
45 static int bcache_major
;
46 static DEFINE_IDA(bcache_device_idx
);
47 static wait_queue_head_t unregister_wait
;
48 struct workqueue_struct
*bcache_wq
;
50 #define BTREE_MAX_PAGES (256 * 1024 / PAGE_SIZE)
51 /* limitation of partitions number on single bcache device */
52 #define BCACHE_MINORS 128
53 /* limitation of bcache devices number on single system */
54 #define BCACHE_DEVICE_IDX_MAX ((1U << MINORBITS)/BCACHE_MINORS)
58 static const char *read_super(struct cache_sb
*sb
, struct block_device
*bdev
,
63 struct buffer_head
*bh
= __bread(bdev
, 1, SB_SIZE
);
69 s
= (struct cache_sb
*) bh
->b_data
;
71 sb
->offset
= le64_to_cpu(s
->offset
);
72 sb
->version
= le64_to_cpu(s
->version
);
74 memcpy(sb
->magic
, s
->magic
, 16);
75 memcpy(sb
->uuid
, s
->uuid
, 16);
76 memcpy(sb
->set_uuid
, s
->set_uuid
, 16);
77 memcpy(sb
->label
, s
->label
, SB_LABEL_SIZE
);
79 sb
->flags
= le64_to_cpu(s
->flags
);
80 sb
->seq
= le64_to_cpu(s
->seq
);
81 sb
->last_mount
= le32_to_cpu(s
->last_mount
);
82 sb
->first_bucket
= le16_to_cpu(s
->first_bucket
);
83 sb
->keys
= le16_to_cpu(s
->keys
);
85 for (i
= 0; i
< SB_JOURNAL_BUCKETS
; i
++)
86 sb
->d
[i
] = le64_to_cpu(s
->d
[i
]);
88 pr_debug("read sb version %llu, flags %llu, seq %llu, journal size %u",
89 sb
->version
, sb
->flags
, sb
->seq
, sb
->keys
);
91 err
= "Not a bcache superblock";
92 if (sb
->offset
!= SB_SECTOR
)
95 if (memcmp(sb
->magic
, bcache_magic
, 16))
98 err
= "Too many journal buckets";
99 if (sb
->keys
> SB_JOURNAL_BUCKETS
)
102 err
= "Bad checksum";
103 if (s
->csum
!= csum_set(s
))
107 if (bch_is_zero(sb
->uuid
, 16))
110 sb
->block_size
= le16_to_cpu(s
->block_size
);
112 err
= "Superblock block size smaller than device block size";
113 if (sb
->block_size
<< 9 < bdev_logical_block_size(bdev
))
116 switch (sb
->version
) {
117 case BCACHE_SB_VERSION_BDEV
:
118 sb
->data_offset
= BDEV_DATA_START_DEFAULT
;
120 case BCACHE_SB_VERSION_BDEV_WITH_OFFSET
:
121 sb
->data_offset
= le64_to_cpu(s
->data_offset
);
123 err
= "Bad data offset";
124 if (sb
->data_offset
< BDEV_DATA_START_DEFAULT
)
128 case BCACHE_SB_VERSION_CDEV
:
129 case BCACHE_SB_VERSION_CDEV_WITH_UUID
:
130 sb
->nbuckets
= le64_to_cpu(s
->nbuckets
);
131 sb
->bucket_size
= le16_to_cpu(s
->bucket_size
);
133 sb
->nr_in_set
= le16_to_cpu(s
->nr_in_set
);
134 sb
->nr_this_dev
= le16_to_cpu(s
->nr_this_dev
);
136 err
= "Too many buckets";
137 if (sb
->nbuckets
> LONG_MAX
)
140 err
= "Not enough buckets";
141 if (sb
->nbuckets
< 1 << 7)
144 err
= "Bad block/bucket size";
145 if (!is_power_of_2(sb
->block_size
) ||
146 sb
->block_size
> PAGE_SECTORS
||
147 !is_power_of_2(sb
->bucket_size
) ||
148 sb
->bucket_size
< PAGE_SECTORS
)
151 err
= "Invalid superblock: device too small";
152 if (get_capacity(bdev
->bd_disk
) < sb
->bucket_size
* sb
->nbuckets
)
156 if (bch_is_zero(sb
->set_uuid
, 16))
159 err
= "Bad cache device number in set";
160 if (!sb
->nr_in_set
||
161 sb
->nr_in_set
<= sb
->nr_this_dev
||
162 sb
->nr_in_set
> MAX_CACHES_PER_SET
)
165 err
= "Journal buckets not sequential";
166 for (i
= 0; i
< sb
->keys
; i
++)
167 if (sb
->d
[i
] != sb
->first_bucket
+ i
)
170 err
= "Too many journal buckets";
171 if (sb
->first_bucket
+ sb
->keys
> sb
->nbuckets
)
174 err
= "Invalid superblock: first bucket comes before end of super";
175 if (sb
->first_bucket
* sb
->bucket_size
< 16)
180 err
= "Unsupported superblock version";
184 sb
->last_mount
= get_seconds();
187 get_page(bh
->b_page
);
194 static void write_bdev_super_endio(struct bio
*bio
)
196 struct cached_dev
*dc
= bio
->bi_private
;
197 /* XXX: error checking */
199 closure_put(&dc
->sb_write
);
202 static void __write_super(struct cache_sb
*sb
, struct bio
*bio
)
204 struct cache_sb
*out
= page_address(bio
->bi_io_vec
[0].bv_page
);
207 bio
->bi_iter
.bi_sector
= SB_SECTOR
;
208 bio
->bi_iter
.bi_size
= SB_SIZE
;
209 bio_set_op_attrs(bio
, REQ_OP_WRITE
, REQ_SYNC
|REQ_META
);
210 bch_bio_map(bio
, NULL
);
212 out
->offset
= cpu_to_le64(sb
->offset
);
213 out
->version
= cpu_to_le64(sb
->version
);
215 memcpy(out
->uuid
, sb
->uuid
, 16);
216 memcpy(out
->set_uuid
, sb
->set_uuid
, 16);
217 memcpy(out
->label
, sb
->label
, SB_LABEL_SIZE
);
219 out
->flags
= cpu_to_le64(sb
->flags
);
220 out
->seq
= cpu_to_le64(sb
->seq
);
222 out
->last_mount
= cpu_to_le32(sb
->last_mount
);
223 out
->first_bucket
= cpu_to_le16(sb
->first_bucket
);
224 out
->keys
= cpu_to_le16(sb
->keys
);
226 for (i
= 0; i
< sb
->keys
; i
++)
227 out
->d
[i
] = cpu_to_le64(sb
->d
[i
]);
229 out
->csum
= csum_set(out
);
231 pr_debug("ver %llu, flags %llu, seq %llu",
232 sb
->version
, sb
->flags
, sb
->seq
);
237 static void bch_write_bdev_super_unlock(struct closure
*cl
)
239 struct cached_dev
*dc
= container_of(cl
, struct cached_dev
, sb_write
);
241 up(&dc
->sb_write_mutex
);
244 void bch_write_bdev_super(struct cached_dev
*dc
, struct closure
*parent
)
246 struct closure
*cl
= &dc
->sb_write
;
247 struct bio
*bio
= &dc
->sb_bio
;
249 down(&dc
->sb_write_mutex
);
250 closure_init(cl
, parent
);
253 bio_set_dev(bio
, dc
->bdev
);
254 bio
->bi_end_io
= write_bdev_super_endio
;
255 bio
->bi_private
= dc
;
258 /* I/O request sent to backing device */
259 __write_super(&dc
->sb
, bio
);
261 closure_return_with_destructor(cl
, bch_write_bdev_super_unlock
);
264 static void write_super_endio(struct bio
*bio
)
266 struct cache
*ca
= bio
->bi_private
;
268 bch_count_io_errors(ca
, bio
->bi_status
, "writing superblock");
269 closure_put(&ca
->set
->sb_write
);
272 static void bcache_write_super_unlock(struct closure
*cl
)
274 struct cache_set
*c
= container_of(cl
, struct cache_set
, sb_write
);
276 up(&c
->sb_write_mutex
);
279 void bcache_write_super(struct cache_set
*c
)
281 struct closure
*cl
= &c
->sb_write
;
285 down(&c
->sb_write_mutex
);
286 closure_init(cl
, &c
->cl
);
290 for_each_cache(ca
, c
, i
) {
291 struct bio
*bio
= &ca
->sb_bio
;
293 ca
->sb
.version
= BCACHE_SB_VERSION_CDEV_WITH_UUID
;
294 ca
->sb
.seq
= c
->sb
.seq
;
295 ca
->sb
.last_mount
= c
->sb
.last_mount
;
297 SET_CACHE_SYNC(&ca
->sb
, CACHE_SYNC(&c
->sb
));
300 bio_set_dev(bio
, ca
->bdev
);
301 bio
->bi_end_io
= write_super_endio
;
302 bio
->bi_private
= ca
;
305 __write_super(&ca
->sb
, bio
);
308 closure_return_with_destructor(cl
, bcache_write_super_unlock
);
313 static void uuid_endio(struct bio
*bio
)
315 struct closure
*cl
= bio
->bi_private
;
316 struct cache_set
*c
= container_of(cl
, struct cache_set
, uuid_write
);
318 cache_set_err_on(bio
->bi_status
, c
, "accessing uuids");
319 bch_bbio_free(bio
, c
);
323 static void uuid_io_unlock(struct closure
*cl
)
325 struct cache_set
*c
= container_of(cl
, struct cache_set
, uuid_write
);
327 up(&c
->uuid_write_mutex
);
330 static void uuid_io(struct cache_set
*c
, int op
, unsigned long op_flags
,
331 struct bkey
*k
, struct closure
*parent
)
333 struct closure
*cl
= &c
->uuid_write
;
334 struct uuid_entry
*u
;
339 down(&c
->uuid_write_mutex
);
340 closure_init(cl
, parent
);
342 for (i
= 0; i
< KEY_PTRS(k
); i
++) {
343 struct bio
*bio
= bch_bbio_alloc(c
);
345 bio
->bi_opf
= REQ_SYNC
| REQ_META
| op_flags
;
346 bio
->bi_iter
.bi_size
= KEY_SIZE(k
) << 9;
348 bio
->bi_end_io
= uuid_endio
;
349 bio
->bi_private
= cl
;
350 bio_set_op_attrs(bio
, op
, REQ_SYNC
|REQ_META
|op_flags
);
351 bch_bio_map(bio
, c
->uuids
);
353 bch_submit_bbio(bio
, c
, k
, i
);
355 if (op
!= REQ_OP_WRITE
)
359 bch_extent_to_text(buf
, sizeof(buf
), k
);
360 pr_debug("%s UUIDs at %s", op
== REQ_OP_WRITE
? "wrote" : "read", buf
);
362 for (u
= c
->uuids
; u
< c
->uuids
+ c
->nr_uuids
; u
++)
363 if (!bch_is_zero(u
->uuid
, 16))
364 pr_debug("Slot %zi: %pU: %s: 1st: %u last: %u inv: %u",
365 u
- c
->uuids
, u
->uuid
, u
->label
,
366 u
->first_reg
, u
->last_reg
, u
->invalidated
);
368 closure_return_with_destructor(cl
, uuid_io_unlock
);
371 static char *uuid_read(struct cache_set
*c
, struct jset
*j
, struct closure
*cl
)
373 struct bkey
*k
= &j
->uuid_bucket
;
375 if (__bch_btree_ptr_invalid(c
, k
))
376 return "bad uuid pointer";
378 bkey_copy(&c
->uuid_bucket
, k
);
379 uuid_io(c
, REQ_OP_READ
, 0, k
, cl
);
381 if (j
->version
< BCACHE_JSET_VERSION_UUIDv1
) {
382 struct uuid_entry_v0
*u0
= (void *) c
->uuids
;
383 struct uuid_entry
*u1
= (void *) c
->uuids
;
389 * Since the new uuid entry is bigger than the old, we have to
390 * convert starting at the highest memory address and work down
391 * in order to do it in place
394 for (i
= c
->nr_uuids
- 1;
397 memcpy(u1
[i
].uuid
, u0
[i
].uuid
, 16);
398 memcpy(u1
[i
].label
, u0
[i
].label
, 32);
400 u1
[i
].first_reg
= u0
[i
].first_reg
;
401 u1
[i
].last_reg
= u0
[i
].last_reg
;
402 u1
[i
].invalidated
= u0
[i
].invalidated
;
412 static int __uuid_write(struct cache_set
*c
)
418 closure_init_stack(&cl
);
420 lockdep_assert_held(&bch_register_lock
);
422 if (bch_bucket_alloc_set(c
, RESERVE_BTREE
, &k
.key
, 1, true))
425 SET_KEY_SIZE(&k
.key
, c
->sb
.bucket_size
);
426 uuid_io(c
, REQ_OP_WRITE
, 0, &k
.key
, &cl
);
429 /* Only one bucket used for uuid write */
430 ca
= PTR_CACHE(c
, &k
.key
, 0);
431 atomic_long_add(ca
->sb
.bucket_size
, &ca
->meta_sectors_written
);
433 bkey_copy(&c
->uuid_bucket
, &k
.key
);
438 int bch_uuid_write(struct cache_set
*c
)
440 int ret
= __uuid_write(c
);
443 bch_journal_meta(c
, NULL
);
448 static struct uuid_entry
*uuid_find(struct cache_set
*c
, const char *uuid
)
450 struct uuid_entry
*u
;
453 u
< c
->uuids
+ c
->nr_uuids
; u
++)
454 if (!memcmp(u
->uuid
, uuid
, 16))
460 static struct uuid_entry
*uuid_find_empty(struct cache_set
*c
)
462 static const char zero_uuid
[16] = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";
463 return uuid_find(c
, zero_uuid
);
467 * Bucket priorities/gens:
469 * For each bucket, we store on disk its
473 * See alloc.c for an explanation of the gen. The priority is used to implement
474 * lru (and in the future other) cache replacement policies; for most purposes
475 * it's just an opaque integer.
477 * The gens and the priorities don't have a whole lot to do with each other, and
478 * it's actually the gens that must be written out at specific times - it's no
479 * big deal if the priorities don't get written, if we lose them we just reuse
480 * buckets in suboptimal order.
482 * On disk they're stored in a packed array, and in as many buckets are required
483 * to fit them all. The buckets we use to store them form a list; the journal
484 * header points to the first bucket, the first bucket points to the second
487 * This code is used by the allocation code; periodically (whenever it runs out
488 * of buckets to allocate from) the allocation code will invalidate some
489 * buckets, but it can't use those buckets until their new gens are safely on
493 static void prio_endio(struct bio
*bio
)
495 struct cache
*ca
= bio
->bi_private
;
497 cache_set_err_on(bio
->bi_status
, ca
->set
, "accessing priorities");
498 bch_bbio_free(bio
, ca
->set
);
499 closure_put(&ca
->prio
);
502 static void prio_io(struct cache
*ca
, uint64_t bucket
, int op
,
503 unsigned long op_flags
)
505 struct closure
*cl
= &ca
->prio
;
506 struct bio
*bio
= bch_bbio_alloc(ca
->set
);
508 closure_init_stack(cl
);
510 bio
->bi_iter
.bi_sector
= bucket
* ca
->sb
.bucket_size
;
511 bio_set_dev(bio
, ca
->bdev
);
512 bio
->bi_iter
.bi_size
= bucket_bytes(ca
);
514 bio
->bi_end_io
= prio_endio
;
515 bio
->bi_private
= ca
;
516 bio_set_op_attrs(bio
, op
, REQ_SYNC
|REQ_META
|op_flags
);
517 bch_bio_map(bio
, ca
->disk_buckets
);
519 closure_bio_submit(ca
->set
, bio
, &ca
->prio
);
523 int bch_prio_write(struct cache
*ca
, bool wait
)
529 pr_debug("free_prio=%zu, free_none=%zu, free_inc=%zu",
530 fifo_used(&ca
->free
[RESERVE_PRIO
]),
531 fifo_used(&ca
->free
[RESERVE_NONE
]),
532 fifo_used(&ca
->free_inc
));
535 * Pre-check if there are enough free buckets. In the non-blocking
536 * scenario it's better to fail early rather than starting to allocate
537 * buckets and do a cleanup later in case of failure.
540 size_t avail
= fifo_used(&ca
->free
[RESERVE_PRIO
]) +
541 fifo_used(&ca
->free
[RESERVE_NONE
]);
542 if (prio_buckets(ca
) > avail
)
546 closure_init_stack(&cl
);
548 lockdep_assert_held(&ca
->set
->bucket_lock
);
550 ca
->disk_buckets
->seq
++;
552 atomic_long_add(ca
->sb
.bucket_size
* prio_buckets(ca
),
553 &ca
->meta_sectors_written
);
555 for (i
= prio_buckets(ca
) - 1; i
>= 0; --i
) {
557 struct prio_set
*p
= ca
->disk_buckets
;
558 struct bucket_disk
*d
= p
->data
;
559 struct bucket_disk
*end
= d
+ prios_per_bucket(ca
);
561 for (b
= ca
->buckets
+ i
* prios_per_bucket(ca
);
562 b
< ca
->buckets
+ ca
->sb
.nbuckets
&& d
< end
;
564 d
->prio
= cpu_to_le16(b
->prio
);
568 p
->next_bucket
= ca
->prio_buckets
[i
+ 1];
569 p
->magic
= pset_magic(&ca
->sb
);
570 p
->csum
= bch_crc64(&p
->magic
, bucket_bytes(ca
) - 8);
572 bucket
= bch_bucket_alloc(ca
, RESERVE_PRIO
, wait
);
573 BUG_ON(bucket
== -1);
575 mutex_unlock(&ca
->set
->bucket_lock
);
576 prio_io(ca
, bucket
, REQ_OP_WRITE
, 0);
577 mutex_lock(&ca
->set
->bucket_lock
);
579 ca
->prio_buckets
[i
] = bucket
;
580 atomic_dec_bug(&ca
->buckets
[bucket
].pin
);
583 mutex_unlock(&ca
->set
->bucket_lock
);
585 bch_journal_meta(ca
->set
, &cl
);
588 mutex_lock(&ca
->set
->bucket_lock
);
591 * Don't want the old priorities to get garbage collected until after we
592 * finish writing the new ones, and they're journalled
594 for (i
= 0; i
< prio_buckets(ca
); i
++) {
595 if (ca
->prio_last_buckets
[i
])
596 __bch_bucket_free(ca
,
597 &ca
->buckets
[ca
->prio_last_buckets
[i
]]);
599 ca
->prio_last_buckets
[i
] = ca
->prio_buckets
[i
];
604 static void prio_read(struct cache
*ca
, uint64_t bucket
)
606 struct prio_set
*p
= ca
->disk_buckets
;
607 struct bucket_disk
*d
= p
->data
+ prios_per_bucket(ca
), *end
= d
;
609 unsigned bucket_nr
= 0;
611 for (b
= ca
->buckets
;
612 b
< ca
->buckets
+ ca
->sb
.nbuckets
;
615 ca
->prio_buckets
[bucket_nr
] = bucket
;
616 ca
->prio_last_buckets
[bucket_nr
] = bucket
;
619 prio_io(ca
, bucket
, REQ_OP_READ
, 0);
621 if (p
->csum
!= bch_crc64(&p
->magic
, bucket_bytes(ca
) - 8))
622 pr_warn("bad csum reading priorities");
624 if (p
->magic
!= pset_magic(&ca
->sb
))
625 pr_warn("bad magic reading priorities");
627 bucket
= p
->next_bucket
;
631 b
->prio
= le16_to_cpu(d
->prio
);
632 b
->gen
= b
->last_gc
= d
->gen
;
638 static int open_dev(struct block_device
*b
, fmode_t mode
)
640 struct bcache_device
*d
= b
->bd_disk
->private_data
;
641 if (test_bit(BCACHE_DEV_CLOSING
, &d
->flags
))
648 static void release_dev(struct gendisk
*b
, fmode_t mode
)
650 struct bcache_device
*d
= b
->private_data
;
654 static int ioctl_dev(struct block_device
*b
, fmode_t mode
,
655 unsigned int cmd
, unsigned long arg
)
657 struct bcache_device
*d
= b
->bd_disk
->private_data
;
659 return d
->ioctl(d
, mode
, cmd
, arg
);
662 static const struct block_device_operations bcache_ops
= {
664 .release
= release_dev
,
666 .owner
= THIS_MODULE
,
669 void bcache_device_stop(struct bcache_device
*d
)
671 if (!test_and_set_bit(BCACHE_DEV_CLOSING
, &d
->flags
))
672 closure_queue(&d
->cl
);
675 static void bcache_device_unlink(struct bcache_device
*d
)
677 lockdep_assert_held(&bch_register_lock
);
679 if (d
->c
&& !test_and_set_bit(BCACHE_DEV_UNLINK_DONE
, &d
->flags
)) {
683 sysfs_remove_link(&d
->c
->kobj
, d
->name
);
684 sysfs_remove_link(&d
->kobj
, "cache");
686 for_each_cache(ca
, d
->c
, i
)
687 bd_unlink_disk_holder(ca
->bdev
, d
->disk
);
691 static void bcache_device_link(struct bcache_device
*d
, struct cache_set
*c
,
697 for_each_cache(ca
, d
->c
, i
)
698 bd_link_disk_holder(ca
->bdev
, d
->disk
);
700 snprintf(d
->name
, BCACHEDEVNAME_SIZE
,
701 "%s%u", name
, d
->id
);
703 WARN(sysfs_create_link(&d
->kobj
, &c
->kobj
, "cache") ||
704 sysfs_create_link(&c
->kobj
, &d
->kobj
, d
->name
),
705 "Couldn't create device <-> cache set symlinks");
707 clear_bit(BCACHE_DEV_UNLINK_DONE
, &d
->flags
);
710 static void bcache_device_detach(struct bcache_device
*d
)
712 lockdep_assert_held(&bch_register_lock
);
714 if (test_bit(BCACHE_DEV_DETACHING
, &d
->flags
)) {
715 struct uuid_entry
*u
= d
->c
->uuids
+ d
->id
;
717 SET_UUID_FLASH_ONLY(u
, 0);
718 memcpy(u
->uuid
, invalid_uuid
, 16);
719 u
->invalidated
= cpu_to_le32(get_seconds());
720 bch_uuid_write(d
->c
);
723 bcache_device_unlink(d
);
725 d
->c
->devices
[d
->id
] = NULL
;
726 closure_put(&d
->c
->caching
);
730 static void bcache_device_attach(struct bcache_device
*d
, struct cache_set
*c
,
737 closure_get(&c
->caching
);
740 static inline int first_minor_to_idx(int first_minor
)
742 return (first_minor
/BCACHE_MINORS
);
745 static inline int idx_to_first_minor(int idx
)
747 return (idx
* BCACHE_MINORS
);
750 static void bcache_device_free(struct bcache_device
*d
)
752 struct gendisk
*disk
= d
->disk
;
754 lockdep_assert_held(&bch_register_lock
);
757 pr_info("%s stopped", disk
->disk_name
);
759 pr_err("bcache device (NULL gendisk) stopped");
762 bcache_device_detach(d
);
765 if (disk
->flags
& GENHD_FL_UP
)
769 blk_cleanup_queue(disk
->queue
);
771 ida_simple_remove(&bcache_device_idx
,
772 first_minor_to_idx(disk
->first_minor
));
777 bioset_free(d
->bio_split
);
778 kvfree(d
->full_dirty_stripes
);
779 kvfree(d
->stripe_sectors_dirty
);
781 closure_debug_destroy(&d
->cl
);
784 static int bcache_device_init(struct bcache_device
*d
, unsigned block_size
,
787 struct request_queue
*q
;
788 const size_t max_stripes
= min_t(size_t, INT_MAX
,
789 SIZE_MAX
/ sizeof(atomic_t
));
794 d
->stripe_size
= 1 << 31;
796 d
->nr_stripes
= DIV_ROUND_UP_ULL(sectors
, d
->stripe_size
);
798 if (!d
->nr_stripes
|| d
->nr_stripes
> max_stripes
) {
799 pr_err("nr_stripes too large or invalid: %u (start sector beyond end of disk?)",
800 (unsigned)d
->nr_stripes
);
804 n
= d
->nr_stripes
* sizeof(atomic_t
);
805 d
->stripe_sectors_dirty
= kvzalloc(n
, GFP_KERNEL
);
806 if (!d
->stripe_sectors_dirty
)
809 n
= BITS_TO_LONGS(d
->nr_stripes
) * sizeof(unsigned long);
810 d
->full_dirty_stripes
= kvzalloc(n
, GFP_KERNEL
);
811 if (!d
->full_dirty_stripes
)
814 idx
= ida_simple_get(&bcache_device_idx
, 0,
815 BCACHE_DEVICE_IDX_MAX
, GFP_KERNEL
);
819 if (!(d
->bio_split
= bioset_create(4, offsetof(struct bbio
, bio
),
821 BIOSET_NEED_RESCUER
)) ||
822 !(d
->disk
= alloc_disk(BCACHE_MINORS
))) {
823 ida_simple_remove(&bcache_device_idx
, idx
);
827 set_capacity(d
->disk
, sectors
);
828 snprintf(d
->disk
->disk_name
, DISK_NAME_LEN
, "bcache%i", idx
);
830 d
->disk
->major
= bcache_major
;
831 d
->disk
->first_minor
= idx_to_first_minor(idx
);
832 d
->disk
->fops
= &bcache_ops
;
833 d
->disk
->private_data
= d
;
835 q
= blk_alloc_queue(GFP_KERNEL
);
839 blk_queue_make_request(q
, NULL
);
842 q
->backing_dev_info
->congested_data
= d
;
843 q
->limits
.max_hw_sectors
= UINT_MAX
;
844 q
->limits
.max_sectors
= UINT_MAX
;
845 q
->limits
.max_segment_size
= UINT_MAX
;
846 q
->limits
.max_segments
= BIO_MAX_PAGES
;
847 blk_queue_max_discard_sectors(q
, UINT_MAX
);
848 q
->limits
.discard_granularity
= 512;
849 q
->limits
.io_min
= block_size
;
850 q
->limits
.logical_block_size
= block_size
;
851 q
->limits
.physical_block_size
= block_size
;
852 set_bit(QUEUE_FLAG_NONROT
, &d
->disk
->queue
->queue_flags
);
853 clear_bit(QUEUE_FLAG_ADD_RANDOM
, &d
->disk
->queue
->queue_flags
);
854 set_bit(QUEUE_FLAG_DISCARD
, &d
->disk
->queue
->queue_flags
);
856 blk_queue_write_cache(q
, true, true);
863 static void calc_cached_dev_sectors(struct cache_set
*c
)
865 uint64_t sectors
= 0;
866 struct cached_dev
*dc
;
868 list_for_each_entry(dc
, &c
->cached_devs
, list
)
869 sectors
+= bdev_sectors(dc
->bdev
);
871 c
->cached_dev_sectors
= sectors
;
874 #define BACKING_DEV_OFFLINE_TIMEOUT 5
875 static int cached_dev_status_update(void *arg
)
877 struct cached_dev
*dc
= arg
;
878 struct request_queue
*q
;
881 * If this delayed worker is stopping outside, directly quit here.
882 * dc->io_disable might be set via sysfs interface, so check it
885 while (!kthread_should_stop() && !dc
->io_disable
) {
886 q
= bdev_get_queue(dc
->bdev
);
887 if (blk_queue_dying(q
))
888 dc
->offline_seconds
++;
890 dc
->offline_seconds
= 0;
892 if (dc
->offline_seconds
>= BACKING_DEV_OFFLINE_TIMEOUT
) {
893 pr_err("%s: device offline for %d seconds",
894 dc
->backing_dev_name
,
895 BACKING_DEV_OFFLINE_TIMEOUT
);
896 pr_err("%s: disable I/O request due to backing "
897 "device offline", dc
->disk
.name
);
898 dc
->io_disable
= true;
899 /* let others know earlier that io_disable is true */
901 bcache_device_stop(&dc
->disk
);
904 schedule_timeout_interruptible(HZ
);
907 wait_for_kthread_stop();
912 void bch_cached_dev_emit_change(struct cached_dev
*dc
)
914 struct bcache_device
*d
= &dc
->disk
;
915 char buf
[SB_LABEL_SIZE
+ 1];
918 kasprintf(GFP_KERNEL
, "CACHED_UUID=%pU", dc
->sb
.uuid
),
923 memcpy(buf
, dc
->sb
.label
, SB_LABEL_SIZE
);
924 buf
[SB_LABEL_SIZE
] = '\0';
925 env
[2] = kasprintf(GFP_KERNEL
, "CACHED_LABEL=%s", buf
);
927 /* won't show up in the uevent file, use udevadm monitor -e instead
928 * only class / kset properties are persistent */
929 kobject_uevent_env(&disk_to_dev(d
->disk
)->kobj
, KOBJ_CHANGE
, env
);
935 void bch_cached_dev_run(struct cached_dev
*dc
)
937 struct bcache_device
*d
= &dc
->disk
;
938 if (atomic_xchg(&dc
->running
, 1)) {
943 BDEV_STATE(&dc
->sb
) != BDEV_STATE_NONE
) {
945 closure_init_stack(&cl
);
947 SET_BDEV_STATE(&dc
->sb
, BDEV_STATE_STALE
);
948 bch_write_bdev_super(dc
, &cl
);
953 bd_link_disk_holder(dc
->bdev
, dc
->disk
.disk
);
955 /* emit change event */
956 bch_cached_dev_emit_change(dc
);
958 if (sysfs_create_link(&d
->kobj
, &disk_to_dev(d
->disk
)->kobj
, "dev") ||
959 sysfs_create_link(&disk_to_dev(d
->disk
)->kobj
, &d
->kobj
, "bcache"))
960 pr_debug("error creating sysfs link");
962 dc
->status_update_thread
= kthread_run(cached_dev_status_update
,
963 dc
, "bcache_status_update");
964 if (IS_ERR(dc
->status_update_thread
)) {
965 pr_warn("failed to create bcache_status_update kthread, "
966 "continue to run without monitoring backing "
972 * If BCACHE_DEV_RATE_DW_RUNNING is set, it means routine of the delayed
973 * work dc->writeback_rate_update is running. Wait until the routine
974 * quits (BCACHE_DEV_RATE_DW_RUNNING is clear), then continue to
975 * cancel it. If BCACHE_DEV_RATE_DW_RUNNING is not clear after time_out
976 * seconds, give up waiting here and continue to cancel it too.
978 static void cancel_writeback_rate_update_dwork(struct cached_dev
*dc
)
980 int time_out
= WRITEBACK_RATE_UPDATE_SECS_MAX
* HZ
;
983 if (!test_bit(BCACHE_DEV_RATE_DW_RUNNING
,
987 schedule_timeout_interruptible(1);
988 } while (time_out
> 0);
991 pr_warn("give up waiting for dc->writeback_write_update to quit");
993 cancel_delayed_work_sync(&dc
->writeback_rate_update
);
996 static void cached_dev_detach_finish(struct work_struct
*w
)
998 struct cached_dev
*dc
= container_of(w
, struct cached_dev
, detach
);
1000 closure_init_stack(&cl
);
1002 BUG_ON(!test_bit(BCACHE_DEV_DETACHING
, &dc
->disk
.flags
));
1003 BUG_ON(refcount_read(&dc
->count
));
1005 mutex_lock(&bch_register_lock
);
1007 if (test_and_clear_bit(BCACHE_DEV_WB_RUNNING
, &dc
->disk
.flags
))
1008 cancel_writeback_rate_update_dwork(dc
);
1010 if (!IS_ERR_OR_NULL(dc
->writeback_thread
)) {
1011 kthread_stop(dc
->writeback_thread
);
1012 dc
->writeback_thread
= NULL
;
1015 memset(&dc
->sb
.set_uuid
, 0, 16);
1016 SET_BDEV_STATE(&dc
->sb
, BDEV_STATE_NONE
);
1018 bch_write_bdev_super(dc
, &cl
);
1021 calc_cached_dev_sectors(dc
->disk
.c
);
1022 bcache_device_detach(&dc
->disk
);
1023 list_move(&dc
->list
, &uncached_devices
);
1025 clear_bit(BCACHE_DEV_DETACHING
, &dc
->disk
.flags
);
1026 clear_bit(BCACHE_DEV_UNLINK_DONE
, &dc
->disk
.flags
);
1028 mutex_unlock(&bch_register_lock
);
1030 pr_info("Caching disabled for %s", dc
->backing_dev_name
);
1032 /* Drop ref we took in cached_dev_detach() */
1033 closure_put(&dc
->disk
.cl
);
1036 void bch_cached_dev_detach(struct cached_dev
*dc
)
1038 lockdep_assert_held(&bch_register_lock
);
1040 if (test_bit(BCACHE_DEV_CLOSING
, &dc
->disk
.flags
))
1043 if (test_and_set_bit(BCACHE_DEV_DETACHING
, &dc
->disk
.flags
))
1047 * Block the device from being closed and freed until we're finished
1050 closure_get(&dc
->disk
.cl
);
1052 bch_writeback_queue(dc
);
1057 int bch_cached_dev_attach(struct cached_dev
*dc
, struct cache_set
*c
,
1060 uint32_t rtime
= cpu_to_le32(get_seconds());
1061 struct uuid_entry
*u
;
1062 struct cached_dev
*exist_dc
, *t
;
1064 if ((set_uuid
&& memcmp(set_uuid
, c
->sb
.set_uuid
, 16)) ||
1065 (!set_uuid
&& memcmp(dc
->sb
.set_uuid
, c
->sb
.set_uuid
, 16)))
1069 pr_err("Can't attach %s: already attached",
1070 dc
->backing_dev_name
);
1074 if (test_bit(CACHE_SET_STOPPING
, &c
->flags
)) {
1075 pr_err("Can't attach %s: shutting down",
1076 dc
->backing_dev_name
);
1080 if (dc
->sb
.block_size
< c
->sb
.block_size
) {
1082 pr_err("Couldn't attach %s: block size less than set's block size",
1083 dc
->backing_dev_name
);
1087 /* Check whether already attached */
1088 list_for_each_entry_safe(exist_dc
, t
, &c
->cached_devs
, list
) {
1089 if (!memcmp(dc
->sb
.uuid
, exist_dc
->sb
.uuid
, 16)) {
1090 pr_err("Tried to attach %s but duplicate UUID already attached",
1091 dc
->backing_dev_name
);
1097 u
= uuid_find(c
, dc
->sb
.uuid
);
1100 (BDEV_STATE(&dc
->sb
) == BDEV_STATE_STALE
||
1101 BDEV_STATE(&dc
->sb
) == BDEV_STATE_NONE
)) {
1102 memcpy(u
->uuid
, invalid_uuid
, 16);
1103 u
->invalidated
= cpu_to_le32(get_seconds());
1108 if (BDEV_STATE(&dc
->sb
) == BDEV_STATE_DIRTY
) {
1109 pr_err("Couldn't find uuid for %s in set",
1110 dc
->backing_dev_name
);
1114 u
= uuid_find_empty(c
);
1116 pr_err("Not caching %s, no room for UUID",
1117 dc
->backing_dev_name
);
1122 /* Deadlocks since we're called via sysfs...
1123 sysfs_remove_file(&dc->kobj, &sysfs_attach);
1126 if (bch_is_zero(u
->uuid
, 16)) {
1128 closure_init_stack(&cl
);
1130 memcpy(u
->uuid
, dc
->sb
.uuid
, 16);
1131 memcpy(u
->label
, dc
->sb
.label
, SB_LABEL_SIZE
);
1132 u
->first_reg
= u
->last_reg
= rtime
;
1135 memcpy(dc
->sb
.set_uuid
, c
->sb
.set_uuid
, 16);
1136 SET_BDEV_STATE(&dc
->sb
, BDEV_STATE_CLEAN
);
1138 bch_write_bdev_super(dc
, &cl
);
1141 u
->last_reg
= rtime
;
1145 bcache_device_attach(&dc
->disk
, c
, u
- c
->uuids
);
1146 list_move(&dc
->list
, &c
->cached_devs
);
1147 calc_cached_dev_sectors(c
);
1151 * dc->c must be set before dc->count != 0 - paired with the mb in
1154 refcount_set(&dc
->count
, 1);
1156 /* Block writeback thread, but spawn it */
1157 down_write(&dc
->writeback_lock
);
1158 if (bch_cached_dev_writeback_start(dc
)) {
1159 up_write(&dc
->writeback_lock
);
1163 if (BDEV_STATE(&dc
->sb
) == BDEV_STATE_DIRTY
) {
1164 atomic_set(&dc
->has_dirty
, 1);
1165 bch_writeback_queue(dc
);
1168 bch_sectors_dirty_init(&dc
->disk
);
1170 bch_cached_dev_run(dc
);
1171 bcache_device_link(&dc
->disk
, c
, "bdev");
1173 /* Allow the writeback thread to proceed */
1174 up_write(&dc
->writeback_lock
);
1176 pr_info("Caching %s as %s on set %pU",
1177 dc
->backing_dev_name
,
1178 dc
->disk
.disk
->disk_name
,
1179 dc
->disk
.c
->sb
.set_uuid
);
1183 void bch_cached_dev_release(struct kobject
*kobj
)
1185 struct cached_dev
*dc
= container_of(kobj
, struct cached_dev
,
1188 module_put(THIS_MODULE
);
1191 static void cached_dev_free(struct closure
*cl
)
1193 struct cached_dev
*dc
= container_of(cl
, struct cached_dev
, disk
.cl
);
1195 if (test_and_clear_bit(BCACHE_DEV_WB_RUNNING
, &dc
->disk
.flags
))
1196 cancel_writeback_rate_update_dwork(dc
);
1198 if (!IS_ERR_OR_NULL(dc
->writeback_thread
))
1199 kthread_stop(dc
->writeback_thread
);
1200 if (!IS_ERR_OR_NULL(dc
->status_update_thread
))
1201 kthread_stop(dc
->status_update_thread
);
1203 mutex_lock(&bch_register_lock
);
1205 if (atomic_read(&dc
->running
))
1206 bd_unlink_disk_holder(dc
->bdev
, dc
->disk
.disk
);
1207 bcache_device_free(&dc
->disk
);
1208 list_del(&dc
->list
);
1210 mutex_unlock(&bch_register_lock
);
1212 if (!IS_ERR_OR_NULL(dc
->bdev
))
1213 blkdev_put(dc
->bdev
, FMODE_READ
|FMODE_WRITE
|FMODE_EXCL
);
1215 wake_up(&unregister_wait
);
1217 kobject_put(&dc
->disk
.kobj
);
1220 static void cached_dev_flush(struct closure
*cl
)
1222 struct cached_dev
*dc
= container_of(cl
, struct cached_dev
, disk
.cl
);
1223 struct bcache_device
*d
= &dc
->disk
;
1225 mutex_lock(&bch_register_lock
);
1226 bcache_device_unlink(d
);
1227 mutex_unlock(&bch_register_lock
);
1229 bch_cache_accounting_destroy(&dc
->accounting
);
1230 kobject_del(&d
->kobj
);
1232 continue_at(cl
, cached_dev_free
, system_wq
);
1235 static int cached_dev_init(struct cached_dev
*dc
, unsigned block_size
)
1239 struct request_queue
*q
= bdev_get_queue(dc
->bdev
);
1241 __module_get(THIS_MODULE
);
1242 INIT_LIST_HEAD(&dc
->list
);
1243 closure_init(&dc
->disk
.cl
, NULL
);
1244 set_closure_fn(&dc
->disk
.cl
, cached_dev_flush
, system_wq
);
1245 kobject_init(&dc
->disk
.kobj
, &bch_cached_dev_ktype
);
1246 INIT_WORK(&dc
->detach
, cached_dev_detach_finish
);
1247 sema_init(&dc
->sb_write_mutex
, 1);
1248 INIT_LIST_HEAD(&dc
->io_lru
);
1249 spin_lock_init(&dc
->io_lock
);
1250 bch_cache_accounting_init(&dc
->accounting
, &dc
->disk
.cl
);
1252 dc
->sequential_cutoff
= 4 << 20;
1254 for (io
= dc
->io
; io
< dc
->io
+ RECENT_IO
; io
++) {
1255 list_add(&io
->lru
, &dc
->io_lru
);
1256 hlist_add_head(&io
->hash
, dc
->io_hash
+ RECENT_IO
);
1259 dc
->disk
.stripe_size
= q
->limits
.io_opt
>> 9;
1261 if (dc
->disk
.stripe_size
)
1262 dc
->partial_stripes_expensive
=
1263 q
->limits
.raid_partial_stripes_expensive
;
1265 ret
= bcache_device_init(&dc
->disk
, block_size
,
1266 dc
->bdev
->bd_part
->nr_sects
- dc
->sb
.data_offset
);
1270 dc
->disk
.disk
->queue
->backing_dev_info
->ra_pages
=
1271 max(dc
->disk
.disk
->queue
->backing_dev_info
->ra_pages
,
1272 q
->backing_dev_info
->ra_pages
);
1274 atomic_set(&dc
->io_errors
, 0);
1275 dc
->io_disable
= false;
1276 dc
->error_limit
= DEFAULT_CACHED_DEV_ERROR_LIMIT
;
1277 /* default to auto */
1278 dc
->stop_when_cache_set_failed
= BCH_CACHED_DEV_STOP_AUTO
;
1280 bch_cached_dev_request_init(dc
);
1281 bch_cached_dev_writeback_init(dc
);
1285 /* Cached device - bcache superblock */
1287 static void register_bdev(struct cache_sb
*sb
, struct page
*sb_page
,
1288 struct block_device
*bdev
,
1289 struct cached_dev
*dc
)
1291 const char *err
= "cannot allocate memory";
1292 struct cache_set
*c
;
1294 bdevname(bdev
, dc
->backing_dev_name
);
1295 memcpy(&dc
->sb
, sb
, sizeof(struct cache_sb
));
1297 dc
->bdev
->bd_holder
= dc
;
1299 bio_init(&dc
->sb_bio
, dc
->sb_bio
.bi_inline_vecs
, 1);
1300 dc
->sb_bio
.bi_io_vec
[0].bv_page
= sb_page
;
1304 if (cached_dev_init(dc
, sb
->block_size
<< 9))
1307 err
= "error creating kobject";
1308 if (kobject_add(&dc
->disk
.kobj
, &part_to_dev(bdev
->bd_part
)->kobj
,
1311 if (bch_cache_accounting_add_kobjs(&dc
->accounting
, &dc
->disk
.kobj
))
1314 pr_info("registered backing device %s", dc
->backing_dev_name
);
1316 list_add(&dc
->list
, &uncached_devices
);
1317 list_for_each_entry(c
, &bch_cache_sets
, list
)
1318 bch_cached_dev_attach(dc
, c
, NULL
);
1320 if (BDEV_STATE(&dc
->sb
) == BDEV_STATE_NONE
||
1321 BDEV_STATE(&dc
->sb
) == BDEV_STATE_STALE
)
1322 bch_cached_dev_run(dc
);
1326 pr_notice("error %s: %s", dc
->backing_dev_name
, err
);
1327 bcache_device_stop(&dc
->disk
);
1330 /* Flash only volumes */
1332 void bch_flash_dev_release(struct kobject
*kobj
)
1334 struct bcache_device
*d
= container_of(kobj
, struct bcache_device
,
1339 static void flash_dev_free(struct closure
*cl
)
1341 struct bcache_device
*d
= container_of(cl
, struct bcache_device
, cl
);
1342 mutex_lock(&bch_register_lock
);
1343 bcache_device_free(d
);
1344 mutex_unlock(&bch_register_lock
);
1345 kobject_put(&d
->kobj
);
1348 static void flash_dev_flush(struct closure
*cl
)
1350 struct bcache_device
*d
= container_of(cl
, struct bcache_device
, cl
);
1352 mutex_lock(&bch_register_lock
);
1353 bcache_device_unlink(d
);
1354 mutex_unlock(&bch_register_lock
);
1355 kobject_del(&d
->kobj
);
1356 continue_at(cl
, flash_dev_free
, system_wq
);
1359 static int flash_dev_run(struct cache_set
*c
, struct uuid_entry
*u
)
1361 struct bcache_device
*d
= kzalloc(sizeof(struct bcache_device
),
1366 closure_init(&d
->cl
, NULL
);
1367 set_closure_fn(&d
->cl
, flash_dev_flush
, system_wq
);
1369 kobject_init(&d
->kobj
, &bch_flash_dev_ktype
);
1371 if (bcache_device_init(d
, block_bytes(c
), u
->sectors
))
1374 bcache_device_attach(d
, c
, u
- c
->uuids
);
1375 bch_sectors_dirty_init(d
);
1376 bch_flash_dev_request_init(d
);
1379 if (kobject_add(&d
->kobj
, &disk_to_dev(d
->disk
)->kobj
, "bcache"))
1382 bcache_device_link(d
, c
, "volume");
1386 kobject_put(&d
->kobj
);
1390 static int flash_devs_run(struct cache_set
*c
)
1393 struct uuid_entry
*u
;
1396 u
< c
->uuids
+ c
->nr_uuids
&& !ret
;
1398 if (UUID_FLASH_ONLY(u
))
1399 ret
= flash_dev_run(c
, u
);
1404 int bch_flash_dev_create(struct cache_set
*c
, uint64_t size
)
1406 struct uuid_entry
*u
;
1408 if (test_bit(CACHE_SET_STOPPING
, &c
->flags
))
1411 if (!test_bit(CACHE_SET_RUNNING
, &c
->flags
))
1414 u
= uuid_find_empty(c
);
1416 pr_err("Can't create volume, no room for UUID");
1420 get_random_bytes(u
->uuid
, 16);
1421 memset(u
->label
, 0, 32);
1422 u
->first_reg
= u
->last_reg
= cpu_to_le32(get_seconds());
1424 SET_UUID_FLASH_ONLY(u
, 1);
1425 u
->sectors
= size
>> 9;
1429 return flash_dev_run(c
, u
);
1432 bool bch_cached_dev_error(struct cached_dev
*dc
)
1434 if (!dc
|| test_bit(BCACHE_DEV_CLOSING
, &dc
->disk
.flags
))
1437 dc
->io_disable
= true;
1438 /* make others know io_disable is true earlier */
1441 pr_err("stop %s: too many IO errors on backing device %s\n",
1442 dc
->disk
.disk
->disk_name
, dc
->backing_dev_name
);
1444 bcache_device_stop(&dc
->disk
);
1451 bool bch_cache_set_error(struct cache_set
*c
, const char *fmt
, ...)
1455 if (c
->on_error
!= ON_ERROR_PANIC
&&
1456 test_bit(CACHE_SET_STOPPING
, &c
->flags
))
1459 if (test_and_set_bit(CACHE_SET_IO_DISABLE
, &c
->flags
))
1460 pr_info("CACHE_SET_IO_DISABLE already set");
1462 /* XXX: we can be called from atomic context
1463 acquire_console_sem();
1466 printk(KERN_ERR
"bcache: error on %pU: ", c
->sb
.set_uuid
);
1468 va_start(args
, fmt
);
1472 printk(", disabling caching\n");
1474 if (c
->on_error
== ON_ERROR_PANIC
)
1475 panic("panic forced after error\n");
1477 bch_cache_set_unregister(c
);
1481 void bch_cache_set_release(struct kobject
*kobj
)
1483 struct cache_set
*c
= container_of(kobj
, struct cache_set
, kobj
);
1485 module_put(THIS_MODULE
);
1488 static void cache_set_free(struct closure
*cl
)
1490 struct cache_set
*c
= container_of(cl
, struct cache_set
, cl
);
1494 debugfs_remove(c
->debug
);
1496 bch_open_buckets_free(c
);
1497 bch_btree_cache_free(c
);
1498 bch_journal_free(c
);
1500 mutex_lock(&bch_register_lock
);
1501 for_each_cache(ca
, c
, i
)
1504 c
->cache
[ca
->sb
.nr_this_dev
] = NULL
;
1505 kobject_put(&ca
->kobj
);
1508 bch_bset_sort_state_free(&c
->sort
);
1509 free_pages((unsigned long) c
->uuids
, ilog2(bucket_pages(c
)));
1511 if (c
->moving_gc_wq
)
1512 destroy_workqueue(c
->moving_gc_wq
);
1514 bioset_free(c
->bio_split
);
1516 mempool_destroy(c
->fill_iter
);
1518 mempool_destroy(c
->bio_meta
);
1520 mempool_destroy(c
->search
);
1524 mutex_unlock(&bch_register_lock
);
1526 pr_info("Cache set %pU unregistered", c
->sb
.set_uuid
);
1527 wake_up(&unregister_wait
);
1529 closure_debug_destroy(&c
->cl
);
1530 kobject_put(&c
->kobj
);
1533 static void cache_set_flush(struct closure
*cl
)
1535 struct cache_set
*c
= container_of(cl
, struct cache_set
, caching
);
1540 bch_cache_accounting_destroy(&c
->accounting
);
1542 kobject_put(&c
->internal
);
1543 kobject_del(&c
->kobj
);
1545 if (!IS_ERR_OR_NULL(c
->gc_thread
))
1546 kthread_stop(c
->gc_thread
);
1548 if (!IS_ERR_OR_NULL(c
->root
))
1549 list_add(&c
->root
->list
, &c
->btree_cache
);
1551 /* Should skip this if we're unregistering because of an error */
1552 list_for_each_entry(b
, &c
->btree_cache
, list
) {
1553 mutex_lock(&b
->write_lock
);
1554 if (btree_node_dirty(b
))
1555 __bch_btree_node_write(b
, NULL
);
1556 mutex_unlock(&b
->write_lock
);
1559 for_each_cache(ca
, c
, i
)
1560 if (ca
->alloc_thread
)
1561 kthread_stop(ca
->alloc_thread
);
1563 if (c
->journal
.cur
) {
1564 cancel_delayed_work_sync(&c
->journal
.work
);
1565 /* flush last journal entry if needed */
1566 c
->journal
.work
.work
.func(&c
->journal
.work
.work
);
1573 * This function is only called when CACHE_SET_IO_DISABLE is set, which means
1574 * cache set is unregistering due to too many I/O errors. In this condition,
1575 * the bcache device might be stopped, it depends on stop_when_cache_set_failed
1576 * value and whether the broken cache has dirty data:
1578 * dc->stop_when_cache_set_failed dc->has_dirty stop bcache device
1579 * BCH_CACHED_STOP_AUTO 0 NO
1580 * BCH_CACHED_STOP_AUTO 1 YES
1581 * BCH_CACHED_DEV_STOP_ALWAYS 0 YES
1582 * BCH_CACHED_DEV_STOP_ALWAYS 1 YES
1584 * The expected behavior is, if stop_when_cache_set_failed is configured to
1585 * "auto" via sysfs interface, the bcache device will not be stopped if the
1586 * backing device is clean on the broken cache device.
1588 static void conditional_stop_bcache_device(struct cache_set
*c
,
1589 struct bcache_device
*d
,
1590 struct cached_dev
*dc
)
1592 if (dc
->stop_when_cache_set_failed
== BCH_CACHED_DEV_STOP_ALWAYS
) {
1593 pr_warn("stop_when_cache_set_failed of %s is \"always\", stop it for failed cache set %pU.",
1594 d
->disk
->disk_name
, c
->sb
.set_uuid
);
1595 bcache_device_stop(d
);
1596 } else if (atomic_read(&dc
->has_dirty
)) {
1598 * dc->stop_when_cache_set_failed == BCH_CACHED_STOP_AUTO
1599 * and dc->has_dirty == 1
1601 pr_warn("stop_when_cache_set_failed of %s is \"auto\" and cache is dirty, stop it to avoid potential data corruption.",
1602 d
->disk
->disk_name
);
1604 * There might be a small time gap that cache set is
1605 * released but bcache device is not. Inside this time
1606 * gap, regular I/O requests will directly go into
1607 * backing device as no cache set attached to. This
1608 * behavior may also introduce potential inconsistence
1609 * data in writeback mode while cache is dirty.
1610 * Therefore before calling bcache_device_stop() due
1611 * to a broken cache device, dc->io_disable should be
1612 * explicitly set to true.
1614 dc
->io_disable
= true;
1615 /* make others know io_disable is true earlier */
1617 bcache_device_stop(d
);
1620 * dc->stop_when_cache_set_failed == BCH_CACHED_STOP_AUTO
1621 * and dc->has_dirty == 0
1623 pr_warn("stop_when_cache_set_failed of %s is \"auto\" and cache is clean, keep it alive.",
1624 d
->disk
->disk_name
);
1628 static void __cache_set_unregister(struct closure
*cl
)
1630 struct cache_set
*c
= container_of(cl
, struct cache_set
, caching
);
1631 struct cached_dev
*dc
;
1632 struct bcache_device
*d
;
1635 mutex_lock(&bch_register_lock
);
1637 for (i
= 0; i
< c
->nr_uuids
; i
++) {
1642 if (!UUID_FLASH_ONLY(&c
->uuids
[i
]) &&
1643 test_bit(CACHE_SET_UNREGISTERING
, &c
->flags
)) {
1644 dc
= container_of(d
, struct cached_dev
, disk
);
1645 bch_cached_dev_detach(dc
);
1646 if (test_bit(CACHE_SET_IO_DISABLE
, &c
->flags
))
1647 conditional_stop_bcache_device(c
, d
, dc
);
1649 bcache_device_stop(d
);
1653 mutex_unlock(&bch_register_lock
);
1655 continue_at(cl
, cache_set_flush
, system_wq
);
1658 void bch_cache_set_stop(struct cache_set
*c
)
1660 if (!test_and_set_bit(CACHE_SET_STOPPING
, &c
->flags
))
1661 closure_queue(&c
->caching
);
1664 void bch_cache_set_unregister(struct cache_set
*c
)
1666 set_bit(CACHE_SET_UNREGISTERING
, &c
->flags
);
1667 bch_cache_set_stop(c
);
1670 #define alloc_bucket_pages(gfp, c) \
1671 ((void *) __get_free_pages(__GFP_ZERO|gfp, ilog2(bucket_pages(c))))
1673 struct cache_set
*bch_cache_set_alloc(struct cache_sb
*sb
)
1676 struct cache_set
*c
= kzalloc(sizeof(struct cache_set
), GFP_KERNEL
);
1680 __module_get(THIS_MODULE
);
1681 closure_init(&c
->cl
, NULL
);
1682 set_closure_fn(&c
->cl
, cache_set_free
, system_wq
);
1684 closure_init(&c
->caching
, &c
->cl
);
1685 set_closure_fn(&c
->caching
, __cache_set_unregister
, system_wq
);
1687 /* Maybe create continue_at_noreturn() and use it here? */
1688 closure_set_stopped(&c
->cl
);
1689 closure_put(&c
->cl
);
1691 kobject_init(&c
->kobj
, &bch_cache_set_ktype
);
1692 kobject_init(&c
->internal
, &bch_cache_set_internal_ktype
);
1694 bch_cache_accounting_init(&c
->accounting
, &c
->cl
);
1696 memcpy(c
->sb
.set_uuid
, sb
->set_uuid
, 16);
1697 c
->sb
.block_size
= sb
->block_size
;
1698 c
->sb
.bucket_size
= sb
->bucket_size
;
1699 c
->sb
.nr_in_set
= sb
->nr_in_set
;
1700 c
->sb
.last_mount
= sb
->last_mount
;
1701 c
->bucket_bits
= ilog2(sb
->bucket_size
);
1702 c
->block_bits
= ilog2(sb
->block_size
);
1703 c
->nr_uuids
= bucket_bytes(c
) / sizeof(struct uuid_entry
);
1705 c
->btree_pages
= bucket_pages(c
);
1706 if (c
->btree_pages
> BTREE_MAX_PAGES
)
1707 c
->btree_pages
= max_t(int, c
->btree_pages
/ 4,
1710 sema_init(&c
->sb_write_mutex
, 1);
1711 mutex_init(&c
->bucket_lock
);
1712 init_waitqueue_head(&c
->btree_cache_wait
);
1713 init_waitqueue_head(&c
->bucket_wait
);
1714 init_waitqueue_head(&c
->gc_wait
);
1715 sema_init(&c
->uuid_write_mutex
, 1);
1717 spin_lock_init(&c
->btree_gc_time
.lock
);
1718 spin_lock_init(&c
->btree_split_time
.lock
);
1719 spin_lock_init(&c
->btree_read_time
.lock
);
1721 bch_moving_init_cache_set(c
);
1723 INIT_LIST_HEAD(&c
->list
);
1724 INIT_LIST_HEAD(&c
->cached_devs
);
1725 INIT_LIST_HEAD(&c
->btree_cache
);
1726 INIT_LIST_HEAD(&c
->btree_cache_freeable
);
1727 INIT_LIST_HEAD(&c
->btree_cache_freed
);
1728 INIT_LIST_HEAD(&c
->data_buckets
);
1730 c
->search
= mempool_create_slab_pool(32, bch_search_cache
);
1734 iter_size
= (sb
->bucket_size
/ sb
->block_size
+ 1) *
1735 sizeof(struct btree_iter_set
);
1737 if (!(c
->devices
= kzalloc(c
->nr_uuids
* sizeof(void *), GFP_KERNEL
)) ||
1738 !(c
->bio_meta
= mempool_create_kmalloc_pool(2,
1739 sizeof(struct bbio
) + sizeof(struct bio_vec
) *
1740 bucket_pages(c
))) ||
1741 !(c
->fill_iter
= mempool_create_kmalloc_pool(1, iter_size
)) ||
1742 !(c
->bio_split
= bioset_create(4, offsetof(struct bbio
, bio
),
1744 BIOSET_NEED_RESCUER
)) ||
1745 !(c
->uuids
= alloc_bucket_pages(GFP_KERNEL
, c
)) ||
1746 !(c
->moving_gc_wq
= alloc_workqueue("bcache_gc",
1747 WQ_MEM_RECLAIM
, 0)) ||
1748 bch_journal_alloc(c
) ||
1749 bch_btree_cache_alloc(c
) ||
1750 bch_open_buckets_alloc(c
) ||
1751 bch_bset_sort_state_init(&c
->sort
, ilog2(c
->btree_pages
)))
1754 c
->congested_read_threshold_us
= 2000;
1755 c
->congested_write_threshold_us
= 20000;
1756 c
->error_limit
= 8 << IO_ERROR_SHIFT
;
1757 WARN_ON(test_and_clear_bit(CACHE_SET_IO_DISABLE
, &c
->flags
));
1761 bch_cache_set_unregister(c
);
1765 static int run_cache_set(struct cache_set
*c
)
1767 const char *err
= "cannot allocate memory";
1768 struct cached_dev
*dc
, *t
;
1773 struct journal_replay
*l
;
1775 closure_init_stack(&cl
);
1777 for_each_cache(ca
, c
, i
)
1778 c
->nbuckets
+= ca
->sb
.nbuckets
;
1781 if (CACHE_SYNC(&c
->sb
)) {
1785 err
= "cannot allocate memory for journal";
1786 if (bch_journal_read(c
, &journal
))
1789 pr_debug("btree_journal_read() done");
1791 err
= "no journal entries found";
1792 if (list_empty(&journal
))
1795 j
= &list_entry(journal
.prev
, struct journal_replay
, list
)->j
;
1797 err
= "IO error reading priorities";
1798 for_each_cache(ca
, c
, i
)
1799 prio_read(ca
, j
->prio_bucket
[ca
->sb
.nr_this_dev
]);
1802 * If prio_read() fails it'll call cache_set_error and we'll
1803 * tear everything down right away, but if we perhaps checked
1804 * sooner we could avoid journal replay.
1809 err
= "bad btree root";
1810 if (__bch_btree_ptr_invalid(c
, k
))
1813 err
= "error reading btree root";
1814 c
->root
= bch_btree_node_get(c
, NULL
, k
, j
->btree_level
, true, NULL
);
1815 if (IS_ERR_OR_NULL(c
->root
))
1818 list_del_init(&c
->root
->list
);
1819 rw_unlock(true, c
->root
);
1821 err
= uuid_read(c
, j
, &cl
);
1825 err
= "error in recovery";
1826 if (bch_btree_check(c
))
1829 bch_journal_mark(c
, &journal
);
1830 bch_initial_gc_finish(c
);
1831 pr_debug("btree_check() done");
1834 * bcache_journal_next() can't happen sooner, or
1835 * btree_gc_finish() will give spurious errors about last_gc >
1836 * gc_gen - this is a hack but oh well.
1838 bch_journal_next(&c
->journal
);
1840 err
= "error starting allocator thread";
1841 for_each_cache(ca
, c
, i
)
1842 if (bch_cache_allocator_start(ca
))
1846 * First place it's safe to allocate: btree_check() and
1847 * btree_gc_finish() have to run before we have buckets to
1848 * allocate, and bch_bucket_alloc_set() might cause a journal
1849 * entry to be written so bcache_journal_next() has to be called
1852 * If the uuids were in the old format we have to rewrite them
1853 * before the next journal entry is written:
1855 if (j
->version
< BCACHE_JSET_VERSION_UUID
)
1858 err
= "bcache: replay journal failed";
1859 if (bch_journal_replay(c
, &journal
))
1862 pr_notice("invalidating existing data");
1864 for_each_cache(ca
, c
, i
) {
1867 ca
->sb
.keys
= clamp_t(int, ca
->sb
.nbuckets
>> 7,
1868 2, SB_JOURNAL_BUCKETS
);
1870 for (j
= 0; j
< ca
->sb
.keys
; j
++)
1871 ca
->sb
.d
[j
] = ca
->sb
.first_bucket
+ j
;
1874 bch_initial_gc_finish(c
);
1876 err
= "error starting allocator thread";
1877 for_each_cache(ca
, c
, i
)
1878 if (bch_cache_allocator_start(ca
))
1881 mutex_lock(&c
->bucket_lock
);
1882 for_each_cache(ca
, c
, i
)
1883 bch_prio_write(ca
, true);
1884 mutex_unlock(&c
->bucket_lock
);
1886 err
= "cannot allocate new UUID bucket";
1887 if (__uuid_write(c
))
1890 err
= "cannot allocate new btree root";
1891 c
->root
= __bch_btree_node_alloc(c
, NULL
, 0, true, NULL
);
1892 if (IS_ERR_OR_NULL(c
->root
))
1895 mutex_lock(&c
->root
->write_lock
);
1896 bkey_copy_key(&c
->root
->key
, &MAX_KEY
);
1897 bch_btree_node_write(c
->root
, &cl
);
1898 mutex_unlock(&c
->root
->write_lock
);
1900 bch_btree_set_root(c
->root
);
1901 rw_unlock(true, c
->root
);
1904 * We don't want to write the first journal entry until
1905 * everything is set up - fortunately journal entries won't be
1906 * written until the SET_CACHE_SYNC() here:
1908 SET_CACHE_SYNC(&c
->sb
, true);
1910 bch_journal_next(&c
->journal
);
1911 bch_journal_meta(c
, &cl
);
1914 err
= "error starting gc thread";
1915 if (bch_gc_thread_start(c
))
1919 c
->sb
.last_mount
= get_seconds();
1920 bcache_write_super(c
);
1922 list_for_each_entry_safe(dc
, t
, &uncached_devices
, list
)
1923 bch_cached_dev_attach(dc
, c
, NULL
);
1927 set_bit(CACHE_SET_RUNNING
, &c
->flags
);
1930 while (!list_empty(&journal
)) {
1931 l
= list_first_entry(&journal
, struct journal_replay
, list
);
1937 /* XXX: test this, it's broken */
1938 bch_cache_set_error(c
, "%s", err
);
1943 static bool can_attach_cache(struct cache
*ca
, struct cache_set
*c
)
1945 return ca
->sb
.block_size
== c
->sb
.block_size
&&
1946 ca
->sb
.bucket_size
== c
->sb
.bucket_size
&&
1947 ca
->sb
.nr_in_set
== c
->sb
.nr_in_set
;
1950 static const char *register_cache_set(struct cache
*ca
)
1953 const char *err
= "cannot allocate memory";
1954 struct cache_set
*c
;
1956 list_for_each_entry(c
, &bch_cache_sets
, list
)
1957 if (!memcmp(c
->sb
.set_uuid
, ca
->sb
.set_uuid
, 16)) {
1958 if (c
->cache
[ca
->sb
.nr_this_dev
])
1959 return "duplicate cache set member";
1961 if (!can_attach_cache(ca
, c
))
1962 return "cache sb does not match set";
1964 if (!CACHE_SYNC(&ca
->sb
))
1965 SET_CACHE_SYNC(&c
->sb
, false);
1970 c
= bch_cache_set_alloc(&ca
->sb
);
1974 err
= "error creating kobject";
1975 if (kobject_add(&c
->kobj
, bcache_kobj
, "%pU", c
->sb
.set_uuid
) ||
1976 kobject_add(&c
->internal
, &c
->kobj
, "internal"))
1979 if (bch_cache_accounting_add_kobjs(&c
->accounting
, &c
->kobj
))
1982 bch_debug_init_cache_set(c
);
1984 list_add(&c
->list
, &bch_cache_sets
);
1986 sprintf(buf
, "cache%i", ca
->sb
.nr_this_dev
);
1987 if (sysfs_create_link(&ca
->kobj
, &c
->kobj
, "set") ||
1988 sysfs_create_link(&c
->kobj
, &ca
->kobj
, buf
))
1991 if (ca
->sb
.seq
> c
->sb
.seq
) {
1992 c
->sb
.version
= ca
->sb
.version
;
1993 memcpy(c
->sb
.set_uuid
, ca
->sb
.set_uuid
, 16);
1994 c
->sb
.flags
= ca
->sb
.flags
;
1995 c
->sb
.seq
= ca
->sb
.seq
;
1996 pr_debug("set version = %llu", c
->sb
.version
);
1999 kobject_get(&ca
->kobj
);
2001 ca
->set
->cache
[ca
->sb
.nr_this_dev
] = ca
;
2002 c
->cache_by_alloc
[c
->caches_loaded
++] = ca
;
2004 if (c
->caches_loaded
== c
->sb
.nr_in_set
) {
2005 err
= "failed to run cache set";
2006 if (run_cache_set(c
) < 0)
2012 bch_cache_set_unregister(c
);
2018 void bch_cache_release(struct kobject
*kobj
)
2020 struct cache
*ca
= container_of(kobj
, struct cache
, kobj
);
2024 BUG_ON(ca
->set
->cache
[ca
->sb
.nr_this_dev
] != ca
);
2025 ca
->set
->cache
[ca
->sb
.nr_this_dev
] = NULL
;
2028 free_pages((unsigned long) ca
->disk_buckets
, ilog2(bucket_pages(ca
)));
2029 kfree(ca
->prio_buckets
);
2032 free_heap(&ca
->heap
);
2033 free_fifo(&ca
->free_inc
);
2035 for (i
= 0; i
< RESERVE_NR
; i
++)
2036 free_fifo(&ca
->free
[i
]);
2038 if (ca
->sb_bio
.bi_inline_vecs
[0].bv_page
)
2039 put_page(ca
->sb_bio
.bi_io_vec
[0].bv_page
);
2041 if (!IS_ERR_OR_NULL(ca
->bdev
))
2042 blkdev_put(ca
->bdev
, FMODE_READ
|FMODE_WRITE
|FMODE_EXCL
);
2045 module_put(THIS_MODULE
);
2048 static int cache_alloc(struct cache
*ca
)
2051 size_t btree_buckets
;
2054 __module_get(THIS_MODULE
);
2055 kobject_init(&ca
->kobj
, &bch_cache_ktype
);
2057 bio_init(&ca
->journal
.bio
, ca
->journal
.bio
.bi_inline_vecs
, 8);
2060 * when ca->sb.njournal_buckets is not zero, journal exists,
2061 * and in bch_journal_replay(), tree node may split,
2062 * so bucket of RESERVE_BTREE type is needed,
2063 * the worst situation is all journal buckets are valid journal,
2064 * and all the keys need to replay,
2065 * so the number of RESERVE_BTREE type buckets should be as much
2066 * as journal buckets
2068 btree_buckets
= ca
->sb
.njournal_buckets
?: 8;
2069 free
= roundup_pow_of_two(ca
->sb
.nbuckets
) >> 10;
2071 if (!init_fifo(&ca
->free
[RESERVE_BTREE
], btree_buckets
, GFP_KERNEL
) ||
2072 !init_fifo_exact(&ca
->free
[RESERVE_PRIO
], prio_buckets(ca
), GFP_KERNEL
) ||
2073 !init_fifo(&ca
->free
[RESERVE_MOVINGGC
], free
, GFP_KERNEL
) ||
2074 !init_fifo(&ca
->free
[RESERVE_NONE
], free
, GFP_KERNEL
) ||
2075 !init_fifo(&ca
->free_inc
, free
<< 2, GFP_KERNEL
) ||
2076 !init_heap(&ca
->heap
, free
<< 3, GFP_KERNEL
) ||
2077 !(ca
->buckets
= vzalloc(sizeof(struct bucket
) *
2078 ca
->sb
.nbuckets
)) ||
2079 !(ca
->prio_buckets
= kzalloc(sizeof(uint64_t) * prio_buckets(ca
) *
2081 !(ca
->disk_buckets
= alloc_bucket_pages(GFP_KERNEL
, ca
)))
2084 ca
->prio_last_buckets
= ca
->prio_buckets
+ prio_buckets(ca
);
2086 for_each_bucket(b
, ca
)
2087 atomic_set(&b
->pin
, 0);
2092 static int register_cache(struct cache_sb
*sb
, struct page
*sb_page
,
2093 struct block_device
*bdev
, struct cache
*ca
)
2095 const char *err
= NULL
; /* must be set for any error case */
2098 bdevname(bdev
, ca
->cache_dev_name
);
2099 memcpy(&ca
->sb
, sb
, sizeof(struct cache_sb
));
2101 ca
->bdev
->bd_holder
= ca
;
2103 bio_init(&ca
->sb_bio
, ca
->sb_bio
.bi_inline_vecs
, 1);
2104 ca
->sb_bio
.bi_io_vec
[0].bv_page
= sb_page
;
2107 if (blk_queue_discard(bdev_get_queue(bdev
)))
2108 ca
->discard
= CACHE_DISCARD(&ca
->sb
);
2110 ret
= cache_alloc(ca
);
2112 blkdev_put(bdev
, FMODE_READ
|FMODE_WRITE
|FMODE_EXCL
);
2114 err
= "cache_alloc(): -ENOMEM";
2116 err
= "cache_alloc(): unknown error";
2120 if (kobject_add(&ca
->kobj
, &part_to_dev(bdev
->bd_part
)->kobj
, "bcache")) {
2121 err
= "error calling kobject_add";
2126 mutex_lock(&bch_register_lock
);
2127 err
= register_cache_set(ca
);
2128 mutex_unlock(&bch_register_lock
);
2135 pr_info("registered cache device %s", ca
->cache_dev_name
);
2138 kobject_put(&ca
->kobj
);
2142 pr_notice("error %s: %s", ca
->cache_dev_name
, err
);
2147 /* Global interfaces/init */
2149 static ssize_t
register_bcache(struct kobject
*, struct kobj_attribute
*,
2150 const char *, size_t);
2152 kobj_attribute_write(register, register_bcache
);
2153 kobj_attribute_write(register_quiet
, register_bcache
);
2155 static bool bch_is_open_backing(struct block_device
*bdev
) {
2156 struct cache_set
*c
, *tc
;
2157 struct cached_dev
*dc
, *t
;
2159 list_for_each_entry_safe(c
, tc
, &bch_cache_sets
, list
)
2160 list_for_each_entry_safe(dc
, t
, &c
->cached_devs
, list
)
2161 if (dc
->bdev
== bdev
)
2163 list_for_each_entry_safe(dc
, t
, &uncached_devices
, list
)
2164 if (dc
->bdev
== bdev
)
2169 static struct cached_dev
*bch_find_cached_dev(struct block_device
*bdev
) {
2170 struct cache_set
*c
, *tc
;
2171 struct cached_dev
*dc
, *t
;
2173 list_for_each_entry_safe(c
, tc
, &bch_cache_sets
, list
)
2174 list_for_each_entry_safe(dc
, t
, &c
->cached_devs
, list
)
2175 if (dc
->bdev
== bdev
)
2177 list_for_each_entry_safe(dc
, t
, &uncached_devices
, list
)
2178 if (dc
->bdev
== bdev
)
2184 static bool bch_is_open_cache(struct block_device
*bdev
) {
2185 struct cache_set
*c
, *tc
;
2189 list_for_each_entry_safe(c
, tc
, &bch_cache_sets
, list
)
2190 for_each_cache(ca
, c
, i
)
2191 if (ca
->bdev
== bdev
)
2196 static bool bch_is_open(struct block_device
*bdev
) {
2197 return bch_is_open_cache(bdev
) || bch_is_open_backing(bdev
);
2200 static ssize_t
register_bcache(struct kobject
*k
, struct kobj_attribute
*attr
,
2201 const char *buffer
, size_t size
)
2204 const char *err
= "cannot allocate memory";
2206 struct cache_sb
*sb
= NULL
;
2207 struct block_device
*bdev
= NULL
;
2208 struct page
*sb_page
= NULL
;
2209 struct cached_dev
*dc
= NULL
;
2211 if (!try_module_get(THIS_MODULE
))
2214 if (!(path
= kstrndup(buffer
, size
, GFP_KERNEL
)) ||
2215 !(sb
= kmalloc(sizeof(struct cache_sb
), GFP_KERNEL
)))
2218 err
= "failed to open device";
2219 bdev
= blkdev_get_by_path(strim(path
),
2220 FMODE_READ
|FMODE_WRITE
|FMODE_EXCL
,
2223 if (bdev
== ERR_PTR(-EBUSY
)) {
2224 bdev
= lookup_bdev(strim(path
), 0);
2225 mutex_lock(&bch_register_lock
);
2226 if (!IS_ERR(bdev
) && bch_is_open(bdev
)) {
2227 err
= "device already registered";
2228 /* emit CHANGE event for backing devices to export
2229 * CACHED_{UUID/LABEL} values to udev */
2230 if (bch_is_open_backing(bdev
)) {
2231 dc
= bch_find_cached_dev(bdev
);
2233 bch_cached_dev_emit_change(dc
);
2234 err
= "device already registered (emitting change event)";
2238 err
= "device busy";
2240 mutex_unlock(&bch_register_lock
);
2243 if (attr
== &ksysfs_register_quiet
)
2249 err
= "failed to set blocksize";
2250 if (set_blocksize(bdev
, 4096))
2253 err
= read_super(sb
, bdev
, &sb_page
);
2257 err
= "failed to register device";
2258 if (SB_IS_BDEV(sb
)) {
2259 struct cached_dev
*dc
= kzalloc(sizeof(*dc
), GFP_KERNEL
);
2263 mutex_lock(&bch_register_lock
);
2264 register_bdev(sb
, sb_page
, bdev
, dc
);
2265 mutex_unlock(&bch_register_lock
);
2267 struct cache
*ca
= kzalloc(sizeof(*ca
), GFP_KERNEL
);
2271 if (register_cache(sb
, sb_page
, bdev
, ca
) != 0)
2279 module_put(THIS_MODULE
);
2283 blkdev_put(bdev
, FMODE_READ
|FMODE_WRITE
|FMODE_EXCL
);
2285 pr_info("error %s: %s", path
, err
);
2290 static int bcache_reboot(struct notifier_block
*n
, unsigned long code
, void *x
)
2292 if (code
== SYS_DOWN
||
2294 code
== SYS_POWER_OFF
) {
2296 unsigned long start
= jiffies
;
2297 bool stopped
= false;
2299 struct cache_set
*c
, *tc
;
2300 struct cached_dev
*dc
, *tdc
;
2302 mutex_lock(&bch_register_lock
);
2304 if (list_empty(&bch_cache_sets
) &&
2305 list_empty(&uncached_devices
))
2308 pr_info("Stopping all devices:");
2310 list_for_each_entry_safe(c
, tc
, &bch_cache_sets
, list
)
2311 bch_cache_set_stop(c
);
2313 list_for_each_entry_safe(dc
, tdc
, &uncached_devices
, list
)
2314 bcache_device_stop(&dc
->disk
);
2316 mutex_unlock(&bch_register_lock
);
2319 * Give an early chance for other kthreads and
2320 * kworkers to stop themselves
2324 /* What's a condition variable? */
2326 long timeout
= start
+ 10 * HZ
- jiffies
;
2328 mutex_lock(&bch_register_lock
);
2329 stopped
= list_empty(&bch_cache_sets
) &&
2330 list_empty(&uncached_devices
);
2332 if (timeout
< 0 || stopped
)
2335 prepare_to_wait(&unregister_wait
, &wait
,
2336 TASK_UNINTERRUPTIBLE
);
2338 mutex_unlock(&bch_register_lock
);
2339 schedule_timeout(timeout
);
2342 finish_wait(&unregister_wait
, &wait
);
2345 pr_info("All devices stopped");
2347 pr_notice("Timeout waiting for devices to be closed");
2349 mutex_unlock(&bch_register_lock
);
2355 static struct notifier_block reboot
= {
2356 .notifier_call
= bcache_reboot
,
2357 .priority
= INT_MAX
, /* before any real devices */
2360 static void bcache_exit(void)
2365 kobject_put(bcache_kobj
);
2367 destroy_workqueue(bcache_wq
);
2369 unregister_blkdev(bcache_major
, "bcache");
2370 unregister_reboot_notifier(&reboot
);
2371 mutex_destroy(&bch_register_lock
);
2374 static int __init
bcache_init(void)
2376 static const struct attribute
*files
[] = {
2377 &ksysfs_register
.attr
,
2378 &ksysfs_register_quiet
.attr
,
2382 mutex_init(&bch_register_lock
);
2383 init_waitqueue_head(&unregister_wait
);
2384 register_reboot_notifier(&reboot
);
2385 closure_debug_init();
2387 bcache_major
= register_blkdev(0, "bcache");
2388 if (bcache_major
< 0) {
2389 unregister_reboot_notifier(&reboot
);
2390 mutex_destroy(&bch_register_lock
);
2391 return bcache_major
;
2394 if (!(bcache_wq
= alloc_workqueue("bcache", WQ_MEM_RECLAIM
, 0)) ||
2395 !(bcache_kobj
= kobject_create_and_add("bcache", fs_kobj
)) ||
2396 bch_request_init() ||
2397 bch_debug_init(bcache_kobj
) ||
2398 sysfs_create_files(bcache_kobj
, files
))
2407 module_exit(bcache_exit
);
2408 module_init(bcache_init
);