2 * bcache setup/teardown code, and some metadata io - read a superblock and
3 * figure out what to do with it.
5 * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
6 * Copyright 2012 Google, Inc.
14 #include "writeback.h"
16 #include <linux/blkdev.h>
17 #include <linux/buffer_head.h>
18 #include <linux/debugfs.h>
19 #include <linux/genhd.h>
20 #include <linux/idr.h>
21 #include <linux/kthread.h>
22 #include <linux/module.h>
23 #include <linux/random.h>
24 #include <linux/reboot.h>
25 #include <linux/sysfs.h>
27 MODULE_LICENSE("GPL");
28 MODULE_AUTHOR("Kent Overstreet <kent.overstreet@gmail.com>");
30 static const char bcache_magic
[] = {
31 0xc6, 0x85, 0x73, 0xf6, 0x4e, 0x1a, 0x45, 0xca,
32 0x82, 0x65, 0xf5, 0x7f, 0x48, 0xba, 0x6d, 0x81
35 static const char invalid_uuid
[] = {
36 0xa0, 0x3e, 0xf8, 0xed, 0x3e, 0xe1, 0xb8, 0x78,
37 0xc8, 0x50, 0xfc, 0x5e, 0xcb, 0x16, 0xcd, 0x99
40 /* Default is -1; we skip past it for struct cached_dev's cache mode */
41 const char * const bch_cache_modes
[] = {
50 static struct kobject
*bcache_kobj
;
51 struct mutex bch_register_lock
;
52 LIST_HEAD(bch_cache_sets
);
53 static LIST_HEAD(uncached_devices
);
55 static int bcache_major
;
56 static DEFINE_IDA(bcache_minor
);
57 static wait_queue_head_t unregister_wait
;
58 struct workqueue_struct
*bcache_wq
;
60 #define BTREE_MAX_PAGES (256 * 1024 / PAGE_SIZE)
64 static const char *read_super(struct cache_sb
*sb
, struct block_device
*bdev
,
69 struct buffer_head
*bh
= __bread(bdev
, 1, SB_SIZE
);
75 s
= (struct cache_sb
*) bh
->b_data
;
77 sb
->offset
= le64_to_cpu(s
->offset
);
78 sb
->version
= le64_to_cpu(s
->version
);
80 memcpy(sb
->magic
, s
->magic
, 16);
81 memcpy(sb
->uuid
, s
->uuid
, 16);
82 memcpy(sb
->set_uuid
, s
->set_uuid
, 16);
83 memcpy(sb
->label
, s
->label
, SB_LABEL_SIZE
);
85 sb
->flags
= le64_to_cpu(s
->flags
);
86 sb
->seq
= le64_to_cpu(s
->seq
);
87 sb
->last_mount
= le32_to_cpu(s
->last_mount
);
88 sb
->first_bucket
= le16_to_cpu(s
->first_bucket
);
89 sb
->keys
= le16_to_cpu(s
->keys
);
91 for (i
= 0; i
< SB_JOURNAL_BUCKETS
; i
++)
92 sb
->d
[i
] = le64_to_cpu(s
->d
[i
]);
94 pr_debug("read sb version %llu, flags %llu, seq %llu, journal size %u",
95 sb
->version
, sb
->flags
, sb
->seq
, sb
->keys
);
97 err
= "Not a bcache superblock";
98 if (sb
->offset
!= SB_SECTOR
)
101 if (memcmp(sb
->magic
, bcache_magic
, 16))
104 err
= "Too many journal buckets";
105 if (sb
->keys
> SB_JOURNAL_BUCKETS
)
108 err
= "Bad checksum";
109 if (s
->csum
!= csum_set(s
))
113 if (bch_is_zero(sb
->uuid
, 16))
116 sb
->block_size
= le16_to_cpu(s
->block_size
);
118 err
= "Superblock block size smaller than device block size";
119 if (sb
->block_size
<< 9 < bdev_logical_block_size(bdev
))
122 switch (sb
->version
) {
123 case BCACHE_SB_VERSION_BDEV
:
124 sb
->data_offset
= BDEV_DATA_START_DEFAULT
;
126 case BCACHE_SB_VERSION_BDEV_WITH_OFFSET
:
127 sb
->data_offset
= le64_to_cpu(s
->data_offset
);
129 err
= "Bad data offset";
130 if (sb
->data_offset
< BDEV_DATA_START_DEFAULT
)
134 case BCACHE_SB_VERSION_CDEV
:
135 case BCACHE_SB_VERSION_CDEV_WITH_UUID
:
136 sb
->nbuckets
= le64_to_cpu(s
->nbuckets
);
137 sb
->bucket_size
= le16_to_cpu(s
->bucket_size
);
139 sb
->nr_in_set
= le16_to_cpu(s
->nr_in_set
);
140 sb
->nr_this_dev
= le16_to_cpu(s
->nr_this_dev
);
142 err
= "Too many buckets";
143 if (sb
->nbuckets
> LONG_MAX
)
146 err
= "Not enough buckets";
147 if (sb
->nbuckets
< 1 << 7)
150 err
= "Bad block/bucket size";
151 if (!is_power_of_2(sb
->block_size
) ||
152 sb
->block_size
> PAGE_SECTORS
||
153 !is_power_of_2(sb
->bucket_size
) ||
154 sb
->bucket_size
< PAGE_SECTORS
)
157 err
= "Invalid superblock: device too small";
158 if (get_capacity(bdev
->bd_disk
) < sb
->bucket_size
* sb
->nbuckets
)
162 if (bch_is_zero(sb
->set_uuid
, 16))
165 err
= "Bad cache device number in set";
166 if (!sb
->nr_in_set
||
167 sb
->nr_in_set
<= sb
->nr_this_dev
||
168 sb
->nr_in_set
> MAX_CACHES_PER_SET
)
171 err
= "Journal buckets not sequential";
172 for (i
= 0; i
< sb
->keys
; i
++)
173 if (sb
->d
[i
] != sb
->first_bucket
+ i
)
176 err
= "Too many journal buckets";
177 if (sb
->first_bucket
+ sb
->keys
> sb
->nbuckets
)
180 err
= "Invalid superblock: first bucket comes before end of super";
181 if (sb
->first_bucket
* sb
->bucket_size
< 16)
186 err
= "Unsupported superblock version";
190 sb
->last_mount
= get_seconds();
193 get_page(bh
->b_page
);
200 static void write_bdev_super_endio(struct bio
*bio
)
202 struct cached_dev
*dc
= bio
->bi_private
;
203 /* XXX: error checking */
205 closure_put(&dc
->sb_write
);
208 static void __write_super(struct cache_sb
*sb
, struct bio
*bio
)
210 struct cache_sb
*out
= page_address(bio
->bi_io_vec
[0].bv_page
);
213 bio
->bi_iter
.bi_sector
= SB_SECTOR
;
214 bio
->bi_iter
.bi_size
= SB_SIZE
;
215 bio_set_op_attrs(bio
, REQ_OP_WRITE
, REQ_SYNC
|REQ_META
);
216 bch_bio_map(bio
, NULL
);
218 out
->offset
= cpu_to_le64(sb
->offset
);
219 out
->version
= cpu_to_le64(sb
->version
);
221 memcpy(out
->uuid
, sb
->uuid
, 16);
222 memcpy(out
->set_uuid
, sb
->set_uuid
, 16);
223 memcpy(out
->label
, sb
->label
, SB_LABEL_SIZE
);
225 out
->flags
= cpu_to_le64(sb
->flags
);
226 out
->seq
= cpu_to_le64(sb
->seq
);
228 out
->last_mount
= cpu_to_le32(sb
->last_mount
);
229 out
->first_bucket
= cpu_to_le16(sb
->first_bucket
);
230 out
->keys
= cpu_to_le16(sb
->keys
);
232 for (i
= 0; i
< sb
->keys
; i
++)
233 out
->d
[i
] = cpu_to_le64(sb
->d
[i
]);
235 out
->csum
= csum_set(out
);
237 pr_debug("ver %llu, flags %llu, seq %llu",
238 sb
->version
, sb
->flags
, sb
->seq
);
243 static void bch_write_bdev_super_unlock(struct closure
*cl
)
245 struct cached_dev
*dc
= container_of(cl
, struct cached_dev
, sb_write
);
247 up(&dc
->sb_write_mutex
);
250 void bch_write_bdev_super(struct cached_dev
*dc
, struct closure
*parent
)
252 struct closure
*cl
= &dc
->sb_write
;
253 struct bio
*bio
= &dc
->sb_bio
;
255 down(&dc
->sb_write_mutex
);
256 closure_init(cl
, parent
);
259 bio
->bi_bdev
= dc
->bdev
;
260 bio
->bi_end_io
= write_bdev_super_endio
;
261 bio
->bi_private
= dc
;
264 __write_super(&dc
->sb
, bio
);
266 closure_return_with_destructor(cl
, bch_write_bdev_super_unlock
);
269 static void write_super_endio(struct bio
*bio
)
271 struct cache
*ca
= bio
->bi_private
;
273 bch_count_io_errors(ca
, bio
->bi_error
, "writing superblock");
274 closure_put(&ca
->set
->sb_write
);
277 static void bcache_write_super_unlock(struct closure
*cl
)
279 struct cache_set
*c
= container_of(cl
, struct cache_set
, sb_write
);
281 up(&c
->sb_write_mutex
);
284 void bcache_write_super(struct cache_set
*c
)
286 struct closure
*cl
= &c
->sb_write
;
290 down(&c
->sb_write_mutex
);
291 closure_init(cl
, &c
->cl
);
295 for_each_cache(ca
, c
, i
) {
296 struct bio
*bio
= &ca
->sb_bio
;
298 ca
->sb
.version
= BCACHE_SB_VERSION_CDEV_WITH_UUID
;
299 ca
->sb
.seq
= c
->sb
.seq
;
300 ca
->sb
.last_mount
= c
->sb
.last_mount
;
302 SET_CACHE_SYNC(&ca
->sb
, CACHE_SYNC(&c
->sb
));
305 bio
->bi_bdev
= ca
->bdev
;
306 bio
->bi_end_io
= write_super_endio
;
307 bio
->bi_private
= ca
;
310 __write_super(&ca
->sb
, bio
);
313 closure_return_with_destructor(cl
, bcache_write_super_unlock
);
318 static void uuid_endio(struct bio
*bio
)
320 struct closure
*cl
= bio
->bi_private
;
321 struct cache_set
*c
= container_of(cl
, struct cache_set
, uuid_write
);
323 cache_set_err_on(bio
->bi_error
, c
, "accessing uuids");
324 bch_bbio_free(bio
, c
);
328 static void uuid_io_unlock(struct closure
*cl
)
330 struct cache_set
*c
= container_of(cl
, struct cache_set
, uuid_write
);
332 up(&c
->uuid_write_mutex
);
335 static void uuid_io(struct cache_set
*c
, int op
, unsigned long op_flags
,
336 struct bkey
*k
, struct closure
*parent
)
338 struct closure
*cl
= &c
->uuid_write
;
339 struct uuid_entry
*u
;
344 down(&c
->uuid_write_mutex
);
345 closure_init(cl
, parent
);
347 for (i
= 0; i
< KEY_PTRS(k
); i
++) {
348 struct bio
*bio
= bch_bbio_alloc(c
);
350 bio
->bi_opf
= REQ_SYNC
| REQ_META
| op_flags
;
351 bio
->bi_iter
.bi_size
= KEY_SIZE(k
) << 9;
353 bio
->bi_end_io
= uuid_endio
;
354 bio
->bi_private
= cl
;
355 bio_set_op_attrs(bio
, op
, REQ_SYNC
|REQ_META
|op_flags
);
356 bch_bio_map(bio
, c
->uuids
);
358 bch_submit_bbio(bio
, c
, k
, i
);
360 if (op
!= REQ_OP_WRITE
)
364 bch_extent_to_text(buf
, sizeof(buf
), k
);
365 pr_debug("%s UUIDs at %s", op
== REQ_OP_WRITE
? "wrote" : "read", buf
);
367 for (u
= c
->uuids
; u
< c
->uuids
+ c
->nr_uuids
; u
++)
368 if (!bch_is_zero(u
->uuid
, 16))
369 pr_debug("Slot %zi: %pU: %s: 1st: %u last: %u inv: %u",
370 u
- c
->uuids
, u
->uuid
, u
->label
,
371 u
->first_reg
, u
->last_reg
, u
->invalidated
);
373 closure_return_with_destructor(cl
, uuid_io_unlock
);
376 static char *uuid_read(struct cache_set
*c
, struct jset
*j
, struct closure
*cl
)
378 struct bkey
*k
= &j
->uuid_bucket
;
380 if (__bch_btree_ptr_invalid(c
, k
))
381 return "bad uuid pointer";
383 bkey_copy(&c
->uuid_bucket
, k
);
384 uuid_io(c
, REQ_OP_READ
, READ_SYNC
, k
, cl
);
386 if (j
->version
< BCACHE_JSET_VERSION_UUIDv1
) {
387 struct uuid_entry_v0
*u0
= (void *) c
->uuids
;
388 struct uuid_entry
*u1
= (void *) c
->uuids
;
394 * Since the new uuid entry is bigger than the old, we have to
395 * convert starting at the highest memory address and work down
396 * in order to do it in place
399 for (i
= c
->nr_uuids
- 1;
402 memcpy(u1
[i
].uuid
, u0
[i
].uuid
, 16);
403 memcpy(u1
[i
].label
, u0
[i
].label
, 32);
405 u1
[i
].first_reg
= u0
[i
].first_reg
;
406 u1
[i
].last_reg
= u0
[i
].last_reg
;
407 u1
[i
].invalidated
= u0
[i
].invalidated
;
417 static int __uuid_write(struct cache_set
*c
)
421 closure_init_stack(&cl
);
423 lockdep_assert_held(&bch_register_lock
);
425 if (bch_bucket_alloc_set(c
, RESERVE_BTREE
, &k
.key
, 1, true))
428 SET_KEY_SIZE(&k
.key
, c
->sb
.bucket_size
);
429 uuid_io(c
, REQ_OP_WRITE
, 0, &k
.key
, &cl
);
432 bkey_copy(&c
->uuid_bucket
, &k
.key
);
437 int bch_uuid_write(struct cache_set
*c
)
439 int ret
= __uuid_write(c
);
442 bch_journal_meta(c
, NULL
);
447 static struct uuid_entry
*uuid_find(struct cache_set
*c
, const char *uuid
)
449 struct uuid_entry
*u
;
452 u
< c
->uuids
+ c
->nr_uuids
; u
++)
453 if (!memcmp(u
->uuid
, uuid
, 16))
459 static struct uuid_entry
*uuid_find_empty(struct cache_set
*c
)
461 static const char zero_uuid
[16] = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";
462 return uuid_find(c
, zero_uuid
);
466 * Bucket priorities/gens:
468 * For each bucket, we store on disk its
472 * See alloc.c for an explanation of the gen. The priority is used to implement
473 * lru (and in the future other) cache replacement policies; for most purposes
474 * it's just an opaque integer.
476 * The gens and the priorities don't have a whole lot to do with each other, and
477 * it's actually the gens that must be written out at specific times - it's no
478 * big deal if the priorities don't get written, if we lose them we just reuse
479 * buckets in suboptimal order.
481 * On disk they're stored in a packed array, and in as many buckets are required
482 * to fit them all. The buckets we use to store them form a list; the journal
483 * header points to the first bucket, the first bucket points to the second
486 * This code is used by the allocation code; periodically (whenever it runs out
487 * of buckets to allocate from) the allocation code will invalidate some
488 * buckets, but it can't use those buckets until their new gens are safely on
492 static void prio_endio(struct bio
*bio
)
494 struct cache
*ca
= bio
->bi_private
;
496 cache_set_err_on(bio
->bi_error
, ca
->set
, "accessing priorities");
497 bch_bbio_free(bio
, ca
->set
);
498 closure_put(&ca
->prio
);
501 static void prio_io(struct cache
*ca
, uint64_t bucket
, int op
,
502 unsigned long op_flags
)
504 struct closure
*cl
= &ca
->prio
;
505 struct bio
*bio
= bch_bbio_alloc(ca
->set
);
507 closure_init_stack(cl
);
509 bio
->bi_iter
.bi_sector
= bucket
* ca
->sb
.bucket_size
;
510 bio
->bi_bdev
= ca
->bdev
;
511 bio
->bi_iter
.bi_size
= bucket_bytes(ca
);
513 bio
->bi_end_io
= prio_endio
;
514 bio
->bi_private
= ca
;
515 bio_set_op_attrs(bio
, op
, REQ_SYNC
|REQ_META
|op_flags
);
516 bch_bio_map(bio
, ca
->disk_buckets
);
518 closure_bio_submit(bio
, &ca
->prio
);
522 void bch_prio_write(struct cache
*ca
)
528 closure_init_stack(&cl
);
530 lockdep_assert_held(&ca
->set
->bucket_lock
);
532 ca
->disk_buckets
->seq
++;
534 atomic_long_add(ca
->sb
.bucket_size
* prio_buckets(ca
),
535 &ca
->meta_sectors_written
);
537 //pr_debug("free %zu, free_inc %zu, unused %zu", fifo_used(&ca->free),
538 // fifo_used(&ca->free_inc), fifo_used(&ca->unused));
540 for (i
= prio_buckets(ca
) - 1; i
>= 0; --i
) {
542 struct prio_set
*p
= ca
->disk_buckets
;
543 struct bucket_disk
*d
= p
->data
;
544 struct bucket_disk
*end
= d
+ prios_per_bucket(ca
);
546 for (b
= ca
->buckets
+ i
* prios_per_bucket(ca
);
547 b
< ca
->buckets
+ ca
->sb
.nbuckets
&& d
< end
;
549 d
->prio
= cpu_to_le16(b
->prio
);
553 p
->next_bucket
= ca
->prio_buckets
[i
+ 1];
554 p
->magic
= pset_magic(&ca
->sb
);
555 p
->csum
= bch_crc64(&p
->magic
, bucket_bytes(ca
) - 8);
557 bucket
= bch_bucket_alloc(ca
, RESERVE_PRIO
, true);
558 BUG_ON(bucket
== -1);
560 mutex_unlock(&ca
->set
->bucket_lock
);
561 prio_io(ca
, bucket
, REQ_OP_WRITE
, 0);
562 mutex_lock(&ca
->set
->bucket_lock
);
564 ca
->prio_buckets
[i
] = bucket
;
565 atomic_dec_bug(&ca
->buckets
[bucket
].pin
);
568 mutex_unlock(&ca
->set
->bucket_lock
);
570 bch_journal_meta(ca
->set
, &cl
);
573 mutex_lock(&ca
->set
->bucket_lock
);
576 * Don't want the old priorities to get garbage collected until after we
577 * finish writing the new ones, and they're journalled
579 for (i
= 0; i
< prio_buckets(ca
); i
++) {
580 if (ca
->prio_last_buckets
[i
])
581 __bch_bucket_free(ca
,
582 &ca
->buckets
[ca
->prio_last_buckets
[i
]]);
584 ca
->prio_last_buckets
[i
] = ca
->prio_buckets
[i
];
588 static void prio_read(struct cache
*ca
, uint64_t bucket
)
590 struct prio_set
*p
= ca
->disk_buckets
;
591 struct bucket_disk
*d
= p
->data
+ prios_per_bucket(ca
), *end
= d
;
593 unsigned bucket_nr
= 0;
595 for (b
= ca
->buckets
;
596 b
< ca
->buckets
+ ca
->sb
.nbuckets
;
599 ca
->prio_buckets
[bucket_nr
] = bucket
;
600 ca
->prio_last_buckets
[bucket_nr
] = bucket
;
603 prio_io(ca
, bucket
, REQ_OP_READ
, READ_SYNC
);
605 if (p
->csum
!= bch_crc64(&p
->magic
, bucket_bytes(ca
) - 8))
606 pr_warn("bad csum reading priorities");
608 if (p
->magic
!= pset_magic(&ca
->sb
))
609 pr_warn("bad magic reading priorities");
611 bucket
= p
->next_bucket
;
615 b
->prio
= le16_to_cpu(d
->prio
);
616 b
->gen
= b
->last_gc
= d
->gen
;
622 static int open_dev(struct block_device
*b
, fmode_t mode
)
624 struct bcache_device
*d
= b
->bd_disk
->private_data
;
625 if (test_bit(BCACHE_DEV_CLOSING
, &d
->flags
))
632 static void release_dev(struct gendisk
*b
, fmode_t mode
)
634 struct bcache_device
*d
= b
->private_data
;
638 static int ioctl_dev(struct block_device
*b
, fmode_t mode
,
639 unsigned int cmd
, unsigned long arg
)
641 struct bcache_device
*d
= b
->bd_disk
->private_data
;
642 return d
->ioctl(d
, mode
, cmd
, arg
);
645 static const struct block_device_operations bcache_ops
= {
647 .release
= release_dev
,
649 .owner
= THIS_MODULE
,
652 void bcache_device_stop(struct bcache_device
*d
)
654 if (!test_and_set_bit(BCACHE_DEV_CLOSING
, &d
->flags
))
655 closure_queue(&d
->cl
);
658 static void bcache_device_unlink(struct bcache_device
*d
)
660 lockdep_assert_held(&bch_register_lock
);
662 if (d
->c
&& !test_and_set_bit(BCACHE_DEV_UNLINK_DONE
, &d
->flags
)) {
666 sysfs_remove_link(&d
->c
->kobj
, d
->name
);
667 sysfs_remove_link(&d
->kobj
, "cache");
669 for_each_cache(ca
, d
->c
, i
)
670 bd_unlink_disk_holder(ca
->bdev
, d
->disk
);
674 static void bcache_device_link(struct bcache_device
*d
, struct cache_set
*c
,
680 for_each_cache(ca
, d
->c
, i
)
681 bd_link_disk_holder(ca
->bdev
, d
->disk
);
683 snprintf(d
->name
, BCACHEDEVNAME_SIZE
,
684 "%s%u", name
, d
->id
);
686 WARN(sysfs_create_link(&d
->kobj
, &c
->kobj
, "cache") ||
687 sysfs_create_link(&c
->kobj
, &d
->kobj
, d
->name
),
688 "Couldn't create device <-> cache set symlinks");
690 clear_bit(BCACHE_DEV_UNLINK_DONE
, &d
->flags
);
693 static void bcache_device_detach(struct bcache_device
*d
)
695 lockdep_assert_held(&bch_register_lock
);
697 if (test_bit(BCACHE_DEV_DETACHING
, &d
->flags
)) {
698 struct uuid_entry
*u
= d
->c
->uuids
+ d
->id
;
700 SET_UUID_FLASH_ONLY(u
, 0);
701 memcpy(u
->uuid
, invalid_uuid
, 16);
702 u
->invalidated
= cpu_to_le32(get_seconds());
703 bch_uuid_write(d
->c
);
706 bcache_device_unlink(d
);
708 d
->c
->devices
[d
->id
] = NULL
;
709 closure_put(&d
->c
->caching
);
713 static void bcache_device_attach(struct bcache_device
*d
, struct cache_set
*c
,
720 closure_get(&c
->caching
);
723 static void bcache_device_free(struct bcache_device
*d
)
725 lockdep_assert_held(&bch_register_lock
);
727 pr_info("%s stopped", d
->disk
->disk_name
);
730 bcache_device_detach(d
);
731 if (d
->disk
&& d
->disk
->flags
& GENHD_FL_UP
)
732 del_gendisk(d
->disk
);
733 if (d
->disk
&& d
->disk
->queue
)
734 blk_cleanup_queue(d
->disk
->queue
);
736 ida_simple_remove(&bcache_minor
, d
->disk
->first_minor
);
741 bioset_free(d
->bio_split
);
742 kvfree(d
->full_dirty_stripes
);
743 kvfree(d
->stripe_sectors_dirty
);
745 closure_debug_destroy(&d
->cl
);
748 static int bcache_device_init(struct bcache_device
*d
, unsigned block_size
,
751 struct request_queue
*q
;
756 d
->stripe_size
= 1 << 31;
758 d
->nr_stripes
= DIV_ROUND_UP_ULL(sectors
, d
->stripe_size
);
760 if (!d
->nr_stripes
||
761 d
->nr_stripes
> INT_MAX
||
762 d
->nr_stripes
> SIZE_MAX
/ sizeof(atomic_t
)) {
763 pr_err("nr_stripes too large or invalid: %u (start sector beyond end of disk?)",
764 (unsigned)d
->nr_stripes
);
768 n
= d
->nr_stripes
* sizeof(atomic_t
);
769 d
->stripe_sectors_dirty
= n
< PAGE_SIZE
<< 6
770 ? kzalloc(n
, GFP_KERNEL
)
772 if (!d
->stripe_sectors_dirty
)
775 n
= BITS_TO_LONGS(d
->nr_stripes
) * sizeof(unsigned long);
776 d
->full_dirty_stripes
= n
< PAGE_SIZE
<< 6
777 ? kzalloc(n
, GFP_KERNEL
)
779 if (!d
->full_dirty_stripes
)
782 minor
= ida_simple_get(&bcache_minor
, 0, MINORMASK
+ 1, GFP_KERNEL
);
786 if (!(d
->bio_split
= bioset_create(4, offsetof(struct bbio
, bio
))) ||
787 !(d
->disk
= alloc_disk(1))) {
788 ida_simple_remove(&bcache_minor
, minor
);
792 set_capacity(d
->disk
, sectors
);
793 snprintf(d
->disk
->disk_name
, DISK_NAME_LEN
, "bcache%i", minor
);
795 d
->disk
->major
= bcache_major
;
796 d
->disk
->first_minor
= minor
;
797 d
->disk
->fops
= &bcache_ops
;
798 d
->disk
->private_data
= d
;
800 q
= blk_alloc_queue(GFP_KERNEL
);
804 blk_queue_make_request(q
, NULL
);
807 q
->backing_dev_info
.congested_data
= d
;
808 q
->limits
.max_hw_sectors
= UINT_MAX
;
809 q
->limits
.max_sectors
= UINT_MAX
;
810 q
->limits
.max_segment_size
= UINT_MAX
;
811 q
->limits
.max_segments
= BIO_MAX_PAGES
;
812 blk_queue_max_discard_sectors(q
, UINT_MAX
);
813 q
->limits
.discard_granularity
= 512;
814 q
->limits
.io_min
= block_size
;
815 q
->limits
.logical_block_size
= block_size
;
816 q
->limits
.physical_block_size
= block_size
;
817 set_bit(QUEUE_FLAG_NONROT
, &d
->disk
->queue
->queue_flags
);
818 clear_bit(QUEUE_FLAG_ADD_RANDOM
, &d
->disk
->queue
->queue_flags
);
819 set_bit(QUEUE_FLAG_DISCARD
, &d
->disk
->queue
->queue_flags
);
821 blk_queue_write_cache(q
, true, true);
828 static void calc_cached_dev_sectors(struct cache_set
*c
)
830 uint64_t sectors
= 0;
831 struct cached_dev
*dc
;
833 list_for_each_entry(dc
, &c
->cached_devs
, list
)
834 sectors
+= bdev_sectors(dc
->bdev
);
836 c
->cached_dev_sectors
= sectors
;
839 void bch_cached_dev_run(struct cached_dev
*dc
)
841 struct bcache_device
*d
= &dc
->disk
;
842 char buf
[SB_LABEL_SIZE
+ 1];
845 kasprintf(GFP_KERNEL
, "CACHED_UUID=%pU", dc
->sb
.uuid
),
850 memcpy(buf
, dc
->sb
.label
, SB_LABEL_SIZE
);
851 buf
[SB_LABEL_SIZE
] = '\0';
852 env
[2] = kasprintf(GFP_KERNEL
, "CACHED_LABEL=%s", buf
);
854 if (atomic_xchg(&dc
->running
, 1)) {
861 BDEV_STATE(&dc
->sb
) != BDEV_STATE_NONE
) {
863 closure_init_stack(&cl
);
865 SET_BDEV_STATE(&dc
->sb
, BDEV_STATE_STALE
);
866 bch_write_bdev_super(dc
, &cl
);
871 bd_link_disk_holder(dc
->bdev
, dc
->disk
.disk
);
872 /* won't show up in the uevent file, use udevadm monitor -e instead
873 * only class / kset properties are persistent */
874 kobject_uevent_env(&disk_to_dev(d
->disk
)->kobj
, KOBJ_CHANGE
, env
);
878 if (sysfs_create_link(&d
->kobj
, &disk_to_dev(d
->disk
)->kobj
, "dev") ||
879 sysfs_create_link(&disk_to_dev(d
->disk
)->kobj
, &d
->kobj
, "bcache"))
880 pr_debug("error creating sysfs link");
883 static void cached_dev_detach_finish(struct work_struct
*w
)
885 struct cached_dev
*dc
= container_of(w
, struct cached_dev
, detach
);
886 char buf
[BDEVNAME_SIZE
];
888 closure_init_stack(&cl
);
890 BUG_ON(!test_bit(BCACHE_DEV_DETACHING
, &dc
->disk
.flags
));
891 BUG_ON(atomic_read(&dc
->count
));
893 mutex_lock(&bch_register_lock
);
895 memset(&dc
->sb
.set_uuid
, 0, 16);
896 SET_BDEV_STATE(&dc
->sb
, BDEV_STATE_NONE
);
898 bch_write_bdev_super(dc
, &cl
);
901 bcache_device_detach(&dc
->disk
);
902 list_move(&dc
->list
, &uncached_devices
);
904 clear_bit(BCACHE_DEV_DETACHING
, &dc
->disk
.flags
);
905 clear_bit(BCACHE_DEV_UNLINK_DONE
, &dc
->disk
.flags
);
907 mutex_unlock(&bch_register_lock
);
909 pr_info("Caching disabled for %s", bdevname(dc
->bdev
, buf
));
911 /* Drop ref we took in cached_dev_detach() */
912 closure_put(&dc
->disk
.cl
);
915 void bch_cached_dev_detach(struct cached_dev
*dc
)
917 lockdep_assert_held(&bch_register_lock
);
919 if (test_bit(BCACHE_DEV_CLOSING
, &dc
->disk
.flags
))
922 if (test_and_set_bit(BCACHE_DEV_DETACHING
, &dc
->disk
.flags
))
926 * Block the device from being closed and freed until we're finished
929 closure_get(&dc
->disk
.cl
);
931 bch_writeback_queue(dc
);
935 int bch_cached_dev_attach(struct cached_dev
*dc
, struct cache_set
*c
)
937 uint32_t rtime
= cpu_to_le32(get_seconds());
938 struct uuid_entry
*u
;
939 char buf
[BDEVNAME_SIZE
];
941 bdevname(dc
->bdev
, buf
);
943 if (memcmp(dc
->sb
.set_uuid
, c
->sb
.set_uuid
, 16))
947 pr_err("Can't attach %s: already attached", buf
);
951 if (test_bit(CACHE_SET_STOPPING
, &c
->flags
)) {
952 pr_err("Can't attach %s: shutting down", buf
);
956 if (dc
->sb
.block_size
< c
->sb
.block_size
) {
958 pr_err("Couldn't attach %s: block size less than set's block size",
963 u
= uuid_find(c
, dc
->sb
.uuid
);
966 (BDEV_STATE(&dc
->sb
) == BDEV_STATE_STALE
||
967 BDEV_STATE(&dc
->sb
) == BDEV_STATE_NONE
)) {
968 memcpy(u
->uuid
, invalid_uuid
, 16);
969 u
->invalidated
= cpu_to_le32(get_seconds());
974 if (BDEV_STATE(&dc
->sb
) == BDEV_STATE_DIRTY
) {
975 pr_err("Couldn't find uuid for %s in set", buf
);
979 u
= uuid_find_empty(c
);
981 pr_err("Not caching %s, no room for UUID", buf
);
986 /* Deadlocks since we're called via sysfs...
987 sysfs_remove_file(&dc->kobj, &sysfs_attach);
990 if (bch_is_zero(u
->uuid
, 16)) {
992 closure_init_stack(&cl
);
994 memcpy(u
->uuid
, dc
->sb
.uuid
, 16);
995 memcpy(u
->label
, dc
->sb
.label
, SB_LABEL_SIZE
);
996 u
->first_reg
= u
->last_reg
= rtime
;
999 memcpy(dc
->sb
.set_uuid
, c
->sb
.set_uuid
, 16);
1000 SET_BDEV_STATE(&dc
->sb
, BDEV_STATE_CLEAN
);
1002 bch_write_bdev_super(dc
, &cl
);
1005 u
->last_reg
= rtime
;
1009 bcache_device_attach(&dc
->disk
, c
, u
- c
->uuids
);
1010 list_move(&dc
->list
, &c
->cached_devs
);
1011 calc_cached_dev_sectors(c
);
1015 * dc->c must be set before dc->count != 0 - paired with the mb in
1018 atomic_set(&dc
->count
, 1);
1020 /* Block writeback thread, but spawn it */
1021 down_write(&dc
->writeback_lock
);
1022 if (bch_cached_dev_writeback_start(dc
)) {
1023 up_write(&dc
->writeback_lock
);
1027 if (BDEV_STATE(&dc
->sb
) == BDEV_STATE_DIRTY
) {
1028 bch_sectors_dirty_init(dc
);
1029 atomic_set(&dc
->has_dirty
, 1);
1030 atomic_inc(&dc
->count
);
1031 bch_writeback_queue(dc
);
1034 bch_cached_dev_run(dc
);
1035 bcache_device_link(&dc
->disk
, c
, "bdev");
1037 /* Allow the writeback thread to proceed */
1038 up_write(&dc
->writeback_lock
);
1040 pr_info("Caching %s as %s on set %pU",
1041 bdevname(dc
->bdev
, buf
), dc
->disk
.disk
->disk_name
,
1042 dc
->disk
.c
->sb
.set_uuid
);
1046 void bch_cached_dev_release(struct kobject
*kobj
)
1048 struct cached_dev
*dc
= container_of(kobj
, struct cached_dev
,
1051 module_put(THIS_MODULE
);
1054 static void cached_dev_free(struct closure
*cl
)
1056 struct cached_dev
*dc
= container_of(cl
, struct cached_dev
, disk
.cl
);
1058 cancel_delayed_work_sync(&dc
->writeback_rate_update
);
1059 if (!IS_ERR_OR_NULL(dc
->writeback_thread
))
1060 kthread_stop(dc
->writeback_thread
);
1062 mutex_lock(&bch_register_lock
);
1064 if (atomic_read(&dc
->running
))
1065 bd_unlink_disk_holder(dc
->bdev
, dc
->disk
.disk
);
1066 bcache_device_free(&dc
->disk
);
1067 list_del(&dc
->list
);
1069 mutex_unlock(&bch_register_lock
);
1071 if (!IS_ERR_OR_NULL(dc
->bdev
))
1072 blkdev_put(dc
->bdev
, FMODE_READ
|FMODE_WRITE
|FMODE_EXCL
);
1074 wake_up(&unregister_wait
);
1076 kobject_put(&dc
->disk
.kobj
);
1079 static void cached_dev_flush(struct closure
*cl
)
1081 struct cached_dev
*dc
= container_of(cl
, struct cached_dev
, disk
.cl
);
1082 struct bcache_device
*d
= &dc
->disk
;
1084 mutex_lock(&bch_register_lock
);
1085 bcache_device_unlink(d
);
1086 mutex_unlock(&bch_register_lock
);
1088 bch_cache_accounting_destroy(&dc
->accounting
);
1089 kobject_del(&d
->kobj
);
1091 continue_at(cl
, cached_dev_free
, system_wq
);
1094 static int cached_dev_init(struct cached_dev
*dc
, unsigned block_size
)
1098 struct request_queue
*q
= bdev_get_queue(dc
->bdev
);
1100 __module_get(THIS_MODULE
);
1101 INIT_LIST_HEAD(&dc
->list
);
1102 closure_init(&dc
->disk
.cl
, NULL
);
1103 set_closure_fn(&dc
->disk
.cl
, cached_dev_flush
, system_wq
);
1104 kobject_init(&dc
->disk
.kobj
, &bch_cached_dev_ktype
);
1105 INIT_WORK(&dc
->detach
, cached_dev_detach_finish
);
1106 sema_init(&dc
->sb_write_mutex
, 1);
1107 INIT_LIST_HEAD(&dc
->io_lru
);
1108 spin_lock_init(&dc
->io_lock
);
1109 bch_cache_accounting_init(&dc
->accounting
, &dc
->disk
.cl
);
1111 dc
->sequential_cutoff
= 4 << 20;
1113 for (io
= dc
->io
; io
< dc
->io
+ RECENT_IO
; io
++) {
1114 list_add(&io
->lru
, &dc
->io_lru
);
1115 hlist_add_head(&io
->hash
, dc
->io_hash
+ RECENT_IO
);
1118 dc
->disk
.stripe_size
= q
->limits
.io_opt
>> 9;
1120 if (dc
->disk
.stripe_size
)
1121 dc
->partial_stripes_expensive
=
1122 q
->limits
.raid_partial_stripes_expensive
;
1124 ret
= bcache_device_init(&dc
->disk
, block_size
,
1125 dc
->bdev
->bd_part
->nr_sects
- dc
->sb
.data_offset
);
1129 set_capacity(dc
->disk
.disk
,
1130 dc
->bdev
->bd_part
->nr_sects
- dc
->sb
.data_offset
);
1132 dc
->disk
.disk
->queue
->backing_dev_info
.ra_pages
=
1133 max(dc
->disk
.disk
->queue
->backing_dev_info
.ra_pages
,
1134 q
->backing_dev_info
.ra_pages
);
1136 bch_cached_dev_request_init(dc
);
1137 bch_cached_dev_writeback_init(dc
);
1141 /* Cached device - bcache superblock */
1143 static void register_bdev(struct cache_sb
*sb
, struct page
*sb_page
,
1144 struct block_device
*bdev
,
1145 struct cached_dev
*dc
)
1147 char name
[BDEVNAME_SIZE
];
1148 const char *err
= "cannot allocate memory";
1149 struct cache_set
*c
;
1151 memcpy(&dc
->sb
, sb
, sizeof(struct cache_sb
));
1153 dc
->bdev
->bd_holder
= dc
;
1155 bio_init(&dc
->sb_bio
);
1156 dc
->sb_bio
.bi_max_vecs
= 1;
1157 dc
->sb_bio
.bi_io_vec
= dc
->sb_bio
.bi_inline_vecs
;
1158 dc
->sb_bio
.bi_io_vec
[0].bv_page
= sb_page
;
1161 if (cached_dev_init(dc
, sb
->block_size
<< 9))
1164 err
= "error creating kobject";
1165 if (kobject_add(&dc
->disk
.kobj
, &part_to_dev(bdev
->bd_part
)->kobj
,
1168 if (bch_cache_accounting_add_kobjs(&dc
->accounting
, &dc
->disk
.kobj
))
1171 pr_info("registered backing device %s", bdevname(bdev
, name
));
1173 list_add(&dc
->list
, &uncached_devices
);
1174 list_for_each_entry(c
, &bch_cache_sets
, list
)
1175 bch_cached_dev_attach(dc
, c
);
1177 if (BDEV_STATE(&dc
->sb
) == BDEV_STATE_NONE
||
1178 BDEV_STATE(&dc
->sb
) == BDEV_STATE_STALE
)
1179 bch_cached_dev_run(dc
);
1183 pr_notice("error opening %s: %s", bdevname(bdev
, name
), err
);
1184 bcache_device_stop(&dc
->disk
);
1187 /* Flash only volumes */
1189 void bch_flash_dev_release(struct kobject
*kobj
)
1191 struct bcache_device
*d
= container_of(kobj
, struct bcache_device
,
1196 static void flash_dev_free(struct closure
*cl
)
1198 struct bcache_device
*d
= container_of(cl
, struct bcache_device
, cl
);
1199 mutex_lock(&bch_register_lock
);
1200 bcache_device_free(d
);
1201 mutex_unlock(&bch_register_lock
);
1202 kobject_put(&d
->kobj
);
1205 static void flash_dev_flush(struct closure
*cl
)
1207 struct bcache_device
*d
= container_of(cl
, struct bcache_device
, cl
);
1209 mutex_lock(&bch_register_lock
);
1210 bcache_device_unlink(d
);
1211 mutex_unlock(&bch_register_lock
);
1212 kobject_del(&d
->kobj
);
1213 continue_at(cl
, flash_dev_free
, system_wq
);
1216 static int flash_dev_run(struct cache_set
*c
, struct uuid_entry
*u
)
1218 struct bcache_device
*d
= kzalloc(sizeof(struct bcache_device
),
1223 closure_init(&d
->cl
, NULL
);
1224 set_closure_fn(&d
->cl
, flash_dev_flush
, system_wq
);
1226 kobject_init(&d
->kobj
, &bch_flash_dev_ktype
);
1228 if (bcache_device_init(d
, block_bytes(c
), u
->sectors
))
1231 bcache_device_attach(d
, c
, u
- c
->uuids
);
1232 bch_flash_dev_request_init(d
);
1235 if (kobject_add(&d
->kobj
, &disk_to_dev(d
->disk
)->kobj
, "bcache"))
1238 bcache_device_link(d
, c
, "volume");
1242 kobject_put(&d
->kobj
);
1246 static int flash_devs_run(struct cache_set
*c
)
1249 struct uuid_entry
*u
;
1252 u
< c
->uuids
+ c
->nr_uuids
&& !ret
;
1254 if (UUID_FLASH_ONLY(u
))
1255 ret
= flash_dev_run(c
, u
);
1260 int bch_flash_dev_create(struct cache_set
*c
, uint64_t size
)
1262 struct uuid_entry
*u
;
1264 if (test_bit(CACHE_SET_STOPPING
, &c
->flags
))
1267 if (!test_bit(CACHE_SET_RUNNING
, &c
->flags
))
1270 u
= uuid_find_empty(c
);
1272 pr_err("Can't create volume, no room for UUID");
1276 get_random_bytes(u
->uuid
, 16);
1277 memset(u
->label
, 0, 32);
1278 u
->first_reg
= u
->last_reg
= cpu_to_le32(get_seconds());
1280 SET_UUID_FLASH_ONLY(u
, 1);
1281 u
->sectors
= size
>> 9;
1285 return flash_dev_run(c
, u
);
1291 bool bch_cache_set_error(struct cache_set
*c
, const char *fmt
, ...)
1295 if (c
->on_error
!= ON_ERROR_PANIC
&&
1296 test_bit(CACHE_SET_STOPPING
, &c
->flags
))
1299 /* XXX: we can be called from atomic context
1300 acquire_console_sem();
1303 printk(KERN_ERR
"bcache: error on %pU: ", c
->sb
.set_uuid
);
1305 va_start(args
, fmt
);
1309 printk(", disabling caching\n");
1311 if (c
->on_error
== ON_ERROR_PANIC
)
1312 panic("panic forced after error\n");
1314 bch_cache_set_unregister(c
);
1318 void bch_cache_set_release(struct kobject
*kobj
)
1320 struct cache_set
*c
= container_of(kobj
, struct cache_set
, kobj
);
1322 module_put(THIS_MODULE
);
1325 static void cache_set_free(struct closure
*cl
)
1327 struct cache_set
*c
= container_of(cl
, struct cache_set
, cl
);
1331 if (!IS_ERR_OR_NULL(c
->debug
))
1332 debugfs_remove(c
->debug
);
1334 bch_open_buckets_free(c
);
1335 bch_btree_cache_free(c
);
1336 bch_journal_free(c
);
1338 for_each_cache(ca
, c
, i
)
1341 c
->cache
[ca
->sb
.nr_this_dev
] = NULL
;
1342 kobject_put(&ca
->kobj
);
1345 bch_bset_sort_state_free(&c
->sort
);
1346 free_pages((unsigned long) c
->uuids
, ilog2(bucket_pages(c
)));
1348 if (c
->moving_gc_wq
)
1349 destroy_workqueue(c
->moving_gc_wq
);
1351 bioset_free(c
->bio_split
);
1353 mempool_destroy(c
->fill_iter
);
1355 mempool_destroy(c
->bio_meta
);
1357 mempool_destroy(c
->search
);
1360 mutex_lock(&bch_register_lock
);
1362 mutex_unlock(&bch_register_lock
);
1364 pr_info("Cache set %pU unregistered", c
->sb
.set_uuid
);
1365 wake_up(&unregister_wait
);
1367 closure_debug_destroy(&c
->cl
);
1368 kobject_put(&c
->kobj
);
1371 static void cache_set_flush(struct closure
*cl
)
1373 struct cache_set
*c
= container_of(cl
, struct cache_set
, caching
);
1381 bch_cache_accounting_destroy(&c
->accounting
);
1383 kobject_put(&c
->internal
);
1384 kobject_del(&c
->kobj
);
1387 kthread_stop(c
->gc_thread
);
1389 if (!IS_ERR_OR_NULL(c
->root
))
1390 list_add(&c
->root
->list
, &c
->btree_cache
);
1392 /* Should skip this if we're unregistering because of an error */
1393 list_for_each_entry(b
, &c
->btree_cache
, list
) {
1394 mutex_lock(&b
->write_lock
);
1395 if (btree_node_dirty(b
))
1396 __bch_btree_node_write(b
, NULL
);
1397 mutex_unlock(&b
->write_lock
);
1400 for_each_cache(ca
, c
, i
)
1401 if (ca
->alloc_thread
)
1402 kthread_stop(ca
->alloc_thread
);
1404 if (c
->journal
.cur
) {
1405 cancel_delayed_work_sync(&c
->journal
.work
);
1406 /* flush last journal entry if needed */
1407 c
->journal
.work
.work
.func(&c
->journal
.work
.work
);
1413 static void __cache_set_unregister(struct closure
*cl
)
1415 struct cache_set
*c
= container_of(cl
, struct cache_set
, caching
);
1416 struct cached_dev
*dc
;
1419 mutex_lock(&bch_register_lock
);
1421 for (i
= 0; i
< c
->nr_uuids
; i
++)
1422 if (c
->devices
[i
]) {
1423 if (!UUID_FLASH_ONLY(&c
->uuids
[i
]) &&
1424 test_bit(CACHE_SET_UNREGISTERING
, &c
->flags
)) {
1425 dc
= container_of(c
->devices
[i
],
1426 struct cached_dev
, disk
);
1427 bch_cached_dev_detach(dc
);
1429 bcache_device_stop(c
->devices
[i
]);
1433 mutex_unlock(&bch_register_lock
);
1435 continue_at(cl
, cache_set_flush
, system_wq
);
1438 void bch_cache_set_stop(struct cache_set
*c
)
1440 if (!test_and_set_bit(CACHE_SET_STOPPING
, &c
->flags
))
1441 closure_queue(&c
->caching
);
1444 void bch_cache_set_unregister(struct cache_set
*c
)
1446 set_bit(CACHE_SET_UNREGISTERING
, &c
->flags
);
1447 bch_cache_set_stop(c
);
1450 #define alloc_bucket_pages(gfp, c) \
1451 ((void *) __get_free_pages(__GFP_ZERO|gfp, ilog2(bucket_pages(c))))
1453 struct cache_set
*bch_cache_set_alloc(struct cache_sb
*sb
)
1456 struct cache_set
*c
= kzalloc(sizeof(struct cache_set
), GFP_KERNEL
);
1460 __module_get(THIS_MODULE
);
1461 closure_init(&c
->cl
, NULL
);
1462 set_closure_fn(&c
->cl
, cache_set_free
, system_wq
);
1464 closure_init(&c
->caching
, &c
->cl
);
1465 set_closure_fn(&c
->caching
, __cache_set_unregister
, system_wq
);
1467 /* Maybe create continue_at_noreturn() and use it here? */
1468 closure_set_stopped(&c
->cl
);
1469 closure_put(&c
->cl
);
1471 kobject_init(&c
->kobj
, &bch_cache_set_ktype
);
1472 kobject_init(&c
->internal
, &bch_cache_set_internal_ktype
);
1474 bch_cache_accounting_init(&c
->accounting
, &c
->cl
);
1476 memcpy(c
->sb
.set_uuid
, sb
->set_uuid
, 16);
1477 c
->sb
.block_size
= sb
->block_size
;
1478 c
->sb
.bucket_size
= sb
->bucket_size
;
1479 c
->sb
.nr_in_set
= sb
->nr_in_set
;
1480 c
->sb
.last_mount
= sb
->last_mount
;
1481 c
->bucket_bits
= ilog2(sb
->bucket_size
);
1482 c
->block_bits
= ilog2(sb
->block_size
);
1483 c
->nr_uuids
= bucket_bytes(c
) / sizeof(struct uuid_entry
);
1485 c
->btree_pages
= bucket_pages(c
);
1486 if (c
->btree_pages
> BTREE_MAX_PAGES
)
1487 c
->btree_pages
= max_t(int, c
->btree_pages
/ 4,
1490 sema_init(&c
->sb_write_mutex
, 1);
1491 mutex_init(&c
->bucket_lock
);
1492 init_waitqueue_head(&c
->btree_cache_wait
);
1493 init_waitqueue_head(&c
->bucket_wait
);
1494 sema_init(&c
->uuid_write_mutex
, 1);
1496 spin_lock_init(&c
->btree_gc_time
.lock
);
1497 spin_lock_init(&c
->btree_split_time
.lock
);
1498 spin_lock_init(&c
->btree_read_time
.lock
);
1500 bch_moving_init_cache_set(c
);
1502 INIT_LIST_HEAD(&c
->list
);
1503 INIT_LIST_HEAD(&c
->cached_devs
);
1504 INIT_LIST_HEAD(&c
->btree_cache
);
1505 INIT_LIST_HEAD(&c
->btree_cache_freeable
);
1506 INIT_LIST_HEAD(&c
->btree_cache_freed
);
1507 INIT_LIST_HEAD(&c
->data_buckets
);
1509 c
->search
= mempool_create_slab_pool(32, bch_search_cache
);
1513 iter_size
= (sb
->bucket_size
/ sb
->block_size
+ 1) *
1514 sizeof(struct btree_iter_set
);
1516 if (!(c
->devices
= kzalloc(c
->nr_uuids
* sizeof(void *), GFP_KERNEL
)) ||
1517 !(c
->bio_meta
= mempool_create_kmalloc_pool(2,
1518 sizeof(struct bbio
) + sizeof(struct bio_vec
) *
1519 bucket_pages(c
))) ||
1520 !(c
->fill_iter
= mempool_create_kmalloc_pool(1, iter_size
)) ||
1521 !(c
->bio_split
= bioset_create(4, offsetof(struct bbio
, bio
))) ||
1522 !(c
->uuids
= alloc_bucket_pages(GFP_KERNEL
, c
)) ||
1523 !(c
->moving_gc_wq
= alloc_workqueue("bcache_gc",
1524 WQ_MEM_RECLAIM
, 0)) ||
1525 bch_journal_alloc(c
) ||
1526 bch_btree_cache_alloc(c
) ||
1527 bch_open_buckets_alloc(c
) ||
1528 bch_bset_sort_state_init(&c
->sort
, ilog2(c
->btree_pages
)))
1531 c
->congested_read_threshold_us
= 2000;
1532 c
->congested_write_threshold_us
= 20000;
1533 c
->error_limit
= 8 << IO_ERROR_SHIFT
;
1537 bch_cache_set_unregister(c
);
1541 static void run_cache_set(struct cache_set
*c
)
1543 const char *err
= "cannot allocate memory";
1544 struct cached_dev
*dc
, *t
;
1549 closure_init_stack(&cl
);
1551 for_each_cache(ca
, c
, i
)
1552 c
->nbuckets
+= ca
->sb
.nbuckets
;
1554 if (CACHE_SYNC(&c
->sb
)) {
1559 err
= "cannot allocate memory for journal";
1560 if (bch_journal_read(c
, &journal
))
1563 pr_debug("btree_journal_read() done");
1565 err
= "no journal entries found";
1566 if (list_empty(&journal
))
1569 j
= &list_entry(journal
.prev
, struct journal_replay
, list
)->j
;
1571 err
= "IO error reading priorities";
1572 for_each_cache(ca
, c
, i
)
1573 prio_read(ca
, j
->prio_bucket
[ca
->sb
.nr_this_dev
]);
1576 * If prio_read() fails it'll call cache_set_error and we'll
1577 * tear everything down right away, but if we perhaps checked
1578 * sooner we could avoid journal replay.
1583 err
= "bad btree root";
1584 if (__bch_btree_ptr_invalid(c
, k
))
1587 err
= "error reading btree root";
1588 c
->root
= bch_btree_node_get(c
, NULL
, k
, j
->btree_level
, true, NULL
);
1589 if (IS_ERR_OR_NULL(c
->root
))
1592 list_del_init(&c
->root
->list
);
1593 rw_unlock(true, c
->root
);
1595 err
= uuid_read(c
, j
, &cl
);
1599 err
= "error in recovery";
1600 if (bch_btree_check(c
))
1603 bch_journal_mark(c
, &journal
);
1604 bch_initial_gc_finish(c
);
1605 pr_debug("btree_check() done");
1608 * bcache_journal_next() can't happen sooner, or
1609 * btree_gc_finish() will give spurious errors about last_gc >
1610 * gc_gen - this is a hack but oh well.
1612 bch_journal_next(&c
->journal
);
1614 err
= "error starting allocator thread";
1615 for_each_cache(ca
, c
, i
)
1616 if (bch_cache_allocator_start(ca
))
1620 * First place it's safe to allocate: btree_check() and
1621 * btree_gc_finish() have to run before we have buckets to
1622 * allocate, and bch_bucket_alloc_set() might cause a journal
1623 * entry to be written so bcache_journal_next() has to be called
1626 * If the uuids were in the old format we have to rewrite them
1627 * before the next journal entry is written:
1629 if (j
->version
< BCACHE_JSET_VERSION_UUID
)
1632 bch_journal_replay(c
, &journal
);
1634 pr_notice("invalidating existing data");
1636 for_each_cache(ca
, c
, i
) {
1639 ca
->sb
.keys
= clamp_t(int, ca
->sb
.nbuckets
>> 7,
1640 2, SB_JOURNAL_BUCKETS
);
1642 for (j
= 0; j
< ca
->sb
.keys
; j
++)
1643 ca
->sb
.d
[j
] = ca
->sb
.first_bucket
+ j
;
1646 bch_initial_gc_finish(c
);
1648 err
= "error starting allocator thread";
1649 for_each_cache(ca
, c
, i
)
1650 if (bch_cache_allocator_start(ca
))
1653 mutex_lock(&c
->bucket_lock
);
1654 for_each_cache(ca
, c
, i
)
1656 mutex_unlock(&c
->bucket_lock
);
1658 err
= "cannot allocate new UUID bucket";
1659 if (__uuid_write(c
))
1662 err
= "cannot allocate new btree root";
1663 c
->root
= __bch_btree_node_alloc(c
, NULL
, 0, true, NULL
);
1664 if (IS_ERR_OR_NULL(c
->root
))
1667 mutex_lock(&c
->root
->write_lock
);
1668 bkey_copy_key(&c
->root
->key
, &MAX_KEY
);
1669 bch_btree_node_write(c
->root
, &cl
);
1670 mutex_unlock(&c
->root
->write_lock
);
1672 bch_btree_set_root(c
->root
);
1673 rw_unlock(true, c
->root
);
1676 * We don't want to write the first journal entry until
1677 * everything is set up - fortunately journal entries won't be
1678 * written until the SET_CACHE_SYNC() here:
1680 SET_CACHE_SYNC(&c
->sb
, true);
1682 bch_journal_next(&c
->journal
);
1683 bch_journal_meta(c
, &cl
);
1686 err
= "error starting gc thread";
1687 if (bch_gc_thread_start(c
))
1691 c
->sb
.last_mount
= get_seconds();
1692 bcache_write_super(c
);
1694 list_for_each_entry_safe(dc
, t
, &uncached_devices
, list
)
1695 bch_cached_dev_attach(dc
, c
);
1699 set_bit(CACHE_SET_RUNNING
, &c
->flags
);
1703 /* XXX: test this, it's broken */
1704 bch_cache_set_error(c
, "%s", err
);
1707 static bool can_attach_cache(struct cache
*ca
, struct cache_set
*c
)
1709 return ca
->sb
.block_size
== c
->sb
.block_size
&&
1710 ca
->sb
.bucket_size
== c
->sb
.bucket_size
&&
1711 ca
->sb
.nr_in_set
== c
->sb
.nr_in_set
;
1714 static const char *register_cache_set(struct cache
*ca
)
1717 const char *err
= "cannot allocate memory";
1718 struct cache_set
*c
;
1720 list_for_each_entry(c
, &bch_cache_sets
, list
)
1721 if (!memcmp(c
->sb
.set_uuid
, ca
->sb
.set_uuid
, 16)) {
1722 if (c
->cache
[ca
->sb
.nr_this_dev
])
1723 return "duplicate cache set member";
1725 if (!can_attach_cache(ca
, c
))
1726 return "cache sb does not match set";
1728 if (!CACHE_SYNC(&ca
->sb
))
1729 SET_CACHE_SYNC(&c
->sb
, false);
1734 c
= bch_cache_set_alloc(&ca
->sb
);
1738 err
= "error creating kobject";
1739 if (kobject_add(&c
->kobj
, bcache_kobj
, "%pU", c
->sb
.set_uuid
) ||
1740 kobject_add(&c
->internal
, &c
->kobj
, "internal"))
1743 if (bch_cache_accounting_add_kobjs(&c
->accounting
, &c
->kobj
))
1746 bch_debug_init_cache_set(c
);
1748 list_add(&c
->list
, &bch_cache_sets
);
1750 sprintf(buf
, "cache%i", ca
->sb
.nr_this_dev
);
1751 if (sysfs_create_link(&ca
->kobj
, &c
->kobj
, "set") ||
1752 sysfs_create_link(&c
->kobj
, &ca
->kobj
, buf
))
1755 if (ca
->sb
.seq
> c
->sb
.seq
) {
1756 c
->sb
.version
= ca
->sb
.version
;
1757 memcpy(c
->sb
.set_uuid
, ca
->sb
.set_uuid
, 16);
1758 c
->sb
.flags
= ca
->sb
.flags
;
1759 c
->sb
.seq
= ca
->sb
.seq
;
1760 pr_debug("set version = %llu", c
->sb
.version
);
1763 kobject_get(&ca
->kobj
);
1765 ca
->set
->cache
[ca
->sb
.nr_this_dev
] = ca
;
1766 c
->cache_by_alloc
[c
->caches_loaded
++] = ca
;
1768 if (c
->caches_loaded
== c
->sb
.nr_in_set
)
1773 bch_cache_set_unregister(c
);
1779 void bch_cache_release(struct kobject
*kobj
)
1781 struct cache
*ca
= container_of(kobj
, struct cache
, kobj
);
1785 BUG_ON(ca
->set
->cache
[ca
->sb
.nr_this_dev
] != ca
);
1786 ca
->set
->cache
[ca
->sb
.nr_this_dev
] = NULL
;
1789 free_pages((unsigned long) ca
->disk_buckets
, ilog2(bucket_pages(ca
)));
1790 kfree(ca
->prio_buckets
);
1793 free_heap(&ca
->heap
);
1794 free_fifo(&ca
->free_inc
);
1796 for (i
= 0; i
< RESERVE_NR
; i
++)
1797 free_fifo(&ca
->free
[i
]);
1799 if (ca
->sb_bio
.bi_inline_vecs
[0].bv_page
)
1800 put_page(ca
->sb_bio
.bi_io_vec
[0].bv_page
);
1802 if (!IS_ERR_OR_NULL(ca
->bdev
))
1803 blkdev_put(ca
->bdev
, FMODE_READ
|FMODE_WRITE
|FMODE_EXCL
);
1806 module_put(THIS_MODULE
);
1809 static int cache_alloc(struct cache
*ca
)
1814 __module_get(THIS_MODULE
);
1815 kobject_init(&ca
->kobj
, &bch_cache_ktype
);
1817 bio_init(&ca
->journal
.bio
);
1818 ca
->journal
.bio
.bi_max_vecs
= 8;
1819 ca
->journal
.bio
.bi_io_vec
= ca
->journal
.bio
.bi_inline_vecs
;
1821 free
= roundup_pow_of_two(ca
->sb
.nbuckets
) >> 10;
1823 if (!init_fifo(&ca
->free
[RESERVE_BTREE
], 8, GFP_KERNEL
) ||
1824 !init_fifo_exact(&ca
->free
[RESERVE_PRIO
], prio_buckets(ca
), GFP_KERNEL
) ||
1825 !init_fifo(&ca
->free
[RESERVE_MOVINGGC
], free
, GFP_KERNEL
) ||
1826 !init_fifo(&ca
->free
[RESERVE_NONE
], free
, GFP_KERNEL
) ||
1827 !init_fifo(&ca
->free_inc
, free
<< 2, GFP_KERNEL
) ||
1828 !init_heap(&ca
->heap
, free
<< 3, GFP_KERNEL
) ||
1829 !(ca
->buckets
= vzalloc(sizeof(struct bucket
) *
1830 ca
->sb
.nbuckets
)) ||
1831 !(ca
->prio_buckets
= kzalloc(sizeof(uint64_t) * prio_buckets(ca
) *
1833 !(ca
->disk_buckets
= alloc_bucket_pages(GFP_KERNEL
, ca
)))
1836 ca
->prio_last_buckets
= ca
->prio_buckets
+ prio_buckets(ca
);
1838 for_each_bucket(b
, ca
)
1839 atomic_set(&b
->pin
, 0);
1844 static int register_cache(struct cache_sb
*sb
, struct page
*sb_page
,
1845 struct block_device
*bdev
, struct cache
*ca
)
1847 char name
[BDEVNAME_SIZE
];
1848 const char *err
= NULL
; /* must be set for any error case */
1851 memcpy(&ca
->sb
, sb
, sizeof(struct cache_sb
));
1853 ca
->bdev
->bd_holder
= ca
;
1855 bio_init(&ca
->sb_bio
);
1856 ca
->sb_bio
.bi_max_vecs
= 1;
1857 ca
->sb_bio
.bi_io_vec
= ca
->sb_bio
.bi_inline_vecs
;
1858 ca
->sb_bio
.bi_io_vec
[0].bv_page
= sb_page
;
1861 if (blk_queue_discard(bdev_get_queue(ca
->bdev
)))
1862 ca
->discard
= CACHE_DISCARD(&ca
->sb
);
1864 ret
= cache_alloc(ca
);
1867 err
= "cache_alloc(): -ENOMEM";
1869 err
= "cache_alloc(): unknown error";
1873 if (kobject_add(&ca
->kobj
, &part_to_dev(bdev
->bd_part
)->kobj
, "bcache")) {
1874 err
= "error calling kobject_add";
1879 mutex_lock(&bch_register_lock
);
1880 err
= register_cache_set(ca
);
1881 mutex_unlock(&bch_register_lock
);
1888 pr_info("registered cache device %s", bdevname(bdev
, name
));
1891 kobject_put(&ca
->kobj
);
1895 pr_notice("error opening %s: %s", bdevname(bdev
, name
), err
);
1900 /* Global interfaces/init */
1902 static ssize_t
register_bcache(struct kobject
*, struct kobj_attribute
*,
1903 const char *, size_t);
1905 kobj_attribute_write(register, register_bcache
);
1906 kobj_attribute_write(register_quiet
, register_bcache
);
1908 static bool bch_is_open_backing(struct block_device
*bdev
) {
1909 struct cache_set
*c
, *tc
;
1910 struct cached_dev
*dc
, *t
;
1912 list_for_each_entry_safe(c
, tc
, &bch_cache_sets
, list
)
1913 list_for_each_entry_safe(dc
, t
, &c
->cached_devs
, list
)
1914 if (dc
->bdev
== bdev
)
1916 list_for_each_entry_safe(dc
, t
, &uncached_devices
, list
)
1917 if (dc
->bdev
== bdev
)
1922 static bool bch_is_open_cache(struct block_device
*bdev
) {
1923 struct cache_set
*c
, *tc
;
1927 list_for_each_entry_safe(c
, tc
, &bch_cache_sets
, list
)
1928 for_each_cache(ca
, c
, i
)
1929 if (ca
->bdev
== bdev
)
1934 static bool bch_is_open(struct block_device
*bdev
) {
1935 return bch_is_open_cache(bdev
) || bch_is_open_backing(bdev
);
1938 static ssize_t
register_bcache(struct kobject
*k
, struct kobj_attribute
*attr
,
1939 const char *buffer
, size_t size
)
1942 const char *err
= "cannot allocate memory";
1944 struct cache_sb
*sb
= NULL
;
1945 struct block_device
*bdev
= NULL
;
1946 struct page
*sb_page
= NULL
;
1948 if (!try_module_get(THIS_MODULE
))
1951 if (!(path
= kstrndup(buffer
, size
, GFP_KERNEL
)) ||
1952 !(sb
= kmalloc(sizeof(struct cache_sb
), GFP_KERNEL
)))
1955 err
= "failed to open device";
1956 bdev
= blkdev_get_by_path(strim(path
),
1957 FMODE_READ
|FMODE_WRITE
|FMODE_EXCL
,
1960 if (bdev
== ERR_PTR(-EBUSY
)) {
1961 bdev
= lookup_bdev(strim(path
));
1962 mutex_lock(&bch_register_lock
);
1963 if (!IS_ERR(bdev
) && bch_is_open(bdev
))
1964 err
= "device already registered";
1966 err
= "device busy";
1967 mutex_unlock(&bch_register_lock
);
1968 if (attr
== &ksysfs_register_quiet
)
1974 err
= "failed to set blocksize";
1975 if (set_blocksize(bdev
, 4096))
1978 err
= read_super(sb
, bdev
, &sb_page
);
1982 if (SB_IS_BDEV(sb
)) {
1983 struct cached_dev
*dc
= kzalloc(sizeof(*dc
), GFP_KERNEL
);
1987 mutex_lock(&bch_register_lock
);
1988 register_bdev(sb
, sb_page
, bdev
, dc
);
1989 mutex_unlock(&bch_register_lock
);
1991 struct cache
*ca
= kzalloc(sizeof(*ca
), GFP_KERNEL
);
1995 if (register_cache(sb
, sb_page
, bdev
, ca
) != 0)
2003 module_put(THIS_MODULE
);
2007 blkdev_put(bdev
, FMODE_READ
|FMODE_WRITE
|FMODE_EXCL
);
2009 pr_info("error opening %s: %s", path
, err
);
2014 static int bcache_reboot(struct notifier_block
*n
, unsigned long code
, void *x
)
2016 if (code
== SYS_DOWN
||
2018 code
== SYS_POWER_OFF
) {
2020 unsigned long start
= jiffies
;
2021 bool stopped
= false;
2023 struct cache_set
*c
, *tc
;
2024 struct cached_dev
*dc
, *tdc
;
2026 mutex_lock(&bch_register_lock
);
2028 if (list_empty(&bch_cache_sets
) &&
2029 list_empty(&uncached_devices
))
2032 pr_info("Stopping all devices:");
2034 list_for_each_entry_safe(c
, tc
, &bch_cache_sets
, list
)
2035 bch_cache_set_stop(c
);
2037 list_for_each_entry_safe(dc
, tdc
, &uncached_devices
, list
)
2038 bcache_device_stop(&dc
->disk
);
2040 /* What's a condition variable? */
2042 long timeout
= start
+ 2 * HZ
- jiffies
;
2044 stopped
= list_empty(&bch_cache_sets
) &&
2045 list_empty(&uncached_devices
);
2047 if (timeout
< 0 || stopped
)
2050 prepare_to_wait(&unregister_wait
, &wait
,
2051 TASK_UNINTERRUPTIBLE
);
2053 mutex_unlock(&bch_register_lock
);
2054 schedule_timeout(timeout
);
2055 mutex_lock(&bch_register_lock
);
2058 finish_wait(&unregister_wait
, &wait
);
2061 pr_info("All devices stopped");
2063 pr_notice("Timeout waiting for devices to be closed");
2065 mutex_unlock(&bch_register_lock
);
2071 static struct notifier_block reboot
= {
2072 .notifier_call
= bcache_reboot
,
2073 .priority
= INT_MAX
, /* before any real devices */
2076 static void bcache_exit(void)
2081 kobject_put(bcache_kobj
);
2083 destroy_workqueue(bcache_wq
);
2085 unregister_blkdev(bcache_major
, "bcache");
2086 unregister_reboot_notifier(&reboot
);
2089 static int __init
bcache_init(void)
2091 static const struct attribute
*files
[] = {
2092 &ksysfs_register
.attr
,
2093 &ksysfs_register_quiet
.attr
,
2097 mutex_init(&bch_register_lock
);
2098 init_waitqueue_head(&unregister_wait
);
2099 register_reboot_notifier(&reboot
);
2100 closure_debug_init();
2102 bcache_major
= register_blkdev(0, "bcache");
2103 if (bcache_major
< 0) {
2104 unregister_reboot_notifier(&reboot
);
2105 return bcache_major
;
2108 if (!(bcache_wq
= alloc_workqueue("bcache", WQ_MEM_RECLAIM
, 0)) ||
2109 !(bcache_kobj
= kobject_create_and_add("bcache", fs_kobj
)) ||
2110 sysfs_create_files(bcache_kobj
, files
) ||
2111 bch_request_init() ||
2112 bch_debug_init(bcache_kobj
))
2121 module_exit(bcache_exit
);
2122 module_init(bcache_init
);