2 * Copyright (C) 2011-2012 Red Hat UK.
4 * This file is released under the GPL.
7 #include "dm-thin-metadata.h"
8 #include "dm-bio-prison-v1.h"
11 #include <linux/device-mapper.h>
12 #include <linux/dm-io.h>
13 #include <linux/dm-kcopyd.h>
14 #include <linux/jiffies.h>
15 #include <linux/log2.h>
16 #include <linux/list.h>
17 #include <linux/rculist.h>
18 #include <linux/init.h>
19 #include <linux/module.h>
20 #include <linux/slab.h>
21 #include <linux/vmalloc.h>
22 #include <linux/sort.h>
23 #include <linux/rbtree.h>
25 #define DM_MSG_PREFIX "thin"
30 #define ENDIO_HOOK_POOL_SIZE 1024
31 #define MAPPING_POOL_SIZE 1024
32 #define COMMIT_PERIOD HZ
33 #define NO_SPACE_TIMEOUT_SECS 60
35 static unsigned no_space_timeout_secs
= NO_SPACE_TIMEOUT_SECS
;
37 DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle
,
38 "A percentage of time allocated for copy on write");
41 * The block size of the device holding pool data must be
42 * between 64KB and 1GB.
44 #define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (64 * 1024 >> SECTOR_SHIFT)
45 #define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT)
48 * Device id is restricted to 24 bits.
50 #define MAX_DEV_ID ((1 << 24) - 1)
53 * How do we handle breaking sharing of data blocks?
54 * =================================================
56 * We use a standard copy-on-write btree to store the mappings for the
57 * devices (note I'm talking about copy-on-write of the metadata here, not
58 * the data). When you take an internal snapshot you clone the root node
59 * of the origin btree. After this there is no concept of an origin or a
60 * snapshot. They are just two device trees that happen to point to the
63 * When we get a write in we decide if it's to a shared data block using
64 * some timestamp magic. If it is, we have to break sharing.
66 * Let's say we write to a shared block in what was the origin. The
69 * i) plug io further to this physical block. (see bio_prison code).
71 * ii) quiesce any read io to that shared data block. Obviously
72 * including all devices that share this block. (see dm_deferred_set code)
74 * iii) copy the data block to a newly allocate block. This step can be
75 * missed out if the io covers the block. (schedule_copy).
77 * iv) insert the new mapping into the origin's btree
78 * (process_prepared_mapping). This act of inserting breaks some
79 * sharing of btree nodes between the two devices. Breaking sharing only
80 * effects the btree of that specific device. Btrees for the other
81 * devices that share the block never change. The btree for the origin
82 * device as it was after the last commit is untouched, ie. we're using
83 * persistent data structures in the functional programming sense.
85 * v) unplug io to this physical block, including the io that triggered
86 * the breaking of sharing.
88 * Steps (ii) and (iii) occur in parallel.
90 * The metadata _doesn't_ need to be committed before the io continues. We
91 * get away with this because the io is always written to a _new_ block.
92 * If there's a crash, then:
94 * - The origin mapping will point to the old origin block (the shared
95 * one). This will contain the data as it was before the io that triggered
96 * the breaking of sharing came in.
98 * - The snap mapping still points to the old block. As it would after
101 * The downside of this scheme is the timestamp magic isn't perfect, and
102 * will continue to think that data block in the snapshot device is shared
103 * even after the write to the origin has broken sharing. I suspect data
104 * blocks will typically be shared by many different devices, so we're
105 * breaking sharing n + 1 times, rather than n, where n is the number of
106 * devices that reference this data block. At the moment I think the
107 * benefits far, far outweigh the disadvantages.
110 /*----------------------------------------------------------------*/
120 static void build_key(struct dm_thin_device
*td
, enum lock_space ls
,
121 dm_block_t b
, dm_block_t e
, struct dm_cell_key
*key
)
123 key
->virtual = (ls
== VIRTUAL
);
124 key
->dev
= dm_thin_dev_id(td
);
125 key
->block_begin
= b
;
129 static void build_data_key(struct dm_thin_device
*td
, dm_block_t b
,
130 struct dm_cell_key
*key
)
132 build_key(td
, PHYSICAL
, b
, b
+ 1llu, key
);
135 static void build_virtual_key(struct dm_thin_device
*td
, dm_block_t b
,
136 struct dm_cell_key
*key
)
138 build_key(td
, VIRTUAL
, b
, b
+ 1llu, key
);
141 /*----------------------------------------------------------------*/
143 #define THROTTLE_THRESHOLD (1 * HZ)
146 struct rw_semaphore lock
;
147 unsigned long threshold
;
148 bool throttle_applied
;
151 static void throttle_init(struct throttle
*t
)
153 init_rwsem(&t
->lock
);
154 t
->throttle_applied
= false;
157 static void throttle_work_start(struct throttle
*t
)
159 t
->threshold
= jiffies
+ THROTTLE_THRESHOLD
;
162 static void throttle_work_update(struct throttle
*t
)
164 if (!t
->throttle_applied
&& jiffies
> t
->threshold
) {
165 down_write(&t
->lock
);
166 t
->throttle_applied
= true;
170 static void throttle_work_complete(struct throttle
*t
)
172 if (t
->throttle_applied
) {
173 t
->throttle_applied
= false;
178 static void throttle_lock(struct throttle
*t
)
183 static void throttle_unlock(struct throttle
*t
)
188 /*----------------------------------------------------------------*/
191 * A pool device ties together a metadata device and a data device. It
192 * also provides the interface for creating and destroying internal
195 struct dm_thin_new_mapping
;
198 * The pool runs in various modes. Ordered in degraded order for comparisons.
201 PM_WRITE
, /* metadata may be changed */
202 PM_OUT_OF_DATA_SPACE
, /* metadata may be changed, though data may not be allocated */
205 * Like READ_ONLY, except may switch back to WRITE on metadata resize. Reported as READ_ONLY.
207 PM_OUT_OF_METADATA_SPACE
,
208 PM_READ_ONLY
, /* metadata may not be changed */
210 PM_FAIL
, /* all I/O fails */
213 struct pool_features
{
216 bool zero_new_blocks
:1;
217 bool discard_enabled
:1;
218 bool discard_passdown
:1;
219 bool error_if_no_space
:1;
223 typedef void (*process_bio_fn
)(struct thin_c
*tc
, struct bio
*bio
);
224 typedef void (*process_cell_fn
)(struct thin_c
*tc
, struct dm_bio_prison_cell
*cell
);
225 typedef void (*process_mapping_fn
)(struct dm_thin_new_mapping
*m
);
227 #define CELL_SORT_ARRAY_SIZE 8192
230 struct list_head list
;
231 struct dm_target
*ti
; /* Only set if a pool target is bound */
233 struct mapped_device
*pool_md
;
234 struct block_device
*md_dev
;
235 struct dm_pool_metadata
*pmd
;
237 dm_block_t low_water_blocks
;
238 uint32_t sectors_per_block
;
239 int sectors_per_block_shift
;
241 struct pool_features pf
;
242 bool low_water_triggered
:1; /* A dm event has been sent */
244 bool out_of_data_space
:1;
246 struct dm_bio_prison
*prison
;
247 struct dm_kcopyd_client
*copier
;
249 struct workqueue_struct
*wq
;
250 struct throttle throttle
;
251 struct work_struct worker
;
252 struct delayed_work waker
;
253 struct delayed_work no_space_timeout
;
255 unsigned long last_commit_jiffies
;
259 struct bio_list deferred_flush_bios
;
260 struct list_head prepared_mappings
;
261 struct list_head prepared_discards
;
262 struct list_head prepared_discards_pt2
;
263 struct list_head active_thins
;
265 struct dm_deferred_set
*shared_read_ds
;
266 struct dm_deferred_set
*all_io_ds
;
268 struct dm_thin_new_mapping
*next_mapping
;
269 mempool_t
*mapping_pool
;
271 process_bio_fn process_bio
;
272 process_bio_fn process_discard
;
274 process_cell_fn process_cell
;
275 process_cell_fn process_discard_cell
;
277 process_mapping_fn process_prepared_mapping
;
278 process_mapping_fn process_prepared_discard
;
279 process_mapping_fn process_prepared_discard_pt2
;
281 struct dm_bio_prison_cell
**cell_sort_array
;
284 static void metadata_operation_failed(struct pool
*pool
, const char *op
, int r
);
286 static enum pool_mode
get_pool_mode(struct pool
*pool
)
288 return pool
->pf
.mode
;
291 static void notify_of_pool_mode_change(struct pool
*pool
)
293 const char *descs
[] = {
300 const char *extra_desc
= NULL
;
301 enum pool_mode mode
= get_pool_mode(pool
);
303 if (mode
== PM_OUT_OF_DATA_SPACE
) {
304 if (!pool
->pf
.error_if_no_space
)
305 extra_desc
= " (queue IO)";
307 extra_desc
= " (error IO)";
310 dm_table_event(pool
->ti
->table
);
311 DMINFO("%s: switching pool to %s%s mode",
312 dm_device_name(pool
->pool_md
),
313 descs
[(int)mode
], extra_desc
? : "");
317 * Target context for a pool.
320 struct dm_target
*ti
;
322 struct dm_dev
*data_dev
;
323 struct dm_dev
*metadata_dev
;
324 struct dm_target_callbacks callbacks
;
326 dm_block_t low_water_blocks
;
327 struct pool_features requested_pf
; /* Features requested during table load */
328 struct pool_features adjusted_pf
; /* Features used after adjusting for constituent devices */
332 * Target context for a thin.
335 struct list_head list
;
336 struct dm_dev
*pool_dev
;
337 struct dm_dev
*origin_dev
;
338 sector_t origin_size
;
342 struct dm_thin_device
*td
;
343 struct mapped_device
*thin_md
;
347 struct list_head deferred_cells
;
348 struct bio_list deferred_bio_list
;
349 struct bio_list retry_on_resume_list
;
350 struct rb_root sort_bio_list
; /* sorted list of deferred bios */
353 * Ensures the thin is not destroyed until the worker has finished
354 * iterating the active_thins list.
357 struct completion can_destroy
;
360 /*----------------------------------------------------------------*/
362 static bool block_size_is_power_of_two(struct pool
*pool
)
364 return pool
->sectors_per_block_shift
>= 0;
367 static sector_t
block_to_sectors(struct pool
*pool
, dm_block_t b
)
369 return block_size_is_power_of_two(pool
) ?
370 (b
<< pool
->sectors_per_block_shift
) :
371 (b
* pool
->sectors_per_block
);
374 /*----------------------------------------------------------------*/
378 struct blk_plug plug
;
379 struct bio
*parent_bio
;
383 static void begin_discard(struct discard_op
*op
, struct thin_c
*tc
, struct bio
*parent
)
388 blk_start_plug(&op
->plug
);
389 op
->parent_bio
= parent
;
393 static int issue_discard(struct discard_op
*op
, dm_block_t data_b
, dm_block_t data_e
)
395 struct thin_c
*tc
= op
->tc
;
396 sector_t s
= block_to_sectors(tc
->pool
, data_b
);
397 sector_t len
= block_to_sectors(tc
->pool
, data_e
- data_b
);
399 return __blkdev_issue_discard(tc
->pool_dev
->bdev
, s
, len
,
400 GFP_NOWAIT
, 0, &op
->bio
);
403 static void end_discard(struct discard_op
*op
, int r
)
407 * Even if one of the calls to issue_discard failed, we
408 * need to wait for the chain to complete.
410 bio_chain(op
->bio
, op
->parent_bio
);
411 bio_set_op_attrs(op
->bio
, REQ_OP_DISCARD
, 0);
415 blk_finish_plug(&op
->plug
);
418 * Even if r is set, there could be sub discards in flight that we
421 if (r
&& !op
->parent_bio
->bi_status
)
422 op
->parent_bio
->bi_status
= errno_to_blk_status(r
);
423 bio_endio(op
->parent_bio
);
426 /*----------------------------------------------------------------*/
429 * wake_worker() is used when new work is queued and when pool_resume is
430 * ready to continue deferred IO processing.
432 static void wake_worker(struct pool
*pool
)
434 queue_work(pool
->wq
, &pool
->worker
);
437 /*----------------------------------------------------------------*/
439 static int bio_detain(struct pool
*pool
, struct dm_cell_key
*key
, struct bio
*bio
,
440 struct dm_bio_prison_cell
**cell_result
)
443 struct dm_bio_prison_cell
*cell_prealloc
;
446 * Allocate a cell from the prison's mempool.
447 * This might block but it can't fail.
449 cell_prealloc
= dm_bio_prison_alloc_cell(pool
->prison
, GFP_NOIO
);
451 r
= dm_bio_detain(pool
->prison
, key
, bio
, cell_prealloc
, cell_result
);
454 * We reused an old cell; we can get rid of
457 dm_bio_prison_free_cell(pool
->prison
, cell_prealloc
);
462 static void cell_release(struct pool
*pool
,
463 struct dm_bio_prison_cell
*cell
,
464 struct bio_list
*bios
)
466 dm_cell_release(pool
->prison
, cell
, bios
);
467 dm_bio_prison_free_cell(pool
->prison
, cell
);
470 static void cell_visit_release(struct pool
*pool
,
471 void (*fn
)(void *, struct dm_bio_prison_cell
*),
473 struct dm_bio_prison_cell
*cell
)
475 dm_cell_visit_release(pool
->prison
, fn
, context
, cell
);
476 dm_bio_prison_free_cell(pool
->prison
, cell
);
479 static void cell_release_no_holder(struct pool
*pool
,
480 struct dm_bio_prison_cell
*cell
,
481 struct bio_list
*bios
)
483 dm_cell_release_no_holder(pool
->prison
, cell
, bios
);
484 dm_bio_prison_free_cell(pool
->prison
, cell
);
487 static void cell_error_with_code(struct pool
*pool
,
488 struct dm_bio_prison_cell
*cell
, blk_status_t error_code
)
490 dm_cell_error(pool
->prison
, cell
, error_code
);
491 dm_bio_prison_free_cell(pool
->prison
, cell
);
494 static blk_status_t
get_pool_io_error_code(struct pool
*pool
)
496 return pool
->out_of_data_space
? BLK_STS_NOSPC
: BLK_STS_IOERR
;
499 static void cell_error(struct pool
*pool
, struct dm_bio_prison_cell
*cell
)
501 cell_error_with_code(pool
, cell
, get_pool_io_error_code(pool
));
504 static void cell_success(struct pool
*pool
, struct dm_bio_prison_cell
*cell
)
506 cell_error_with_code(pool
, cell
, 0);
509 static void cell_requeue(struct pool
*pool
, struct dm_bio_prison_cell
*cell
)
511 cell_error_with_code(pool
, cell
, BLK_STS_DM_REQUEUE
);
514 /*----------------------------------------------------------------*/
517 * A global list of pools that uses a struct mapped_device as a key.
519 static struct dm_thin_pool_table
{
521 struct list_head pools
;
522 } dm_thin_pool_table
;
524 static void pool_table_init(void)
526 mutex_init(&dm_thin_pool_table
.mutex
);
527 INIT_LIST_HEAD(&dm_thin_pool_table
.pools
);
530 static void __pool_table_insert(struct pool
*pool
)
532 BUG_ON(!mutex_is_locked(&dm_thin_pool_table
.mutex
));
533 list_add(&pool
->list
, &dm_thin_pool_table
.pools
);
536 static void __pool_table_remove(struct pool
*pool
)
538 BUG_ON(!mutex_is_locked(&dm_thin_pool_table
.mutex
));
539 list_del(&pool
->list
);
542 static struct pool
*__pool_table_lookup(struct mapped_device
*md
)
544 struct pool
*pool
= NULL
, *tmp
;
546 BUG_ON(!mutex_is_locked(&dm_thin_pool_table
.mutex
));
548 list_for_each_entry(tmp
, &dm_thin_pool_table
.pools
, list
) {
549 if (tmp
->pool_md
== md
) {
558 static struct pool
*__pool_table_lookup_metadata_dev(struct block_device
*md_dev
)
560 struct pool
*pool
= NULL
, *tmp
;
562 BUG_ON(!mutex_is_locked(&dm_thin_pool_table
.mutex
));
564 list_for_each_entry(tmp
, &dm_thin_pool_table
.pools
, list
) {
565 if (tmp
->md_dev
== md_dev
) {
574 /*----------------------------------------------------------------*/
576 struct dm_thin_endio_hook
{
578 struct dm_deferred_entry
*shared_read_entry
;
579 struct dm_deferred_entry
*all_io_entry
;
580 struct dm_thin_new_mapping
*overwrite_mapping
;
581 struct rb_node rb_node
;
582 struct dm_bio_prison_cell
*cell
;
585 static void __merge_bio_list(struct bio_list
*bios
, struct bio_list
*master
)
587 bio_list_merge(bios
, master
);
588 bio_list_init(master
);
591 static void error_bio_list(struct bio_list
*bios
, blk_status_t error
)
595 while ((bio
= bio_list_pop(bios
))) {
596 bio
->bi_status
= error
;
601 static void error_thin_bio_list(struct thin_c
*tc
, struct bio_list
*master
,
604 struct bio_list bios
;
607 bio_list_init(&bios
);
609 spin_lock_irqsave(&tc
->lock
, flags
);
610 __merge_bio_list(&bios
, master
);
611 spin_unlock_irqrestore(&tc
->lock
, flags
);
613 error_bio_list(&bios
, error
);
616 static void requeue_deferred_cells(struct thin_c
*tc
)
618 struct pool
*pool
= tc
->pool
;
620 struct list_head cells
;
621 struct dm_bio_prison_cell
*cell
, *tmp
;
623 INIT_LIST_HEAD(&cells
);
625 spin_lock_irqsave(&tc
->lock
, flags
);
626 list_splice_init(&tc
->deferred_cells
, &cells
);
627 spin_unlock_irqrestore(&tc
->lock
, flags
);
629 list_for_each_entry_safe(cell
, tmp
, &cells
, user_list
)
630 cell_requeue(pool
, cell
);
633 static void requeue_io(struct thin_c
*tc
)
635 struct bio_list bios
;
638 bio_list_init(&bios
);
640 spin_lock_irqsave(&tc
->lock
, flags
);
641 __merge_bio_list(&bios
, &tc
->deferred_bio_list
);
642 __merge_bio_list(&bios
, &tc
->retry_on_resume_list
);
643 spin_unlock_irqrestore(&tc
->lock
, flags
);
645 error_bio_list(&bios
, BLK_STS_DM_REQUEUE
);
646 requeue_deferred_cells(tc
);
649 static void error_retry_list_with_code(struct pool
*pool
, blk_status_t error
)
654 list_for_each_entry_rcu(tc
, &pool
->active_thins
, list
)
655 error_thin_bio_list(tc
, &tc
->retry_on_resume_list
, error
);
659 static void error_retry_list(struct pool
*pool
)
661 error_retry_list_with_code(pool
, get_pool_io_error_code(pool
));
665 * This section of code contains the logic for processing a thin device's IO.
666 * Much of the code depends on pool object resources (lists, workqueues, etc)
667 * but most is exclusively called from the thin target rather than the thin-pool
671 static dm_block_t
get_bio_block(struct thin_c
*tc
, struct bio
*bio
)
673 struct pool
*pool
= tc
->pool
;
674 sector_t block_nr
= bio
->bi_iter
.bi_sector
;
676 if (block_size_is_power_of_two(pool
))
677 block_nr
>>= pool
->sectors_per_block_shift
;
679 (void) sector_div(block_nr
, pool
->sectors_per_block
);
685 * Returns the _complete_ blocks that this bio covers.
687 static void get_bio_block_range(struct thin_c
*tc
, struct bio
*bio
,
688 dm_block_t
*begin
, dm_block_t
*end
)
690 struct pool
*pool
= tc
->pool
;
691 sector_t b
= bio
->bi_iter
.bi_sector
;
692 sector_t e
= b
+ (bio
->bi_iter
.bi_size
>> SECTOR_SHIFT
);
694 b
+= pool
->sectors_per_block
- 1ull; /* so we round up */
696 if (block_size_is_power_of_two(pool
)) {
697 b
>>= pool
->sectors_per_block_shift
;
698 e
>>= pool
->sectors_per_block_shift
;
700 (void) sector_div(b
, pool
->sectors_per_block
);
701 (void) sector_div(e
, pool
->sectors_per_block
);
705 /* Can happen if the bio is within a single block. */
712 static void remap(struct thin_c
*tc
, struct bio
*bio
, dm_block_t block
)
714 struct pool
*pool
= tc
->pool
;
715 sector_t bi_sector
= bio
->bi_iter
.bi_sector
;
717 bio_set_dev(bio
, tc
->pool_dev
->bdev
);
718 if (block_size_is_power_of_two(pool
))
719 bio
->bi_iter
.bi_sector
=
720 (block
<< pool
->sectors_per_block_shift
) |
721 (bi_sector
& (pool
->sectors_per_block
- 1));
723 bio
->bi_iter
.bi_sector
= (block
* pool
->sectors_per_block
) +
724 sector_div(bi_sector
, pool
->sectors_per_block
);
727 static void remap_to_origin(struct thin_c
*tc
, struct bio
*bio
)
729 bio_set_dev(bio
, tc
->origin_dev
->bdev
);
732 static int bio_triggers_commit(struct thin_c
*tc
, struct bio
*bio
)
734 return op_is_flush(bio
->bi_opf
) &&
735 dm_thin_changed_this_transaction(tc
->td
);
738 static void inc_all_io_entry(struct pool
*pool
, struct bio
*bio
)
740 struct dm_thin_endio_hook
*h
;
742 if (bio_op(bio
) == REQ_OP_DISCARD
)
745 h
= dm_per_bio_data(bio
, sizeof(struct dm_thin_endio_hook
));
746 h
->all_io_entry
= dm_deferred_entry_inc(pool
->all_io_ds
);
749 static void issue(struct thin_c
*tc
, struct bio
*bio
)
751 struct pool
*pool
= tc
->pool
;
754 if (!bio_triggers_commit(tc
, bio
)) {
755 generic_make_request(bio
);
760 * Complete bio with an error if earlier I/O caused changes to
761 * the metadata that can't be committed e.g, due to I/O errors
762 * on the metadata device.
764 if (dm_thin_aborted_changes(tc
->td
)) {
770 * Batch together any bios that trigger commits and then issue a
771 * single commit for them in process_deferred_bios().
773 spin_lock_irqsave(&pool
->lock
, flags
);
774 bio_list_add(&pool
->deferred_flush_bios
, bio
);
775 spin_unlock_irqrestore(&pool
->lock
, flags
);
778 static void remap_to_origin_and_issue(struct thin_c
*tc
, struct bio
*bio
)
780 remap_to_origin(tc
, bio
);
784 static void remap_and_issue(struct thin_c
*tc
, struct bio
*bio
,
787 remap(tc
, bio
, block
);
791 /*----------------------------------------------------------------*/
794 * Bio endio functions.
796 struct dm_thin_new_mapping
{
797 struct list_head list
;
803 * Track quiescing, copying and zeroing preparation actions. When this
804 * counter hits zero the block is prepared and can be inserted into the
807 atomic_t prepare_actions
;
811 dm_block_t virt_begin
, virt_end
;
812 dm_block_t data_block
;
813 struct dm_bio_prison_cell
*cell
;
816 * If the bio covers the whole area of a block then we can avoid
817 * zeroing or copying. Instead this bio is hooked. The bio will
818 * still be in the cell, so care has to be taken to avoid issuing
822 bio_end_io_t
*saved_bi_end_io
;
825 static void __complete_mapping_preparation(struct dm_thin_new_mapping
*m
)
827 struct pool
*pool
= m
->tc
->pool
;
829 if (atomic_dec_and_test(&m
->prepare_actions
)) {
830 list_add_tail(&m
->list
, &pool
->prepared_mappings
);
835 static void complete_mapping_preparation(struct dm_thin_new_mapping
*m
)
838 struct pool
*pool
= m
->tc
->pool
;
840 spin_lock_irqsave(&pool
->lock
, flags
);
841 __complete_mapping_preparation(m
);
842 spin_unlock_irqrestore(&pool
->lock
, flags
);
845 static void copy_complete(int read_err
, unsigned long write_err
, void *context
)
847 struct dm_thin_new_mapping
*m
= context
;
849 m
->status
= read_err
|| write_err
? BLK_STS_IOERR
: 0;
850 complete_mapping_preparation(m
);
853 static void overwrite_endio(struct bio
*bio
)
855 struct dm_thin_endio_hook
*h
= dm_per_bio_data(bio
, sizeof(struct dm_thin_endio_hook
));
856 struct dm_thin_new_mapping
*m
= h
->overwrite_mapping
;
858 bio
->bi_end_io
= m
->saved_bi_end_io
;
860 m
->status
= bio
->bi_status
;
861 complete_mapping_preparation(m
);
864 /*----------------------------------------------------------------*/
871 * Prepared mapping jobs.
875 * This sends the bios in the cell, except the original holder, back
876 * to the deferred_bios list.
878 static void cell_defer_no_holder(struct thin_c
*tc
, struct dm_bio_prison_cell
*cell
)
880 struct pool
*pool
= tc
->pool
;
883 spin_lock_irqsave(&tc
->lock
, flags
);
884 cell_release_no_holder(pool
, cell
, &tc
->deferred_bio_list
);
885 spin_unlock_irqrestore(&tc
->lock
, flags
);
890 static void thin_defer_bio(struct thin_c
*tc
, struct bio
*bio
);
894 struct bio_list defer_bios
;
895 struct bio_list issue_bios
;
898 static void __inc_remap_and_issue_cell(void *context
,
899 struct dm_bio_prison_cell
*cell
)
901 struct remap_info
*info
= context
;
904 while ((bio
= bio_list_pop(&cell
->bios
))) {
905 if (op_is_flush(bio
->bi_opf
) || bio_op(bio
) == REQ_OP_DISCARD
)
906 bio_list_add(&info
->defer_bios
, bio
);
908 inc_all_io_entry(info
->tc
->pool
, bio
);
911 * We can't issue the bios with the bio prison lock
912 * held, so we add them to a list to issue on
913 * return from this function.
915 bio_list_add(&info
->issue_bios
, bio
);
920 static void inc_remap_and_issue_cell(struct thin_c
*tc
,
921 struct dm_bio_prison_cell
*cell
,
925 struct remap_info info
;
928 bio_list_init(&info
.defer_bios
);
929 bio_list_init(&info
.issue_bios
);
932 * We have to be careful to inc any bios we're about to issue
933 * before the cell is released, and avoid a race with new bios
934 * being added to the cell.
936 cell_visit_release(tc
->pool
, __inc_remap_and_issue_cell
,
939 while ((bio
= bio_list_pop(&info
.defer_bios
)))
940 thin_defer_bio(tc
, bio
);
942 while ((bio
= bio_list_pop(&info
.issue_bios
)))
943 remap_and_issue(info
.tc
, bio
, block
);
946 static void process_prepared_mapping_fail(struct dm_thin_new_mapping
*m
)
948 cell_error(m
->tc
->pool
, m
->cell
);
950 mempool_free(m
, m
->tc
->pool
->mapping_pool
);
953 static void process_prepared_mapping(struct dm_thin_new_mapping
*m
)
955 struct thin_c
*tc
= m
->tc
;
956 struct pool
*pool
= tc
->pool
;
957 struct bio
*bio
= m
->bio
;
961 cell_error(pool
, m
->cell
);
966 * Commit the prepared block into the mapping btree.
967 * Any I/O for this block arriving after this point will get
968 * remapped to it directly.
970 r
= dm_thin_insert_block(tc
->td
, m
->virt_begin
, m
->data_block
);
972 metadata_operation_failed(pool
, "dm_thin_insert_block", r
);
973 cell_error(pool
, m
->cell
);
978 * Release any bios held while the block was being provisioned.
979 * If we are processing a write bio that completely covers the block,
980 * we already processed it so can ignore it now when processing
981 * the bios in the cell.
984 inc_remap_and_issue_cell(tc
, m
->cell
, m
->data_block
);
987 inc_all_io_entry(tc
->pool
, m
->cell
->holder
);
988 remap_and_issue(tc
, m
->cell
->holder
, m
->data_block
);
989 inc_remap_and_issue_cell(tc
, m
->cell
, m
->data_block
);
994 mempool_free(m
, pool
->mapping_pool
);
997 /*----------------------------------------------------------------*/
999 static void free_discard_mapping(struct dm_thin_new_mapping
*m
)
1001 struct thin_c
*tc
= m
->tc
;
1003 cell_defer_no_holder(tc
, m
->cell
);
1004 mempool_free(m
, tc
->pool
->mapping_pool
);
1007 static void process_prepared_discard_fail(struct dm_thin_new_mapping
*m
)
1009 bio_io_error(m
->bio
);
1010 free_discard_mapping(m
);
1013 static void process_prepared_discard_success(struct dm_thin_new_mapping
*m
)
1016 free_discard_mapping(m
);
1019 static void process_prepared_discard_no_passdown(struct dm_thin_new_mapping
*m
)
1022 struct thin_c
*tc
= m
->tc
;
1024 r
= dm_thin_remove_range(tc
->td
, m
->cell
->key
.block_begin
, m
->cell
->key
.block_end
);
1026 metadata_operation_failed(tc
->pool
, "dm_thin_remove_range", r
);
1027 bio_io_error(m
->bio
);
1031 cell_defer_no_holder(tc
, m
->cell
);
1032 mempool_free(m
, tc
->pool
->mapping_pool
);
1035 /*----------------------------------------------------------------*/
1037 static void passdown_double_checking_shared_status(struct dm_thin_new_mapping
*m
,
1038 struct bio
*discard_parent
)
1041 * We've already unmapped this range of blocks, but before we
1042 * passdown we have to check that these blocks are now unused.
1046 struct thin_c
*tc
= m
->tc
;
1047 struct pool
*pool
= tc
->pool
;
1048 dm_block_t b
= m
->data_block
, e
, end
= m
->data_block
+ m
->virt_end
- m
->virt_begin
;
1049 struct discard_op op
;
1051 begin_discard(&op
, tc
, discard_parent
);
1053 /* find start of unmapped run */
1054 for (; b
< end
; b
++) {
1055 r
= dm_pool_block_is_shared(pool
->pmd
, b
, &shared
);
1066 /* find end of run */
1067 for (e
= b
+ 1; e
!= end
; e
++) {
1068 r
= dm_pool_block_is_shared(pool
->pmd
, e
, &shared
);
1076 r
= issue_discard(&op
, b
, e
);
1083 end_discard(&op
, r
);
1086 static void queue_passdown_pt2(struct dm_thin_new_mapping
*m
)
1088 unsigned long flags
;
1089 struct pool
*pool
= m
->tc
->pool
;
1091 spin_lock_irqsave(&pool
->lock
, flags
);
1092 list_add_tail(&m
->list
, &pool
->prepared_discards_pt2
);
1093 spin_unlock_irqrestore(&pool
->lock
, flags
);
1097 static void passdown_endio(struct bio
*bio
)
1100 * It doesn't matter if the passdown discard failed, we still want
1101 * to unmap (we ignore err).
1103 queue_passdown_pt2(bio
->bi_private
);
1107 static void process_prepared_discard_passdown_pt1(struct dm_thin_new_mapping
*m
)
1110 struct thin_c
*tc
= m
->tc
;
1111 struct pool
*pool
= tc
->pool
;
1112 struct bio
*discard_parent
;
1113 dm_block_t data_end
= m
->data_block
+ (m
->virt_end
- m
->virt_begin
);
1116 * Only this thread allocates blocks, so we can be sure that the
1117 * newly unmapped blocks will not be allocated before the end of
1120 r
= dm_thin_remove_range(tc
->td
, m
->virt_begin
, m
->virt_end
);
1122 metadata_operation_failed(pool
, "dm_thin_remove_range", r
);
1123 bio_io_error(m
->bio
);
1124 cell_defer_no_holder(tc
, m
->cell
);
1125 mempool_free(m
, pool
->mapping_pool
);
1130 * Increment the unmapped blocks. This prevents a race between the
1131 * passdown io and reallocation of freed blocks.
1133 r
= dm_pool_inc_data_range(pool
->pmd
, m
->data_block
, data_end
);
1135 metadata_operation_failed(pool
, "dm_pool_inc_data_range", r
);
1136 bio_io_error(m
->bio
);
1137 cell_defer_no_holder(tc
, m
->cell
);
1138 mempool_free(m
, pool
->mapping_pool
);
1142 discard_parent
= bio_alloc(GFP_NOIO
, 1);
1143 if (!discard_parent
) {
1144 DMWARN("%s: unable to allocate top level discard bio for passdown. Skipping passdown.",
1145 dm_device_name(tc
->pool
->pool_md
));
1146 queue_passdown_pt2(m
);
1149 discard_parent
->bi_end_io
= passdown_endio
;
1150 discard_parent
->bi_private
= m
;
1152 if (m
->maybe_shared
)
1153 passdown_double_checking_shared_status(m
, discard_parent
);
1155 struct discard_op op
;
1157 begin_discard(&op
, tc
, discard_parent
);
1158 r
= issue_discard(&op
, m
->data_block
, data_end
);
1159 end_discard(&op
, r
);
1164 static void process_prepared_discard_passdown_pt2(struct dm_thin_new_mapping
*m
)
1167 struct thin_c
*tc
= m
->tc
;
1168 struct pool
*pool
= tc
->pool
;
1171 * The passdown has completed, so now we can decrement all those
1174 r
= dm_pool_dec_data_range(pool
->pmd
, m
->data_block
,
1175 m
->data_block
+ (m
->virt_end
- m
->virt_begin
));
1177 metadata_operation_failed(pool
, "dm_pool_dec_data_range", r
);
1178 bio_io_error(m
->bio
);
1182 cell_defer_no_holder(tc
, m
->cell
);
1183 mempool_free(m
, pool
->mapping_pool
);
1186 static void process_prepared(struct pool
*pool
, struct list_head
*head
,
1187 process_mapping_fn
*fn
)
1189 unsigned long flags
;
1190 struct list_head maps
;
1191 struct dm_thin_new_mapping
*m
, *tmp
;
1193 INIT_LIST_HEAD(&maps
);
1194 spin_lock_irqsave(&pool
->lock
, flags
);
1195 list_splice_init(head
, &maps
);
1196 spin_unlock_irqrestore(&pool
->lock
, flags
);
1198 list_for_each_entry_safe(m
, tmp
, &maps
, list
)
1203 * Deferred bio jobs.
1205 static int io_overlaps_block(struct pool
*pool
, struct bio
*bio
)
1207 return bio
->bi_iter
.bi_size
==
1208 (pool
->sectors_per_block
<< SECTOR_SHIFT
);
1211 static int io_overwrites_block(struct pool
*pool
, struct bio
*bio
)
1213 return (bio_data_dir(bio
) == WRITE
) &&
1214 io_overlaps_block(pool
, bio
);
1217 static void save_and_set_endio(struct bio
*bio
, bio_end_io_t
**save
,
1220 *save
= bio
->bi_end_io
;
1221 bio
->bi_end_io
= fn
;
1224 static int ensure_next_mapping(struct pool
*pool
)
1226 if (pool
->next_mapping
)
1229 pool
->next_mapping
= mempool_alloc(pool
->mapping_pool
, GFP_ATOMIC
);
1231 return pool
->next_mapping
? 0 : -ENOMEM
;
1234 static struct dm_thin_new_mapping
*get_next_mapping(struct pool
*pool
)
1236 struct dm_thin_new_mapping
*m
= pool
->next_mapping
;
1238 BUG_ON(!pool
->next_mapping
);
1240 memset(m
, 0, sizeof(struct dm_thin_new_mapping
));
1241 INIT_LIST_HEAD(&m
->list
);
1244 pool
->next_mapping
= NULL
;
1249 static void ll_zero(struct thin_c
*tc
, struct dm_thin_new_mapping
*m
,
1250 sector_t begin
, sector_t end
)
1253 struct dm_io_region to
;
1255 to
.bdev
= tc
->pool_dev
->bdev
;
1257 to
.count
= end
- begin
;
1259 r
= dm_kcopyd_zero(tc
->pool
->copier
, 1, &to
, 0, copy_complete
, m
);
1261 DMERR_LIMIT("dm_kcopyd_zero() failed");
1262 copy_complete(1, 1, m
);
1266 static void remap_and_issue_overwrite(struct thin_c
*tc
, struct bio
*bio
,
1267 dm_block_t data_begin
,
1268 struct dm_thin_new_mapping
*m
)
1270 struct pool
*pool
= tc
->pool
;
1271 struct dm_thin_endio_hook
*h
= dm_per_bio_data(bio
, sizeof(struct dm_thin_endio_hook
));
1273 h
->overwrite_mapping
= m
;
1275 save_and_set_endio(bio
, &m
->saved_bi_end_io
, overwrite_endio
);
1276 inc_all_io_entry(pool
, bio
);
1277 remap_and_issue(tc
, bio
, data_begin
);
1281 * A partial copy also needs to zero the uncopied region.
1283 static void schedule_copy(struct thin_c
*tc
, dm_block_t virt_block
,
1284 struct dm_dev
*origin
, dm_block_t data_origin
,
1285 dm_block_t data_dest
,
1286 struct dm_bio_prison_cell
*cell
, struct bio
*bio
,
1290 struct pool
*pool
= tc
->pool
;
1291 struct dm_thin_new_mapping
*m
= get_next_mapping(pool
);
1294 m
->virt_begin
= virt_block
;
1295 m
->virt_end
= virt_block
+ 1u;
1296 m
->data_block
= data_dest
;
1300 * quiesce action + copy action + an extra reference held for the
1301 * duration of this function (we may need to inc later for a
1304 atomic_set(&m
->prepare_actions
, 3);
1306 if (!dm_deferred_set_add_work(pool
->shared_read_ds
, &m
->list
))
1307 complete_mapping_preparation(m
); /* already quiesced */
1310 * IO to pool_dev remaps to the pool target's data_dev.
1312 * If the whole block of data is being overwritten, we can issue the
1313 * bio immediately. Otherwise we use kcopyd to clone the data first.
1315 if (io_overwrites_block(pool
, bio
))
1316 remap_and_issue_overwrite(tc
, bio
, data_dest
, m
);
1318 struct dm_io_region from
, to
;
1320 from
.bdev
= origin
->bdev
;
1321 from
.sector
= data_origin
* pool
->sectors_per_block
;
1324 to
.bdev
= tc
->pool_dev
->bdev
;
1325 to
.sector
= data_dest
* pool
->sectors_per_block
;
1328 r
= dm_kcopyd_copy(pool
->copier
, &from
, 1, &to
,
1329 0, copy_complete
, m
);
1331 DMERR_LIMIT("dm_kcopyd_copy() failed");
1332 copy_complete(1, 1, m
);
1335 * We allow the zero to be issued, to simplify the
1336 * error path. Otherwise we'd need to start
1337 * worrying about decrementing the prepare_actions
1343 * Do we need to zero a tail region?
1345 if (len
< pool
->sectors_per_block
&& pool
->pf
.zero_new_blocks
) {
1346 atomic_inc(&m
->prepare_actions
);
1348 data_dest
* pool
->sectors_per_block
+ len
,
1349 (data_dest
+ 1) * pool
->sectors_per_block
);
1353 complete_mapping_preparation(m
); /* drop our ref */
1356 static void schedule_internal_copy(struct thin_c
*tc
, dm_block_t virt_block
,
1357 dm_block_t data_origin
, dm_block_t data_dest
,
1358 struct dm_bio_prison_cell
*cell
, struct bio
*bio
)
1360 schedule_copy(tc
, virt_block
, tc
->pool_dev
,
1361 data_origin
, data_dest
, cell
, bio
,
1362 tc
->pool
->sectors_per_block
);
1365 static void schedule_zero(struct thin_c
*tc
, dm_block_t virt_block
,
1366 dm_block_t data_block
, struct dm_bio_prison_cell
*cell
,
1369 struct pool
*pool
= tc
->pool
;
1370 struct dm_thin_new_mapping
*m
= get_next_mapping(pool
);
1372 atomic_set(&m
->prepare_actions
, 1); /* no need to quiesce */
1374 m
->virt_begin
= virt_block
;
1375 m
->virt_end
= virt_block
+ 1u;
1376 m
->data_block
= data_block
;
1380 * If the whole block of data is being overwritten or we are not
1381 * zeroing pre-existing data, we can issue the bio immediately.
1382 * Otherwise we use kcopyd to zero the data first.
1384 if (pool
->pf
.zero_new_blocks
) {
1385 if (io_overwrites_block(pool
, bio
))
1386 remap_and_issue_overwrite(tc
, bio
, data_block
, m
);
1388 ll_zero(tc
, m
, data_block
* pool
->sectors_per_block
,
1389 (data_block
+ 1) * pool
->sectors_per_block
);
1391 process_prepared_mapping(m
);
1394 static void schedule_external_copy(struct thin_c
*tc
, dm_block_t virt_block
,
1395 dm_block_t data_dest
,
1396 struct dm_bio_prison_cell
*cell
, struct bio
*bio
)
1398 struct pool
*pool
= tc
->pool
;
1399 sector_t virt_block_begin
= virt_block
* pool
->sectors_per_block
;
1400 sector_t virt_block_end
= (virt_block
+ 1) * pool
->sectors_per_block
;
1402 if (virt_block_end
<= tc
->origin_size
)
1403 schedule_copy(tc
, virt_block
, tc
->origin_dev
,
1404 virt_block
, data_dest
, cell
, bio
,
1405 pool
->sectors_per_block
);
1407 else if (virt_block_begin
< tc
->origin_size
)
1408 schedule_copy(tc
, virt_block
, tc
->origin_dev
,
1409 virt_block
, data_dest
, cell
, bio
,
1410 tc
->origin_size
- virt_block_begin
);
1413 schedule_zero(tc
, virt_block
, data_dest
, cell
, bio
);
1416 static void set_pool_mode(struct pool
*pool
, enum pool_mode new_mode
);
1418 static void requeue_bios(struct pool
*pool
);
1420 static bool is_read_only_pool_mode(enum pool_mode mode
)
1422 return (mode
== PM_OUT_OF_METADATA_SPACE
|| mode
== PM_READ_ONLY
);
1425 static bool is_read_only(struct pool
*pool
)
1427 return is_read_only_pool_mode(get_pool_mode(pool
));
1430 static void check_for_metadata_space(struct pool
*pool
)
1433 const char *ooms_reason
= NULL
;
1436 r
= dm_pool_get_free_metadata_block_count(pool
->pmd
, &nr_free
);
1438 ooms_reason
= "Could not get free metadata blocks";
1440 ooms_reason
= "No free metadata blocks";
1442 if (ooms_reason
&& !is_read_only(pool
)) {
1443 DMERR("%s", ooms_reason
);
1444 set_pool_mode(pool
, PM_OUT_OF_METADATA_SPACE
);
1448 static void check_for_data_space(struct pool
*pool
)
1453 if (get_pool_mode(pool
) != PM_OUT_OF_DATA_SPACE
)
1456 r
= dm_pool_get_free_block_count(pool
->pmd
, &nr_free
);
1461 set_pool_mode(pool
, PM_WRITE
);
1467 * A non-zero return indicates read_only or fail_io mode.
1468 * Many callers don't care about the return value.
1470 static int commit(struct pool
*pool
)
1474 if (get_pool_mode(pool
) >= PM_OUT_OF_METADATA_SPACE
)
1477 r
= dm_pool_commit_metadata(pool
->pmd
);
1479 metadata_operation_failed(pool
, "dm_pool_commit_metadata", r
);
1481 check_for_metadata_space(pool
);
1482 check_for_data_space(pool
);
1488 static void check_low_water_mark(struct pool
*pool
, dm_block_t free_blocks
)
1490 unsigned long flags
;
1492 if (free_blocks
<= pool
->low_water_blocks
&& !pool
->low_water_triggered
) {
1493 DMWARN("%s: reached low water mark for data device: sending event.",
1494 dm_device_name(pool
->pool_md
));
1495 spin_lock_irqsave(&pool
->lock
, flags
);
1496 pool
->low_water_triggered
= true;
1497 spin_unlock_irqrestore(&pool
->lock
, flags
);
1498 dm_table_event(pool
->ti
->table
);
1502 static int alloc_data_block(struct thin_c
*tc
, dm_block_t
*result
)
1505 dm_block_t free_blocks
;
1506 struct pool
*pool
= tc
->pool
;
1508 if (WARN_ON(get_pool_mode(pool
) != PM_WRITE
))
1511 r
= dm_pool_get_free_block_count(pool
->pmd
, &free_blocks
);
1513 metadata_operation_failed(pool
, "dm_pool_get_free_block_count", r
);
1517 check_low_water_mark(pool
, free_blocks
);
1521 * Try to commit to see if that will free up some
1528 r
= dm_pool_get_free_block_count(pool
->pmd
, &free_blocks
);
1530 metadata_operation_failed(pool
, "dm_pool_get_free_block_count", r
);
1535 set_pool_mode(pool
, PM_OUT_OF_DATA_SPACE
);
1540 r
= dm_pool_alloc_data_block(pool
->pmd
, result
);
1543 set_pool_mode(pool
, PM_OUT_OF_DATA_SPACE
);
1545 metadata_operation_failed(pool
, "dm_pool_alloc_data_block", r
);
1549 r
= dm_pool_get_free_metadata_block_count(pool
->pmd
, &free_blocks
);
1551 metadata_operation_failed(pool
, "dm_pool_get_free_metadata_block_count", r
);
1556 /* Let's commit before we use up the metadata reserve. */
1566 * If we have run out of space, queue bios until the device is
1567 * resumed, presumably after having been reloaded with more space.
1569 static void retry_on_resume(struct bio
*bio
)
1571 struct dm_thin_endio_hook
*h
= dm_per_bio_data(bio
, sizeof(struct dm_thin_endio_hook
));
1572 struct thin_c
*tc
= h
->tc
;
1573 unsigned long flags
;
1575 spin_lock_irqsave(&tc
->lock
, flags
);
1576 bio_list_add(&tc
->retry_on_resume_list
, bio
);
1577 spin_unlock_irqrestore(&tc
->lock
, flags
);
1580 static blk_status_t
should_error_unserviceable_bio(struct pool
*pool
)
1582 enum pool_mode m
= get_pool_mode(pool
);
1586 /* Shouldn't get here */
1587 DMERR_LIMIT("bio unserviceable, yet pool is in PM_WRITE mode");
1588 return BLK_STS_IOERR
;
1590 case PM_OUT_OF_DATA_SPACE
:
1591 return pool
->pf
.error_if_no_space
? BLK_STS_NOSPC
: 0;
1593 case PM_OUT_OF_METADATA_SPACE
:
1596 return BLK_STS_IOERR
;
1598 /* Shouldn't get here */
1599 DMERR_LIMIT("bio unserviceable, yet pool has an unknown mode");
1600 return BLK_STS_IOERR
;
1604 static void handle_unserviceable_bio(struct pool
*pool
, struct bio
*bio
)
1606 blk_status_t error
= should_error_unserviceable_bio(pool
);
1609 bio
->bi_status
= error
;
1612 retry_on_resume(bio
);
1615 static void retry_bios_on_resume(struct pool
*pool
, struct dm_bio_prison_cell
*cell
)
1618 struct bio_list bios
;
1621 error
= should_error_unserviceable_bio(pool
);
1623 cell_error_with_code(pool
, cell
, error
);
1627 bio_list_init(&bios
);
1628 cell_release(pool
, cell
, &bios
);
1630 while ((bio
= bio_list_pop(&bios
)))
1631 retry_on_resume(bio
);
1634 static void process_discard_cell_no_passdown(struct thin_c
*tc
,
1635 struct dm_bio_prison_cell
*virt_cell
)
1637 struct pool
*pool
= tc
->pool
;
1638 struct dm_thin_new_mapping
*m
= get_next_mapping(pool
);
1641 * We don't need to lock the data blocks, since there's no
1642 * passdown. We only lock data blocks for allocation and breaking sharing.
1645 m
->virt_begin
= virt_cell
->key
.block_begin
;
1646 m
->virt_end
= virt_cell
->key
.block_end
;
1647 m
->cell
= virt_cell
;
1648 m
->bio
= virt_cell
->holder
;
1650 if (!dm_deferred_set_add_work(pool
->all_io_ds
, &m
->list
))
1651 pool
->process_prepared_discard(m
);
1654 static void break_up_discard_bio(struct thin_c
*tc
, dm_block_t begin
, dm_block_t end
,
1657 struct pool
*pool
= tc
->pool
;
1661 struct dm_cell_key data_key
;
1662 struct dm_bio_prison_cell
*data_cell
;
1663 struct dm_thin_new_mapping
*m
;
1664 dm_block_t virt_begin
, virt_end
, data_begin
;
1666 while (begin
!= end
) {
1667 r
= ensure_next_mapping(pool
);
1669 /* we did our best */
1672 r
= dm_thin_find_mapped_range(tc
->td
, begin
, end
, &virt_begin
, &virt_end
,
1673 &data_begin
, &maybe_shared
);
1676 * Silently fail, letting any mappings we've
1681 build_key(tc
->td
, PHYSICAL
, data_begin
, data_begin
+ (virt_end
- virt_begin
), &data_key
);
1682 if (bio_detain(tc
->pool
, &data_key
, NULL
, &data_cell
)) {
1683 /* contention, we'll give up with this range */
1689 * IO may still be going to the destination block. We must
1690 * quiesce before we can do the removal.
1692 m
= get_next_mapping(pool
);
1694 m
->maybe_shared
= maybe_shared
;
1695 m
->virt_begin
= virt_begin
;
1696 m
->virt_end
= virt_end
;
1697 m
->data_block
= data_begin
;
1698 m
->cell
= data_cell
;
1702 * The parent bio must not complete before sub discard bios are
1703 * chained to it (see end_discard's bio_chain)!
1705 * This per-mapping bi_remaining increment is paired with
1706 * the implicit decrement that occurs via bio_endio() in
1709 bio_inc_remaining(bio
);
1710 if (!dm_deferred_set_add_work(pool
->all_io_ds
, &m
->list
))
1711 pool
->process_prepared_discard(m
);
1717 static void process_discard_cell_passdown(struct thin_c
*tc
, struct dm_bio_prison_cell
*virt_cell
)
1719 struct bio
*bio
= virt_cell
->holder
;
1720 struct dm_thin_endio_hook
*h
= dm_per_bio_data(bio
, sizeof(struct dm_thin_endio_hook
));
1723 * The virt_cell will only get freed once the origin bio completes.
1724 * This means it will remain locked while all the individual
1725 * passdown bios are in flight.
1727 h
->cell
= virt_cell
;
1728 break_up_discard_bio(tc
, virt_cell
->key
.block_begin
, virt_cell
->key
.block_end
, bio
);
1731 * We complete the bio now, knowing that the bi_remaining field
1732 * will prevent completion until the sub range discards have
1738 static void process_discard_bio(struct thin_c
*tc
, struct bio
*bio
)
1740 dm_block_t begin
, end
;
1741 struct dm_cell_key virt_key
;
1742 struct dm_bio_prison_cell
*virt_cell
;
1744 get_bio_block_range(tc
, bio
, &begin
, &end
);
1747 * The discard covers less than a block.
1753 build_key(tc
->td
, VIRTUAL
, begin
, end
, &virt_key
);
1754 if (bio_detain(tc
->pool
, &virt_key
, bio
, &virt_cell
))
1756 * Potential starvation issue: We're relying on the
1757 * fs/application being well behaved, and not trying to
1758 * send IO to a region at the same time as discarding it.
1759 * If they do this persistently then it's possible this
1760 * cell will never be granted.
1764 tc
->pool
->process_discard_cell(tc
, virt_cell
);
1767 static void break_sharing(struct thin_c
*tc
, struct bio
*bio
, dm_block_t block
,
1768 struct dm_cell_key
*key
,
1769 struct dm_thin_lookup_result
*lookup_result
,
1770 struct dm_bio_prison_cell
*cell
)
1773 dm_block_t data_block
;
1774 struct pool
*pool
= tc
->pool
;
1776 r
= alloc_data_block(tc
, &data_block
);
1779 schedule_internal_copy(tc
, block
, lookup_result
->block
,
1780 data_block
, cell
, bio
);
1784 retry_bios_on_resume(pool
, cell
);
1788 DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",
1790 cell_error(pool
, cell
);
1795 static void __remap_and_issue_shared_cell(void *context
,
1796 struct dm_bio_prison_cell
*cell
)
1798 struct remap_info
*info
= context
;
1801 while ((bio
= bio_list_pop(&cell
->bios
))) {
1802 if (bio_data_dir(bio
) == WRITE
|| op_is_flush(bio
->bi_opf
) ||
1803 bio_op(bio
) == REQ_OP_DISCARD
)
1804 bio_list_add(&info
->defer_bios
, bio
);
1806 struct dm_thin_endio_hook
*h
= dm_per_bio_data(bio
, sizeof(struct dm_thin_endio_hook
));;
1808 h
->shared_read_entry
= dm_deferred_entry_inc(info
->tc
->pool
->shared_read_ds
);
1809 inc_all_io_entry(info
->tc
->pool
, bio
);
1810 bio_list_add(&info
->issue_bios
, bio
);
1815 static void remap_and_issue_shared_cell(struct thin_c
*tc
,
1816 struct dm_bio_prison_cell
*cell
,
1820 struct remap_info info
;
1823 bio_list_init(&info
.defer_bios
);
1824 bio_list_init(&info
.issue_bios
);
1826 cell_visit_release(tc
->pool
, __remap_and_issue_shared_cell
,
1829 while ((bio
= bio_list_pop(&info
.defer_bios
)))
1830 thin_defer_bio(tc
, bio
);
1832 while ((bio
= bio_list_pop(&info
.issue_bios
)))
1833 remap_and_issue(tc
, bio
, block
);
1836 static void process_shared_bio(struct thin_c
*tc
, struct bio
*bio
,
1838 struct dm_thin_lookup_result
*lookup_result
,
1839 struct dm_bio_prison_cell
*virt_cell
)
1841 struct dm_bio_prison_cell
*data_cell
;
1842 struct pool
*pool
= tc
->pool
;
1843 struct dm_cell_key key
;
1846 * If cell is already occupied, then sharing is already in the process
1847 * of being broken so we have nothing further to do here.
1849 build_data_key(tc
->td
, lookup_result
->block
, &key
);
1850 if (bio_detain(pool
, &key
, bio
, &data_cell
)) {
1851 cell_defer_no_holder(tc
, virt_cell
);
1855 if (bio_data_dir(bio
) == WRITE
&& bio
->bi_iter
.bi_size
) {
1856 break_sharing(tc
, bio
, block
, &key
, lookup_result
, data_cell
);
1857 cell_defer_no_holder(tc
, virt_cell
);
1859 struct dm_thin_endio_hook
*h
= dm_per_bio_data(bio
, sizeof(struct dm_thin_endio_hook
));
1861 h
->shared_read_entry
= dm_deferred_entry_inc(pool
->shared_read_ds
);
1862 inc_all_io_entry(pool
, bio
);
1863 remap_and_issue(tc
, bio
, lookup_result
->block
);
1865 remap_and_issue_shared_cell(tc
, data_cell
, lookup_result
->block
);
1866 remap_and_issue_shared_cell(tc
, virt_cell
, lookup_result
->block
);
1870 static void provision_block(struct thin_c
*tc
, struct bio
*bio
, dm_block_t block
,
1871 struct dm_bio_prison_cell
*cell
)
1874 dm_block_t data_block
;
1875 struct pool
*pool
= tc
->pool
;
1878 * Remap empty bios (flushes) immediately, without provisioning.
1880 if (!bio
->bi_iter
.bi_size
) {
1881 inc_all_io_entry(pool
, bio
);
1882 cell_defer_no_holder(tc
, cell
);
1884 remap_and_issue(tc
, bio
, 0);
1889 * Fill read bios with zeroes and complete them immediately.
1891 if (bio_data_dir(bio
) == READ
) {
1893 cell_defer_no_holder(tc
, cell
);
1898 r
= alloc_data_block(tc
, &data_block
);
1902 schedule_external_copy(tc
, block
, data_block
, cell
, bio
);
1904 schedule_zero(tc
, block
, data_block
, cell
, bio
);
1908 retry_bios_on_resume(pool
, cell
);
1912 DMERR_LIMIT("%s: alloc_data_block() failed: error = %d",
1914 cell_error(pool
, cell
);
1919 static void process_cell(struct thin_c
*tc
, struct dm_bio_prison_cell
*cell
)
1922 struct pool
*pool
= tc
->pool
;
1923 struct bio
*bio
= cell
->holder
;
1924 dm_block_t block
= get_bio_block(tc
, bio
);
1925 struct dm_thin_lookup_result lookup_result
;
1927 if (tc
->requeue_mode
) {
1928 cell_requeue(pool
, cell
);
1932 r
= dm_thin_find_block(tc
->td
, block
, 1, &lookup_result
);
1935 if (lookup_result
.shared
)
1936 process_shared_bio(tc
, bio
, block
, &lookup_result
, cell
);
1938 inc_all_io_entry(pool
, bio
);
1939 remap_and_issue(tc
, bio
, lookup_result
.block
);
1940 inc_remap_and_issue_cell(tc
, cell
, lookup_result
.block
);
1945 if (bio_data_dir(bio
) == READ
&& tc
->origin_dev
) {
1946 inc_all_io_entry(pool
, bio
);
1947 cell_defer_no_holder(tc
, cell
);
1949 if (bio_end_sector(bio
) <= tc
->origin_size
)
1950 remap_to_origin_and_issue(tc
, bio
);
1952 else if (bio
->bi_iter
.bi_sector
< tc
->origin_size
) {
1954 bio
->bi_iter
.bi_size
= (tc
->origin_size
- bio
->bi_iter
.bi_sector
) << SECTOR_SHIFT
;
1955 remap_to_origin_and_issue(tc
, bio
);
1962 provision_block(tc
, bio
, block
, cell
);
1966 DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",
1968 cell_defer_no_holder(tc
, cell
);
1974 static void process_bio(struct thin_c
*tc
, struct bio
*bio
)
1976 struct pool
*pool
= tc
->pool
;
1977 dm_block_t block
= get_bio_block(tc
, bio
);
1978 struct dm_bio_prison_cell
*cell
;
1979 struct dm_cell_key key
;
1982 * If cell is already occupied, then the block is already
1983 * being provisioned so we have nothing further to do here.
1985 build_virtual_key(tc
->td
, block
, &key
);
1986 if (bio_detain(pool
, &key
, bio
, &cell
))
1989 process_cell(tc
, cell
);
1992 static void __process_bio_read_only(struct thin_c
*tc
, struct bio
*bio
,
1993 struct dm_bio_prison_cell
*cell
)
1996 int rw
= bio_data_dir(bio
);
1997 dm_block_t block
= get_bio_block(tc
, bio
);
1998 struct dm_thin_lookup_result lookup_result
;
2000 r
= dm_thin_find_block(tc
->td
, block
, 1, &lookup_result
);
2003 if (lookup_result
.shared
&& (rw
== WRITE
) && bio
->bi_iter
.bi_size
) {
2004 handle_unserviceable_bio(tc
->pool
, bio
);
2006 cell_defer_no_holder(tc
, cell
);
2008 inc_all_io_entry(tc
->pool
, bio
);
2009 remap_and_issue(tc
, bio
, lookup_result
.block
);
2011 inc_remap_and_issue_cell(tc
, cell
, lookup_result
.block
);
2017 cell_defer_no_holder(tc
, cell
);
2019 handle_unserviceable_bio(tc
->pool
, bio
);
2023 if (tc
->origin_dev
) {
2024 inc_all_io_entry(tc
->pool
, bio
);
2025 remap_to_origin_and_issue(tc
, bio
);
2034 DMERR_LIMIT("%s: dm_thin_find_block() failed: error = %d",
2037 cell_defer_no_holder(tc
, cell
);
2043 static void process_bio_read_only(struct thin_c
*tc
, struct bio
*bio
)
2045 __process_bio_read_only(tc
, bio
, NULL
);
2048 static void process_cell_read_only(struct thin_c
*tc
, struct dm_bio_prison_cell
*cell
)
2050 __process_bio_read_only(tc
, cell
->holder
, cell
);
2053 static void process_bio_success(struct thin_c
*tc
, struct bio
*bio
)
2058 static void process_bio_fail(struct thin_c
*tc
, struct bio
*bio
)
2063 static void process_cell_success(struct thin_c
*tc
, struct dm_bio_prison_cell
*cell
)
2065 cell_success(tc
->pool
, cell
);
2068 static void process_cell_fail(struct thin_c
*tc
, struct dm_bio_prison_cell
*cell
)
2070 cell_error(tc
->pool
, cell
);
2074 * FIXME: should we also commit due to size of transaction, measured in
2077 static int need_commit_due_to_time(struct pool
*pool
)
2079 return !time_in_range(jiffies
, pool
->last_commit_jiffies
,
2080 pool
->last_commit_jiffies
+ COMMIT_PERIOD
);
2083 #define thin_pbd(node) rb_entry((node), struct dm_thin_endio_hook, rb_node)
2084 #define thin_bio(pbd) dm_bio_from_per_bio_data((pbd), sizeof(struct dm_thin_endio_hook))
2086 static void __thin_bio_rb_add(struct thin_c
*tc
, struct bio
*bio
)
2088 struct rb_node
**rbp
, *parent
;
2089 struct dm_thin_endio_hook
*pbd
;
2090 sector_t bi_sector
= bio
->bi_iter
.bi_sector
;
2092 rbp
= &tc
->sort_bio_list
.rb_node
;
2096 pbd
= thin_pbd(parent
);
2098 if (bi_sector
< thin_bio(pbd
)->bi_iter
.bi_sector
)
2099 rbp
= &(*rbp
)->rb_left
;
2101 rbp
= &(*rbp
)->rb_right
;
2104 pbd
= dm_per_bio_data(bio
, sizeof(struct dm_thin_endio_hook
));
2105 rb_link_node(&pbd
->rb_node
, parent
, rbp
);
2106 rb_insert_color(&pbd
->rb_node
, &tc
->sort_bio_list
);
2109 static void __extract_sorted_bios(struct thin_c
*tc
)
2111 struct rb_node
*node
;
2112 struct dm_thin_endio_hook
*pbd
;
2115 for (node
= rb_first(&tc
->sort_bio_list
); node
; node
= rb_next(node
)) {
2116 pbd
= thin_pbd(node
);
2117 bio
= thin_bio(pbd
);
2119 bio_list_add(&tc
->deferred_bio_list
, bio
);
2120 rb_erase(&pbd
->rb_node
, &tc
->sort_bio_list
);
2123 WARN_ON(!RB_EMPTY_ROOT(&tc
->sort_bio_list
));
2126 static void __sort_thin_deferred_bios(struct thin_c
*tc
)
2129 struct bio_list bios
;
2131 bio_list_init(&bios
);
2132 bio_list_merge(&bios
, &tc
->deferred_bio_list
);
2133 bio_list_init(&tc
->deferred_bio_list
);
2135 /* Sort deferred_bio_list using rb-tree */
2136 while ((bio
= bio_list_pop(&bios
)))
2137 __thin_bio_rb_add(tc
, bio
);
2140 * Transfer the sorted bios in sort_bio_list back to
2141 * deferred_bio_list to allow lockless submission of
2144 __extract_sorted_bios(tc
);
2147 static void process_thin_deferred_bios(struct thin_c
*tc
)
2149 struct pool
*pool
= tc
->pool
;
2150 unsigned long flags
;
2152 struct bio_list bios
;
2153 struct blk_plug plug
;
2156 if (tc
->requeue_mode
) {
2157 error_thin_bio_list(tc
, &tc
->deferred_bio_list
,
2158 BLK_STS_DM_REQUEUE
);
2162 bio_list_init(&bios
);
2164 spin_lock_irqsave(&tc
->lock
, flags
);
2166 if (bio_list_empty(&tc
->deferred_bio_list
)) {
2167 spin_unlock_irqrestore(&tc
->lock
, flags
);
2171 __sort_thin_deferred_bios(tc
);
2173 bio_list_merge(&bios
, &tc
->deferred_bio_list
);
2174 bio_list_init(&tc
->deferred_bio_list
);
2176 spin_unlock_irqrestore(&tc
->lock
, flags
);
2178 blk_start_plug(&plug
);
2179 while ((bio
= bio_list_pop(&bios
))) {
2181 * If we've got no free new_mapping structs, and processing
2182 * this bio might require one, we pause until there are some
2183 * prepared mappings to process.
2185 if (ensure_next_mapping(pool
)) {
2186 spin_lock_irqsave(&tc
->lock
, flags
);
2187 bio_list_add(&tc
->deferred_bio_list
, bio
);
2188 bio_list_merge(&tc
->deferred_bio_list
, &bios
);
2189 spin_unlock_irqrestore(&tc
->lock
, flags
);
2193 if (bio_op(bio
) == REQ_OP_DISCARD
)
2194 pool
->process_discard(tc
, bio
);
2196 pool
->process_bio(tc
, bio
);
2198 if ((count
++ & 127) == 0) {
2199 throttle_work_update(&pool
->throttle
);
2200 dm_pool_issue_prefetches(pool
->pmd
);
2203 blk_finish_plug(&plug
);
2206 static int cmp_cells(const void *lhs
, const void *rhs
)
2208 struct dm_bio_prison_cell
*lhs_cell
= *((struct dm_bio_prison_cell
**) lhs
);
2209 struct dm_bio_prison_cell
*rhs_cell
= *((struct dm_bio_prison_cell
**) rhs
);
2211 BUG_ON(!lhs_cell
->holder
);
2212 BUG_ON(!rhs_cell
->holder
);
2214 if (lhs_cell
->holder
->bi_iter
.bi_sector
< rhs_cell
->holder
->bi_iter
.bi_sector
)
2217 if (lhs_cell
->holder
->bi_iter
.bi_sector
> rhs_cell
->holder
->bi_iter
.bi_sector
)
2223 static unsigned sort_cells(struct pool
*pool
, struct list_head
*cells
)
2226 struct dm_bio_prison_cell
*cell
, *tmp
;
2228 list_for_each_entry_safe(cell
, tmp
, cells
, user_list
) {
2229 if (count
>= CELL_SORT_ARRAY_SIZE
)
2232 pool
->cell_sort_array
[count
++] = cell
;
2233 list_del(&cell
->user_list
);
2236 sort(pool
->cell_sort_array
, count
, sizeof(cell
), cmp_cells
, NULL
);
2241 static void process_thin_deferred_cells(struct thin_c
*tc
)
2243 struct pool
*pool
= tc
->pool
;
2244 unsigned long flags
;
2245 struct list_head cells
;
2246 struct dm_bio_prison_cell
*cell
;
2247 unsigned i
, j
, count
;
2249 INIT_LIST_HEAD(&cells
);
2251 spin_lock_irqsave(&tc
->lock
, flags
);
2252 list_splice_init(&tc
->deferred_cells
, &cells
);
2253 spin_unlock_irqrestore(&tc
->lock
, flags
);
2255 if (list_empty(&cells
))
2259 count
= sort_cells(tc
->pool
, &cells
);
2261 for (i
= 0; i
< count
; i
++) {
2262 cell
= pool
->cell_sort_array
[i
];
2263 BUG_ON(!cell
->holder
);
2266 * If we've got no free new_mapping structs, and processing
2267 * this bio might require one, we pause until there are some
2268 * prepared mappings to process.
2270 if (ensure_next_mapping(pool
)) {
2271 for (j
= i
; j
< count
; j
++)
2272 list_add(&pool
->cell_sort_array
[j
]->user_list
, &cells
);
2274 spin_lock_irqsave(&tc
->lock
, flags
);
2275 list_splice(&cells
, &tc
->deferred_cells
);
2276 spin_unlock_irqrestore(&tc
->lock
, flags
);
2280 if (bio_op(cell
->holder
) == REQ_OP_DISCARD
)
2281 pool
->process_discard_cell(tc
, cell
);
2283 pool
->process_cell(tc
, cell
);
2285 } while (!list_empty(&cells
));
2288 static void thin_get(struct thin_c
*tc
);
2289 static void thin_put(struct thin_c
*tc
);
2292 * We can't hold rcu_read_lock() around code that can block. So we
2293 * find a thin with the rcu lock held; bump a refcount; then drop
2296 static struct thin_c
*get_first_thin(struct pool
*pool
)
2298 struct thin_c
*tc
= NULL
;
2301 if (!list_empty(&pool
->active_thins
)) {
2302 tc
= list_entry_rcu(pool
->active_thins
.next
, struct thin_c
, list
);
2310 static struct thin_c
*get_next_thin(struct pool
*pool
, struct thin_c
*tc
)
2312 struct thin_c
*old_tc
= tc
;
2315 list_for_each_entry_continue_rcu(tc
, &pool
->active_thins
, list
) {
2327 static void process_deferred_bios(struct pool
*pool
)
2329 unsigned long flags
;
2331 struct bio_list bios
;
2334 tc
= get_first_thin(pool
);
2336 process_thin_deferred_cells(tc
);
2337 process_thin_deferred_bios(tc
);
2338 tc
= get_next_thin(pool
, tc
);
2342 * If there are any deferred flush bios, we must commit
2343 * the metadata before issuing them.
2345 bio_list_init(&bios
);
2346 spin_lock_irqsave(&pool
->lock
, flags
);
2347 bio_list_merge(&bios
, &pool
->deferred_flush_bios
);
2348 bio_list_init(&pool
->deferred_flush_bios
);
2349 spin_unlock_irqrestore(&pool
->lock
, flags
);
2351 if (bio_list_empty(&bios
) &&
2352 !(dm_pool_changed_this_transaction(pool
->pmd
) && need_commit_due_to_time(pool
)))
2356 while ((bio
= bio_list_pop(&bios
)))
2360 pool
->last_commit_jiffies
= jiffies
;
2362 while ((bio
= bio_list_pop(&bios
)))
2363 generic_make_request(bio
);
2366 static void do_worker(struct work_struct
*ws
)
2368 struct pool
*pool
= container_of(ws
, struct pool
, worker
);
2370 throttle_work_start(&pool
->throttle
);
2371 dm_pool_issue_prefetches(pool
->pmd
);
2372 throttle_work_update(&pool
->throttle
);
2373 process_prepared(pool
, &pool
->prepared_mappings
, &pool
->process_prepared_mapping
);
2374 throttle_work_update(&pool
->throttle
);
2375 process_prepared(pool
, &pool
->prepared_discards
, &pool
->process_prepared_discard
);
2376 throttle_work_update(&pool
->throttle
);
2377 process_prepared(pool
, &pool
->prepared_discards_pt2
, &pool
->process_prepared_discard_pt2
);
2378 throttle_work_update(&pool
->throttle
);
2379 process_deferred_bios(pool
);
2380 throttle_work_complete(&pool
->throttle
);
2384 * We want to commit periodically so that not too much
2385 * unwritten data builds up.
2387 static void do_waker(struct work_struct
*ws
)
2389 struct pool
*pool
= container_of(to_delayed_work(ws
), struct pool
, waker
);
2391 queue_delayed_work(pool
->wq
, &pool
->waker
, COMMIT_PERIOD
);
2395 * We're holding onto IO to allow userland time to react. After the
2396 * timeout either the pool will have been resized (and thus back in
2397 * PM_WRITE mode), or we degrade to PM_OUT_OF_DATA_SPACE w/ error_if_no_space.
2399 static void do_no_space_timeout(struct work_struct
*ws
)
2401 struct pool
*pool
= container_of(to_delayed_work(ws
), struct pool
,
2404 if (get_pool_mode(pool
) == PM_OUT_OF_DATA_SPACE
&& !pool
->pf
.error_if_no_space
) {
2405 pool
->pf
.error_if_no_space
= true;
2406 notify_of_pool_mode_change(pool
);
2407 error_retry_list_with_code(pool
, BLK_STS_NOSPC
);
2411 /*----------------------------------------------------------------*/
2414 struct work_struct worker
;
2415 struct completion complete
;
2418 static struct pool_work
*to_pool_work(struct work_struct
*ws
)
2420 return container_of(ws
, struct pool_work
, worker
);
2423 static void pool_work_complete(struct pool_work
*pw
)
2425 complete(&pw
->complete
);
2428 static void pool_work_wait(struct pool_work
*pw
, struct pool
*pool
,
2429 void (*fn
)(struct work_struct
*))
2431 INIT_WORK_ONSTACK(&pw
->worker
, fn
);
2432 init_completion(&pw
->complete
);
2433 queue_work(pool
->wq
, &pw
->worker
);
2434 wait_for_completion(&pw
->complete
);
2437 /*----------------------------------------------------------------*/
2439 struct noflush_work
{
2440 struct pool_work pw
;
2444 static struct noflush_work
*to_noflush(struct work_struct
*ws
)
2446 return container_of(to_pool_work(ws
), struct noflush_work
, pw
);
2449 static void do_noflush_start(struct work_struct
*ws
)
2451 struct noflush_work
*w
= to_noflush(ws
);
2452 w
->tc
->requeue_mode
= true;
2454 pool_work_complete(&w
->pw
);
2457 static void do_noflush_stop(struct work_struct
*ws
)
2459 struct noflush_work
*w
= to_noflush(ws
);
2460 w
->tc
->requeue_mode
= false;
2461 pool_work_complete(&w
->pw
);
2464 static void noflush_work(struct thin_c
*tc
, void (*fn
)(struct work_struct
*))
2466 struct noflush_work w
;
2469 pool_work_wait(&w
.pw
, tc
->pool
, fn
);
2472 /*----------------------------------------------------------------*/
2474 static bool passdown_enabled(struct pool_c
*pt
)
2476 return pt
->adjusted_pf
.discard_passdown
;
2479 static void set_discard_callbacks(struct pool
*pool
)
2481 struct pool_c
*pt
= pool
->ti
->private;
2483 if (passdown_enabled(pt
)) {
2484 pool
->process_discard_cell
= process_discard_cell_passdown
;
2485 pool
->process_prepared_discard
= process_prepared_discard_passdown_pt1
;
2486 pool
->process_prepared_discard_pt2
= process_prepared_discard_passdown_pt2
;
2488 pool
->process_discard_cell
= process_discard_cell_no_passdown
;
2489 pool
->process_prepared_discard
= process_prepared_discard_no_passdown
;
2493 static void set_pool_mode(struct pool
*pool
, enum pool_mode new_mode
)
2495 struct pool_c
*pt
= pool
->ti
->private;
2496 bool needs_check
= dm_pool_metadata_needs_check(pool
->pmd
);
2497 enum pool_mode old_mode
= get_pool_mode(pool
);
2498 unsigned long no_space_timeout
= READ_ONCE(no_space_timeout_secs
) * HZ
;
2501 * Never allow the pool to transition to PM_WRITE mode if user
2502 * intervention is required to verify metadata and data consistency.
2504 if (new_mode
== PM_WRITE
&& needs_check
) {
2505 DMERR("%s: unable to switch pool to write mode until repaired.",
2506 dm_device_name(pool
->pool_md
));
2507 if (old_mode
!= new_mode
)
2508 new_mode
= old_mode
;
2510 new_mode
= PM_READ_ONLY
;
2513 * If we were in PM_FAIL mode, rollback of metadata failed. We're
2514 * not going to recover without a thin_repair. So we never let the
2515 * pool move out of the old mode.
2517 if (old_mode
== PM_FAIL
)
2518 new_mode
= old_mode
;
2522 dm_pool_metadata_read_only(pool
->pmd
);
2523 pool
->process_bio
= process_bio_fail
;
2524 pool
->process_discard
= process_bio_fail
;
2525 pool
->process_cell
= process_cell_fail
;
2526 pool
->process_discard_cell
= process_cell_fail
;
2527 pool
->process_prepared_mapping
= process_prepared_mapping_fail
;
2528 pool
->process_prepared_discard
= process_prepared_discard_fail
;
2530 error_retry_list(pool
);
2533 case PM_OUT_OF_METADATA_SPACE
:
2535 dm_pool_metadata_read_only(pool
->pmd
);
2536 pool
->process_bio
= process_bio_read_only
;
2537 pool
->process_discard
= process_bio_success
;
2538 pool
->process_cell
= process_cell_read_only
;
2539 pool
->process_discard_cell
= process_cell_success
;
2540 pool
->process_prepared_mapping
= process_prepared_mapping_fail
;
2541 pool
->process_prepared_discard
= process_prepared_discard_success
;
2543 error_retry_list(pool
);
2546 case PM_OUT_OF_DATA_SPACE
:
2548 * Ideally we'd never hit this state; the low water mark
2549 * would trigger userland to extend the pool before we
2550 * completely run out of data space. However, many small
2551 * IOs to unprovisioned space can consume data space at an
2552 * alarming rate. Adjust your low water mark if you're
2553 * frequently seeing this mode.
2555 pool
->out_of_data_space
= true;
2556 pool
->process_bio
= process_bio_read_only
;
2557 pool
->process_discard
= process_discard_bio
;
2558 pool
->process_cell
= process_cell_read_only
;
2559 pool
->process_prepared_mapping
= process_prepared_mapping
;
2560 set_discard_callbacks(pool
);
2562 if (!pool
->pf
.error_if_no_space
&& no_space_timeout
)
2563 queue_delayed_work(pool
->wq
, &pool
->no_space_timeout
, no_space_timeout
);
2567 if (old_mode
== PM_OUT_OF_DATA_SPACE
)
2568 cancel_delayed_work_sync(&pool
->no_space_timeout
);
2569 pool
->out_of_data_space
= false;
2570 pool
->pf
.error_if_no_space
= pt
->requested_pf
.error_if_no_space
;
2571 dm_pool_metadata_read_write(pool
->pmd
);
2572 pool
->process_bio
= process_bio
;
2573 pool
->process_discard
= process_discard_bio
;
2574 pool
->process_cell
= process_cell
;
2575 pool
->process_prepared_mapping
= process_prepared_mapping
;
2576 set_discard_callbacks(pool
);
2580 pool
->pf
.mode
= new_mode
;
2582 * The pool mode may have changed, sync it so bind_control_target()
2583 * doesn't cause an unexpected mode transition on resume.
2585 pt
->adjusted_pf
.mode
= new_mode
;
2587 if (old_mode
!= new_mode
)
2588 notify_of_pool_mode_change(pool
);
2591 static void abort_transaction(struct pool
*pool
)
2593 const char *dev_name
= dm_device_name(pool
->pool_md
);
2595 DMERR_LIMIT("%s: aborting current metadata transaction", dev_name
);
2596 if (dm_pool_abort_metadata(pool
->pmd
)) {
2597 DMERR("%s: failed to abort metadata transaction", dev_name
);
2598 set_pool_mode(pool
, PM_FAIL
);
2601 if (dm_pool_metadata_set_needs_check(pool
->pmd
)) {
2602 DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name
);
2603 set_pool_mode(pool
, PM_FAIL
);
2607 static void metadata_operation_failed(struct pool
*pool
, const char *op
, int r
)
2609 DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d",
2610 dm_device_name(pool
->pool_md
), op
, r
);
2612 abort_transaction(pool
);
2613 set_pool_mode(pool
, PM_READ_ONLY
);
2616 /*----------------------------------------------------------------*/
2619 * Mapping functions.
2623 * Called only while mapping a thin bio to hand it over to the workqueue.
2625 static void thin_defer_bio(struct thin_c
*tc
, struct bio
*bio
)
2627 unsigned long flags
;
2628 struct pool
*pool
= tc
->pool
;
2630 spin_lock_irqsave(&tc
->lock
, flags
);
2631 bio_list_add(&tc
->deferred_bio_list
, bio
);
2632 spin_unlock_irqrestore(&tc
->lock
, flags
);
2637 static void thin_defer_bio_with_throttle(struct thin_c
*tc
, struct bio
*bio
)
2639 struct pool
*pool
= tc
->pool
;
2641 throttle_lock(&pool
->throttle
);
2642 thin_defer_bio(tc
, bio
);
2643 throttle_unlock(&pool
->throttle
);
2646 static void thin_defer_cell(struct thin_c
*tc
, struct dm_bio_prison_cell
*cell
)
2648 unsigned long flags
;
2649 struct pool
*pool
= tc
->pool
;
2651 throttle_lock(&pool
->throttle
);
2652 spin_lock_irqsave(&tc
->lock
, flags
);
2653 list_add_tail(&cell
->user_list
, &tc
->deferred_cells
);
2654 spin_unlock_irqrestore(&tc
->lock
, flags
);
2655 throttle_unlock(&pool
->throttle
);
2660 static void thin_hook_bio(struct thin_c
*tc
, struct bio
*bio
)
2662 struct dm_thin_endio_hook
*h
= dm_per_bio_data(bio
, sizeof(struct dm_thin_endio_hook
));
2665 h
->shared_read_entry
= NULL
;
2666 h
->all_io_entry
= NULL
;
2667 h
->overwrite_mapping
= NULL
;
2672 * Non-blocking function called from the thin target's map function.
2674 static int thin_bio_map(struct dm_target
*ti
, struct bio
*bio
)
2677 struct thin_c
*tc
= ti
->private;
2678 dm_block_t block
= get_bio_block(tc
, bio
);
2679 struct dm_thin_device
*td
= tc
->td
;
2680 struct dm_thin_lookup_result result
;
2681 struct dm_bio_prison_cell
*virt_cell
, *data_cell
;
2682 struct dm_cell_key key
;
2684 thin_hook_bio(tc
, bio
);
2686 if (tc
->requeue_mode
) {
2687 bio
->bi_status
= BLK_STS_DM_REQUEUE
;
2689 return DM_MAPIO_SUBMITTED
;
2692 if (get_pool_mode(tc
->pool
) == PM_FAIL
) {
2694 return DM_MAPIO_SUBMITTED
;
2697 if (op_is_flush(bio
->bi_opf
) || bio_op(bio
) == REQ_OP_DISCARD
) {
2698 thin_defer_bio_with_throttle(tc
, bio
);
2699 return DM_MAPIO_SUBMITTED
;
2703 * We must hold the virtual cell before doing the lookup, otherwise
2704 * there's a race with discard.
2706 build_virtual_key(tc
->td
, block
, &key
);
2707 if (bio_detain(tc
->pool
, &key
, bio
, &virt_cell
))
2708 return DM_MAPIO_SUBMITTED
;
2710 r
= dm_thin_find_block(td
, block
, 0, &result
);
2713 * Note that we defer readahead too.
2717 if (unlikely(result
.shared
)) {
2719 * We have a race condition here between the
2720 * result.shared value returned by the lookup and
2721 * snapshot creation, which may cause new
2724 * To avoid this always quiesce the origin before
2725 * taking the snap. You want to do this anyway to
2726 * ensure a consistent application view
2729 * More distant ancestors are irrelevant. The
2730 * shared flag will be set in their case.
2732 thin_defer_cell(tc
, virt_cell
);
2733 return DM_MAPIO_SUBMITTED
;
2736 build_data_key(tc
->td
, result
.block
, &key
);
2737 if (bio_detain(tc
->pool
, &key
, bio
, &data_cell
)) {
2738 cell_defer_no_holder(tc
, virt_cell
);
2739 return DM_MAPIO_SUBMITTED
;
2742 inc_all_io_entry(tc
->pool
, bio
);
2743 cell_defer_no_holder(tc
, data_cell
);
2744 cell_defer_no_holder(tc
, virt_cell
);
2746 remap(tc
, bio
, result
.block
);
2747 return DM_MAPIO_REMAPPED
;
2751 thin_defer_cell(tc
, virt_cell
);
2752 return DM_MAPIO_SUBMITTED
;
2756 * Must always call bio_io_error on failure.
2757 * dm_thin_find_block can fail with -EINVAL if the
2758 * pool is switched to fail-io mode.
2761 cell_defer_no_holder(tc
, virt_cell
);
2762 return DM_MAPIO_SUBMITTED
;
2766 static int pool_is_congested(struct dm_target_callbacks
*cb
, int bdi_bits
)
2768 struct pool_c
*pt
= container_of(cb
, struct pool_c
, callbacks
);
2769 struct request_queue
*q
;
2771 if (get_pool_mode(pt
->pool
) == PM_OUT_OF_DATA_SPACE
)
2774 q
= bdev_get_queue(pt
->data_dev
->bdev
);
2775 return bdi_congested(q
->backing_dev_info
, bdi_bits
);
2778 static void requeue_bios(struct pool
*pool
)
2780 unsigned long flags
;
2784 list_for_each_entry_rcu(tc
, &pool
->active_thins
, list
) {
2785 spin_lock_irqsave(&tc
->lock
, flags
);
2786 bio_list_merge(&tc
->deferred_bio_list
, &tc
->retry_on_resume_list
);
2787 bio_list_init(&tc
->retry_on_resume_list
);
2788 spin_unlock_irqrestore(&tc
->lock
, flags
);
2793 /*----------------------------------------------------------------
2794 * Binding of control targets to a pool object
2795 *--------------------------------------------------------------*/
2796 static bool data_dev_supports_discard(struct pool_c
*pt
)
2798 struct request_queue
*q
= bdev_get_queue(pt
->data_dev
->bdev
);
2800 return q
&& blk_queue_discard(q
);
2803 static bool is_factor(sector_t block_size
, uint32_t n
)
2805 return !sector_div(block_size
, n
);
2809 * If discard_passdown was enabled verify that the data device
2810 * supports discards. Disable discard_passdown if not.
2812 static void disable_passdown_if_not_supported(struct pool_c
*pt
)
2814 struct pool
*pool
= pt
->pool
;
2815 struct block_device
*data_bdev
= pt
->data_dev
->bdev
;
2816 struct queue_limits
*data_limits
= &bdev_get_queue(data_bdev
)->limits
;
2817 const char *reason
= NULL
;
2818 char buf
[BDEVNAME_SIZE
];
2820 if (!pt
->adjusted_pf
.discard_passdown
)
2823 if (!data_dev_supports_discard(pt
))
2824 reason
= "discard unsupported";
2826 else if (data_limits
->max_discard_sectors
< pool
->sectors_per_block
)
2827 reason
= "max discard sectors smaller than a block";
2830 DMWARN("Data device (%s) %s: Disabling discard passdown.", bdevname(data_bdev
, buf
), reason
);
2831 pt
->adjusted_pf
.discard_passdown
= false;
2835 static int bind_control_target(struct pool
*pool
, struct dm_target
*ti
)
2837 struct pool_c
*pt
= ti
->private;
2840 * We want to make sure that a pool in PM_FAIL mode is never upgraded.
2842 enum pool_mode old_mode
= get_pool_mode(pool
);
2843 enum pool_mode new_mode
= pt
->adjusted_pf
.mode
;
2846 * Don't change the pool's mode until set_pool_mode() below.
2847 * Otherwise the pool's process_* function pointers may
2848 * not match the desired pool mode.
2850 pt
->adjusted_pf
.mode
= old_mode
;
2853 pool
->pf
= pt
->adjusted_pf
;
2854 pool
->low_water_blocks
= pt
->low_water_blocks
;
2856 set_pool_mode(pool
, new_mode
);
2861 static void unbind_control_target(struct pool
*pool
, struct dm_target
*ti
)
2867 /*----------------------------------------------------------------
2869 *--------------------------------------------------------------*/
2870 /* Initialize pool features. */
2871 static void pool_features_init(struct pool_features
*pf
)
2873 pf
->mode
= PM_WRITE
;
2874 pf
->zero_new_blocks
= true;
2875 pf
->discard_enabled
= true;
2876 pf
->discard_passdown
= true;
2877 pf
->error_if_no_space
= false;
2880 static void __pool_destroy(struct pool
*pool
)
2882 __pool_table_remove(pool
);
2884 vfree(pool
->cell_sort_array
);
2885 if (dm_pool_metadata_close(pool
->pmd
) < 0)
2886 DMWARN("%s: dm_pool_metadata_close() failed.", __func__
);
2888 dm_bio_prison_destroy(pool
->prison
);
2889 dm_kcopyd_client_destroy(pool
->copier
);
2892 destroy_workqueue(pool
->wq
);
2894 if (pool
->next_mapping
)
2895 mempool_free(pool
->next_mapping
, pool
->mapping_pool
);
2896 mempool_destroy(pool
->mapping_pool
);
2897 dm_deferred_set_destroy(pool
->shared_read_ds
);
2898 dm_deferred_set_destroy(pool
->all_io_ds
);
2902 static struct kmem_cache
*_new_mapping_cache
;
2904 static struct pool
*pool_create(struct mapped_device
*pool_md
,
2905 struct block_device
*metadata_dev
,
2906 unsigned long block_size
,
2907 int read_only
, char **error
)
2912 struct dm_pool_metadata
*pmd
;
2913 bool format_device
= read_only
? false : true;
2915 pmd
= dm_pool_metadata_open(metadata_dev
, block_size
, format_device
);
2917 *error
= "Error creating metadata object";
2918 return (struct pool
*)pmd
;
2921 pool
= kmalloc(sizeof(*pool
), GFP_KERNEL
);
2923 *error
= "Error allocating memory for pool";
2924 err_p
= ERR_PTR(-ENOMEM
);
2929 pool
->sectors_per_block
= block_size
;
2930 if (block_size
& (block_size
- 1))
2931 pool
->sectors_per_block_shift
= -1;
2933 pool
->sectors_per_block_shift
= __ffs(block_size
);
2934 pool
->low_water_blocks
= 0;
2935 pool_features_init(&pool
->pf
);
2936 pool
->prison
= dm_bio_prison_create();
2937 if (!pool
->prison
) {
2938 *error
= "Error creating pool's bio prison";
2939 err_p
= ERR_PTR(-ENOMEM
);
2943 pool
->copier
= dm_kcopyd_client_create(&dm_kcopyd_throttle
);
2944 if (IS_ERR(pool
->copier
)) {
2945 r
= PTR_ERR(pool
->copier
);
2946 *error
= "Error creating pool's kcopyd client";
2948 goto bad_kcopyd_client
;
2952 * Create singlethreaded workqueue that will service all devices
2953 * that use this metadata.
2955 pool
->wq
= alloc_ordered_workqueue("dm-" DM_MSG_PREFIX
, WQ_MEM_RECLAIM
);
2957 *error
= "Error creating pool's workqueue";
2958 err_p
= ERR_PTR(-ENOMEM
);
2962 throttle_init(&pool
->throttle
);
2963 INIT_WORK(&pool
->worker
, do_worker
);
2964 INIT_DELAYED_WORK(&pool
->waker
, do_waker
);
2965 INIT_DELAYED_WORK(&pool
->no_space_timeout
, do_no_space_timeout
);
2966 spin_lock_init(&pool
->lock
);
2967 bio_list_init(&pool
->deferred_flush_bios
);
2968 INIT_LIST_HEAD(&pool
->prepared_mappings
);
2969 INIT_LIST_HEAD(&pool
->prepared_discards
);
2970 INIT_LIST_HEAD(&pool
->prepared_discards_pt2
);
2971 INIT_LIST_HEAD(&pool
->active_thins
);
2972 pool
->low_water_triggered
= false;
2973 pool
->suspended
= true;
2974 pool
->out_of_data_space
= false;
2976 pool
->shared_read_ds
= dm_deferred_set_create();
2977 if (!pool
->shared_read_ds
) {
2978 *error
= "Error creating pool's shared read deferred set";
2979 err_p
= ERR_PTR(-ENOMEM
);
2980 goto bad_shared_read_ds
;
2983 pool
->all_io_ds
= dm_deferred_set_create();
2984 if (!pool
->all_io_ds
) {
2985 *error
= "Error creating pool's all io deferred set";
2986 err_p
= ERR_PTR(-ENOMEM
);
2990 pool
->next_mapping
= NULL
;
2991 pool
->mapping_pool
= mempool_create_slab_pool(MAPPING_POOL_SIZE
,
2992 _new_mapping_cache
);
2993 if (!pool
->mapping_pool
) {
2994 *error
= "Error creating pool's mapping mempool";
2995 err_p
= ERR_PTR(-ENOMEM
);
2996 goto bad_mapping_pool
;
2999 pool
->cell_sort_array
= vmalloc(sizeof(*pool
->cell_sort_array
) * CELL_SORT_ARRAY_SIZE
);
3000 if (!pool
->cell_sort_array
) {
3001 *error
= "Error allocating cell sort array";
3002 err_p
= ERR_PTR(-ENOMEM
);
3003 goto bad_sort_array
;
3006 pool
->ref_count
= 1;
3007 pool
->last_commit_jiffies
= jiffies
;
3008 pool
->pool_md
= pool_md
;
3009 pool
->md_dev
= metadata_dev
;
3010 __pool_table_insert(pool
);
3015 mempool_destroy(pool
->mapping_pool
);
3017 dm_deferred_set_destroy(pool
->all_io_ds
);
3019 dm_deferred_set_destroy(pool
->shared_read_ds
);
3021 destroy_workqueue(pool
->wq
);
3023 dm_kcopyd_client_destroy(pool
->copier
);
3025 dm_bio_prison_destroy(pool
->prison
);
3029 if (dm_pool_metadata_close(pmd
))
3030 DMWARN("%s: dm_pool_metadata_close() failed.", __func__
);
3035 static void __pool_inc(struct pool
*pool
)
3037 BUG_ON(!mutex_is_locked(&dm_thin_pool_table
.mutex
));
3041 static void __pool_dec(struct pool
*pool
)
3043 BUG_ON(!mutex_is_locked(&dm_thin_pool_table
.mutex
));
3044 BUG_ON(!pool
->ref_count
);
3045 if (!--pool
->ref_count
)
3046 __pool_destroy(pool
);
3049 static struct pool
*__pool_find(struct mapped_device
*pool_md
,
3050 struct block_device
*metadata_dev
,
3051 unsigned long block_size
, int read_only
,
3052 char **error
, int *created
)
3054 struct pool
*pool
= __pool_table_lookup_metadata_dev(metadata_dev
);
3057 if (pool
->pool_md
!= pool_md
) {
3058 *error
= "metadata device already in use by a pool";
3059 return ERR_PTR(-EBUSY
);
3064 pool
= __pool_table_lookup(pool_md
);
3066 if (pool
->md_dev
!= metadata_dev
) {
3067 *error
= "different pool cannot replace a pool";
3068 return ERR_PTR(-EINVAL
);
3073 pool
= pool_create(pool_md
, metadata_dev
, block_size
, read_only
, error
);
3081 /*----------------------------------------------------------------
3082 * Pool target methods
3083 *--------------------------------------------------------------*/
3084 static void pool_dtr(struct dm_target
*ti
)
3086 struct pool_c
*pt
= ti
->private;
3088 mutex_lock(&dm_thin_pool_table
.mutex
);
3090 unbind_control_target(pt
->pool
, ti
);
3091 __pool_dec(pt
->pool
);
3092 dm_put_device(ti
, pt
->metadata_dev
);
3093 dm_put_device(ti
, pt
->data_dev
);
3096 mutex_unlock(&dm_thin_pool_table
.mutex
);
3099 static int parse_pool_features(struct dm_arg_set
*as
, struct pool_features
*pf
,
3100 struct dm_target
*ti
)
3104 const char *arg_name
;
3106 static const struct dm_arg _args
[] = {
3107 {0, 4, "Invalid number of pool feature arguments"},
3111 * No feature arguments supplied.
3116 r
= dm_read_arg_group(_args
, as
, &argc
, &ti
->error
);
3120 while (argc
&& !r
) {
3121 arg_name
= dm_shift_arg(as
);
3124 if (!strcasecmp(arg_name
, "skip_block_zeroing"))
3125 pf
->zero_new_blocks
= false;
3127 else if (!strcasecmp(arg_name
, "ignore_discard"))
3128 pf
->discard_enabled
= false;
3130 else if (!strcasecmp(arg_name
, "no_discard_passdown"))
3131 pf
->discard_passdown
= false;
3133 else if (!strcasecmp(arg_name
, "read_only"))
3134 pf
->mode
= PM_READ_ONLY
;
3136 else if (!strcasecmp(arg_name
, "error_if_no_space"))
3137 pf
->error_if_no_space
= true;
3140 ti
->error
= "Unrecognised pool feature requested";
3149 static void metadata_low_callback(void *context
)
3151 struct pool
*pool
= context
;
3153 DMWARN("%s: reached low water mark for metadata device: sending event.",
3154 dm_device_name(pool
->pool_md
));
3156 dm_table_event(pool
->ti
->table
);
3159 static sector_t
get_dev_size(struct block_device
*bdev
)
3161 return i_size_read(bdev
->bd_inode
) >> SECTOR_SHIFT
;
3164 static void warn_if_metadata_device_too_big(struct block_device
*bdev
)
3166 sector_t metadata_dev_size
= get_dev_size(bdev
);
3167 char buffer
[BDEVNAME_SIZE
];
3169 if (metadata_dev_size
> THIN_METADATA_MAX_SECTORS_WARNING
)
3170 DMWARN("Metadata device %s is larger than %u sectors: excess space will not be used.",
3171 bdevname(bdev
, buffer
), THIN_METADATA_MAX_SECTORS
);
3174 static sector_t
get_metadata_dev_size(struct block_device
*bdev
)
3176 sector_t metadata_dev_size
= get_dev_size(bdev
);
3178 if (metadata_dev_size
> THIN_METADATA_MAX_SECTORS
)
3179 metadata_dev_size
= THIN_METADATA_MAX_SECTORS
;
3181 return metadata_dev_size
;
3184 static dm_block_t
get_metadata_dev_size_in_blocks(struct block_device
*bdev
)
3186 sector_t metadata_dev_size
= get_metadata_dev_size(bdev
);
3188 sector_div(metadata_dev_size
, THIN_METADATA_BLOCK_SIZE
);
3190 return metadata_dev_size
;
3194 * When a metadata threshold is crossed a dm event is triggered, and
3195 * userland should respond by growing the metadata device. We could let
3196 * userland set the threshold, like we do with the data threshold, but I'm
3197 * not sure they know enough to do this well.
3199 static dm_block_t
calc_metadata_threshold(struct pool_c
*pt
)
3202 * 4M is ample for all ops with the possible exception of thin
3203 * device deletion which is harmless if it fails (just retry the
3204 * delete after you've grown the device).
3206 dm_block_t quarter
= get_metadata_dev_size_in_blocks(pt
->metadata_dev
->bdev
) / 4;
3207 return min((dm_block_t
)1024ULL /* 4M */, quarter
);
3211 * thin-pool <metadata dev> <data dev>
3212 * <data block size (sectors)>
3213 * <low water mark (blocks)>
3214 * [<#feature args> [<arg>]*]
3216 * Optional feature arguments are:
3217 * skip_block_zeroing: skips the zeroing of newly-provisioned blocks.
3218 * ignore_discard: disable discard
3219 * no_discard_passdown: don't pass discards down to the data device
3220 * read_only: Don't allow any changes to be made to the pool metadata.
3221 * error_if_no_space: error IOs, instead of queueing, if no space.
3223 static int pool_ctr(struct dm_target
*ti
, unsigned argc
, char **argv
)
3225 int r
, pool_created
= 0;
3228 struct pool_features pf
;
3229 struct dm_arg_set as
;
3230 struct dm_dev
*data_dev
;
3231 unsigned long block_size
;
3232 dm_block_t low_water_blocks
;
3233 struct dm_dev
*metadata_dev
;
3234 fmode_t metadata_mode
;
3237 * FIXME Remove validation from scope of lock.
3239 mutex_lock(&dm_thin_pool_table
.mutex
);
3242 ti
->error
= "Invalid argument count";
3251 * Set default pool features.
3253 pool_features_init(&pf
);
3255 dm_consume_args(&as
, 4);
3256 r
= parse_pool_features(&as
, &pf
, ti
);
3260 metadata_mode
= FMODE_READ
| ((pf
.mode
== PM_READ_ONLY
) ? 0 : FMODE_WRITE
);
3261 r
= dm_get_device(ti
, argv
[0], metadata_mode
, &metadata_dev
);
3263 ti
->error
= "Error opening metadata block device";
3266 warn_if_metadata_device_too_big(metadata_dev
->bdev
);
3268 r
= dm_get_device(ti
, argv
[1], FMODE_READ
| FMODE_WRITE
, &data_dev
);
3270 ti
->error
= "Error getting data device";
3274 if (kstrtoul(argv
[2], 10, &block_size
) || !block_size
||
3275 block_size
< DATA_DEV_BLOCK_SIZE_MIN_SECTORS
||
3276 block_size
> DATA_DEV_BLOCK_SIZE_MAX_SECTORS
||
3277 block_size
& (DATA_DEV_BLOCK_SIZE_MIN_SECTORS
- 1)) {
3278 ti
->error
= "Invalid block size";
3283 if (kstrtoull(argv
[3], 10, (unsigned long long *)&low_water_blocks
)) {
3284 ti
->error
= "Invalid low water mark";
3289 pt
= kzalloc(sizeof(*pt
), GFP_KERNEL
);
3295 pool
= __pool_find(dm_table_get_md(ti
->table
), metadata_dev
->bdev
,
3296 block_size
, pf
.mode
== PM_READ_ONLY
, &ti
->error
, &pool_created
);
3303 * 'pool_created' reflects whether this is the first table load.
3304 * Top level discard support is not allowed to be changed after
3305 * initial load. This would require a pool reload to trigger thin
3308 if (!pool_created
&& pf
.discard_enabled
!= pool
->pf
.discard_enabled
) {
3309 ti
->error
= "Discard support cannot be disabled once enabled";
3311 goto out_flags_changed
;
3316 pt
->metadata_dev
= metadata_dev
;
3317 pt
->data_dev
= data_dev
;
3318 pt
->low_water_blocks
= low_water_blocks
;
3319 pt
->adjusted_pf
= pt
->requested_pf
= pf
;
3320 ti
->num_flush_bios
= 1;
3323 * Only need to enable discards if the pool should pass
3324 * them down to the data device. The thin device's discard
3325 * processing will cause mappings to be removed from the btree.
3327 if (pf
.discard_enabled
&& pf
.discard_passdown
) {
3328 ti
->num_discard_bios
= 1;
3331 * Setting 'discards_supported' circumvents the normal
3332 * stacking of discard limits (this keeps the pool and
3333 * thin devices' discard limits consistent).
3335 ti
->discards_supported
= true;
3339 r
= dm_pool_register_metadata_threshold(pt
->pool
->pmd
,
3340 calc_metadata_threshold(pt
),
3341 metadata_low_callback
,
3344 goto out_flags_changed
;
3346 pt
->callbacks
.congested_fn
= pool_is_congested
;
3347 dm_table_add_target_callbacks(ti
->table
, &pt
->callbacks
);
3349 mutex_unlock(&dm_thin_pool_table
.mutex
);
3358 dm_put_device(ti
, data_dev
);
3360 dm_put_device(ti
, metadata_dev
);
3362 mutex_unlock(&dm_thin_pool_table
.mutex
);
3367 static int pool_map(struct dm_target
*ti
, struct bio
*bio
)
3370 struct pool_c
*pt
= ti
->private;
3371 struct pool
*pool
= pt
->pool
;
3372 unsigned long flags
;
3375 * As this is a singleton target, ti->begin is always zero.
3377 spin_lock_irqsave(&pool
->lock
, flags
);
3378 bio_set_dev(bio
, pt
->data_dev
->bdev
);
3379 r
= DM_MAPIO_REMAPPED
;
3380 spin_unlock_irqrestore(&pool
->lock
, flags
);
3385 static int maybe_resize_data_dev(struct dm_target
*ti
, bool *need_commit
)
3388 struct pool_c
*pt
= ti
->private;
3389 struct pool
*pool
= pt
->pool
;
3390 sector_t data_size
= ti
->len
;
3391 dm_block_t sb_data_size
;
3393 *need_commit
= false;
3395 (void) sector_div(data_size
, pool
->sectors_per_block
);
3397 r
= dm_pool_get_data_dev_size(pool
->pmd
, &sb_data_size
);
3399 DMERR("%s: failed to retrieve data device size",
3400 dm_device_name(pool
->pool_md
));
3404 if (data_size
< sb_data_size
) {
3405 DMERR("%s: pool target (%llu blocks) too small: expected %llu",
3406 dm_device_name(pool
->pool_md
),
3407 (unsigned long long)data_size
, sb_data_size
);
3410 } else if (data_size
> sb_data_size
) {
3411 if (dm_pool_metadata_needs_check(pool
->pmd
)) {
3412 DMERR("%s: unable to grow the data device until repaired.",
3413 dm_device_name(pool
->pool_md
));
3418 DMINFO("%s: growing the data device from %llu to %llu blocks",
3419 dm_device_name(pool
->pool_md
),
3420 sb_data_size
, (unsigned long long)data_size
);
3421 r
= dm_pool_resize_data_dev(pool
->pmd
, data_size
);
3423 metadata_operation_failed(pool
, "dm_pool_resize_data_dev", r
);
3427 *need_commit
= true;
3433 static int maybe_resize_metadata_dev(struct dm_target
*ti
, bool *need_commit
)
3436 struct pool_c
*pt
= ti
->private;
3437 struct pool
*pool
= pt
->pool
;
3438 dm_block_t metadata_dev_size
, sb_metadata_dev_size
;
3440 *need_commit
= false;
3442 metadata_dev_size
= get_metadata_dev_size_in_blocks(pool
->md_dev
);
3444 r
= dm_pool_get_metadata_dev_size(pool
->pmd
, &sb_metadata_dev_size
);
3446 DMERR("%s: failed to retrieve metadata device size",
3447 dm_device_name(pool
->pool_md
));
3451 if (metadata_dev_size
< sb_metadata_dev_size
) {
3452 DMERR("%s: metadata device (%llu blocks) too small: expected %llu",
3453 dm_device_name(pool
->pool_md
),
3454 metadata_dev_size
, sb_metadata_dev_size
);
3457 } else if (metadata_dev_size
> sb_metadata_dev_size
) {
3458 if (dm_pool_metadata_needs_check(pool
->pmd
)) {
3459 DMERR("%s: unable to grow the metadata device until repaired.",
3460 dm_device_name(pool
->pool_md
));
3464 warn_if_metadata_device_too_big(pool
->md_dev
);
3465 DMINFO("%s: growing the metadata device from %llu to %llu blocks",
3466 dm_device_name(pool
->pool_md
),
3467 sb_metadata_dev_size
, metadata_dev_size
);
3469 if (get_pool_mode(pool
) == PM_OUT_OF_METADATA_SPACE
)
3470 set_pool_mode(pool
, PM_WRITE
);
3472 r
= dm_pool_resize_metadata_dev(pool
->pmd
, metadata_dev_size
);
3474 metadata_operation_failed(pool
, "dm_pool_resize_metadata_dev", r
);
3478 *need_commit
= true;
3485 * Retrieves the number of blocks of the data device from
3486 * the superblock and compares it to the actual device size,
3487 * thus resizing the data device in case it has grown.
3489 * This both copes with opening preallocated data devices in the ctr
3490 * being followed by a resume
3492 * calling the resume method individually after userspace has
3493 * grown the data device in reaction to a table event.
3495 static int pool_preresume(struct dm_target
*ti
)
3498 bool need_commit1
, need_commit2
;
3499 struct pool_c
*pt
= ti
->private;
3500 struct pool
*pool
= pt
->pool
;
3503 * Take control of the pool object.
3505 r
= bind_control_target(pool
, ti
);
3509 r
= maybe_resize_data_dev(ti
, &need_commit1
);
3513 r
= maybe_resize_metadata_dev(ti
, &need_commit2
);
3517 if (need_commit1
|| need_commit2
)
3518 (void) commit(pool
);
3523 static void pool_suspend_active_thins(struct pool
*pool
)
3527 /* Suspend all active thin devices */
3528 tc
= get_first_thin(pool
);
3530 dm_internal_suspend_noflush(tc
->thin_md
);
3531 tc
= get_next_thin(pool
, tc
);
3535 static void pool_resume_active_thins(struct pool
*pool
)
3539 /* Resume all active thin devices */
3540 tc
= get_first_thin(pool
);
3542 dm_internal_resume(tc
->thin_md
);
3543 tc
= get_next_thin(pool
, tc
);
3547 static void pool_resume(struct dm_target
*ti
)
3549 struct pool_c
*pt
= ti
->private;
3550 struct pool
*pool
= pt
->pool
;
3551 unsigned long flags
;
3554 * Must requeue active_thins' bios and then resume
3555 * active_thins _before_ clearing 'suspend' flag.
3558 pool_resume_active_thins(pool
);
3560 spin_lock_irqsave(&pool
->lock
, flags
);
3561 pool
->low_water_triggered
= false;
3562 pool
->suspended
= false;
3563 spin_unlock_irqrestore(&pool
->lock
, flags
);
3565 do_waker(&pool
->waker
.work
);
3568 static void pool_presuspend(struct dm_target
*ti
)
3570 struct pool_c
*pt
= ti
->private;
3571 struct pool
*pool
= pt
->pool
;
3572 unsigned long flags
;
3574 spin_lock_irqsave(&pool
->lock
, flags
);
3575 pool
->suspended
= true;
3576 spin_unlock_irqrestore(&pool
->lock
, flags
);
3578 pool_suspend_active_thins(pool
);
3581 static void pool_presuspend_undo(struct dm_target
*ti
)
3583 struct pool_c
*pt
= ti
->private;
3584 struct pool
*pool
= pt
->pool
;
3585 unsigned long flags
;
3587 pool_resume_active_thins(pool
);
3589 spin_lock_irqsave(&pool
->lock
, flags
);
3590 pool
->suspended
= false;
3591 spin_unlock_irqrestore(&pool
->lock
, flags
);
3594 static void pool_postsuspend(struct dm_target
*ti
)
3596 struct pool_c
*pt
= ti
->private;
3597 struct pool
*pool
= pt
->pool
;
3599 cancel_delayed_work_sync(&pool
->waker
);
3600 cancel_delayed_work_sync(&pool
->no_space_timeout
);
3601 flush_workqueue(pool
->wq
);
3602 (void) commit(pool
);
3605 static int check_arg_count(unsigned argc
, unsigned args_required
)
3607 if (argc
!= args_required
) {
3608 DMWARN("Message received with %u arguments instead of %u.",
3609 argc
, args_required
);
3616 static int read_dev_id(char *arg
, dm_thin_id
*dev_id
, int warning
)
3618 if (!kstrtoull(arg
, 10, (unsigned long long *)dev_id
) &&
3619 *dev_id
<= MAX_DEV_ID
)
3623 DMWARN("Message received with invalid device id: %s", arg
);
3628 static int process_create_thin_mesg(unsigned argc
, char **argv
, struct pool
*pool
)
3633 r
= check_arg_count(argc
, 2);
3637 r
= read_dev_id(argv
[1], &dev_id
, 1);
3641 r
= dm_pool_create_thin(pool
->pmd
, dev_id
);
3643 DMWARN("Creation of new thinly-provisioned device with id %s failed.",
3651 static int process_create_snap_mesg(unsigned argc
, char **argv
, struct pool
*pool
)
3654 dm_thin_id origin_dev_id
;
3657 r
= check_arg_count(argc
, 3);
3661 r
= read_dev_id(argv
[1], &dev_id
, 1);
3665 r
= read_dev_id(argv
[2], &origin_dev_id
, 1);
3669 r
= dm_pool_create_snap(pool
->pmd
, dev_id
, origin_dev_id
);
3671 DMWARN("Creation of new snapshot %s of device %s failed.",
3679 static int process_delete_mesg(unsigned argc
, char **argv
, struct pool
*pool
)
3684 r
= check_arg_count(argc
, 2);
3688 r
= read_dev_id(argv
[1], &dev_id
, 1);
3692 r
= dm_pool_delete_thin_device(pool
->pmd
, dev_id
);
3694 DMWARN("Deletion of thin device %s failed.", argv
[1]);
3699 static int process_set_transaction_id_mesg(unsigned argc
, char **argv
, struct pool
*pool
)
3701 dm_thin_id old_id
, new_id
;
3704 r
= check_arg_count(argc
, 3);
3708 if (kstrtoull(argv
[1], 10, (unsigned long long *)&old_id
)) {
3709 DMWARN("set_transaction_id message: Unrecognised id %s.", argv
[1]);
3713 if (kstrtoull(argv
[2], 10, (unsigned long long *)&new_id
)) {
3714 DMWARN("set_transaction_id message: Unrecognised new id %s.", argv
[2]);
3718 r
= dm_pool_set_metadata_transaction_id(pool
->pmd
, old_id
, new_id
);
3720 DMWARN("Failed to change transaction id from %s to %s.",
3728 static int process_reserve_metadata_snap_mesg(unsigned argc
, char **argv
, struct pool
*pool
)
3732 r
= check_arg_count(argc
, 1);
3736 (void) commit(pool
);
3738 r
= dm_pool_reserve_metadata_snap(pool
->pmd
);
3740 DMWARN("reserve_metadata_snap message failed.");
3745 static int process_release_metadata_snap_mesg(unsigned argc
, char **argv
, struct pool
*pool
)
3749 r
= check_arg_count(argc
, 1);
3753 r
= dm_pool_release_metadata_snap(pool
->pmd
);
3755 DMWARN("release_metadata_snap message failed.");
3761 * Messages supported:
3762 * create_thin <dev_id>
3763 * create_snap <dev_id> <origin_id>
3765 * set_transaction_id <current_trans_id> <new_trans_id>
3766 * reserve_metadata_snap
3767 * release_metadata_snap
3769 static int pool_message(struct dm_target
*ti
, unsigned argc
, char **argv
)
3772 struct pool_c
*pt
= ti
->private;
3773 struct pool
*pool
= pt
->pool
;
3775 if (get_pool_mode(pool
) >= PM_OUT_OF_METADATA_SPACE
) {
3776 DMERR("%s: unable to service pool target messages in READ_ONLY or FAIL mode",
3777 dm_device_name(pool
->pool_md
));
3781 if (!strcasecmp(argv
[0], "create_thin"))
3782 r
= process_create_thin_mesg(argc
, argv
, pool
);
3784 else if (!strcasecmp(argv
[0], "create_snap"))
3785 r
= process_create_snap_mesg(argc
, argv
, pool
);
3787 else if (!strcasecmp(argv
[0], "delete"))
3788 r
= process_delete_mesg(argc
, argv
, pool
);
3790 else if (!strcasecmp(argv
[0], "set_transaction_id"))
3791 r
= process_set_transaction_id_mesg(argc
, argv
, pool
);
3793 else if (!strcasecmp(argv
[0], "reserve_metadata_snap"))
3794 r
= process_reserve_metadata_snap_mesg(argc
, argv
, pool
);
3796 else if (!strcasecmp(argv
[0], "release_metadata_snap"))
3797 r
= process_release_metadata_snap_mesg(argc
, argv
, pool
);
3800 DMWARN("Unrecognised thin pool target message received: %s", argv
[0]);
3803 (void) commit(pool
);
3808 static void emit_flags(struct pool_features
*pf
, char *result
,
3809 unsigned sz
, unsigned maxlen
)
3811 unsigned count
= !pf
->zero_new_blocks
+ !pf
->discard_enabled
+
3812 !pf
->discard_passdown
+ (pf
->mode
== PM_READ_ONLY
) +
3813 pf
->error_if_no_space
;
3814 DMEMIT("%u ", count
);
3816 if (!pf
->zero_new_blocks
)
3817 DMEMIT("skip_block_zeroing ");
3819 if (!pf
->discard_enabled
)
3820 DMEMIT("ignore_discard ");
3822 if (!pf
->discard_passdown
)
3823 DMEMIT("no_discard_passdown ");
3825 if (pf
->mode
== PM_READ_ONLY
)
3826 DMEMIT("read_only ");
3828 if (pf
->error_if_no_space
)
3829 DMEMIT("error_if_no_space ");
3834 * <transaction id> <used metadata sectors>/<total metadata sectors>
3835 * <used data sectors>/<total data sectors> <held metadata root>
3836 * <pool mode> <discard config> <no space config> <needs_check>
3838 static void pool_status(struct dm_target
*ti
, status_type_t type
,
3839 unsigned status_flags
, char *result
, unsigned maxlen
)
3843 uint64_t transaction_id
;
3844 dm_block_t nr_free_blocks_data
;
3845 dm_block_t nr_free_blocks_metadata
;
3846 dm_block_t nr_blocks_data
;
3847 dm_block_t nr_blocks_metadata
;
3848 dm_block_t held_root
;
3849 enum pool_mode mode
;
3850 char buf
[BDEVNAME_SIZE
];
3851 char buf2
[BDEVNAME_SIZE
];
3852 struct pool_c
*pt
= ti
->private;
3853 struct pool
*pool
= pt
->pool
;
3856 case STATUSTYPE_INFO
:
3857 if (get_pool_mode(pool
) == PM_FAIL
) {
3862 /* Commit to ensure statistics aren't out-of-date */
3863 if (!(status_flags
& DM_STATUS_NOFLUSH_FLAG
) && !dm_suspended(ti
))
3864 (void) commit(pool
);
3866 r
= dm_pool_get_metadata_transaction_id(pool
->pmd
, &transaction_id
);
3868 DMERR("%s: dm_pool_get_metadata_transaction_id returned %d",
3869 dm_device_name(pool
->pool_md
), r
);
3873 r
= dm_pool_get_free_metadata_block_count(pool
->pmd
, &nr_free_blocks_metadata
);
3875 DMERR("%s: dm_pool_get_free_metadata_block_count returned %d",
3876 dm_device_name(pool
->pool_md
), r
);
3880 r
= dm_pool_get_metadata_dev_size(pool
->pmd
, &nr_blocks_metadata
);
3882 DMERR("%s: dm_pool_get_metadata_dev_size returned %d",
3883 dm_device_name(pool
->pool_md
), r
);
3887 r
= dm_pool_get_free_block_count(pool
->pmd
, &nr_free_blocks_data
);
3889 DMERR("%s: dm_pool_get_free_block_count returned %d",
3890 dm_device_name(pool
->pool_md
), r
);
3894 r
= dm_pool_get_data_dev_size(pool
->pmd
, &nr_blocks_data
);
3896 DMERR("%s: dm_pool_get_data_dev_size returned %d",
3897 dm_device_name(pool
->pool_md
), r
);
3901 r
= dm_pool_get_metadata_snap(pool
->pmd
, &held_root
);
3903 DMERR("%s: dm_pool_get_metadata_snap returned %d",
3904 dm_device_name(pool
->pool_md
), r
);
3908 DMEMIT("%llu %llu/%llu %llu/%llu ",
3909 (unsigned long long)transaction_id
,
3910 (unsigned long long)(nr_blocks_metadata
- nr_free_blocks_metadata
),
3911 (unsigned long long)nr_blocks_metadata
,
3912 (unsigned long long)(nr_blocks_data
- nr_free_blocks_data
),
3913 (unsigned long long)nr_blocks_data
);
3916 DMEMIT("%llu ", held_root
);
3920 mode
= get_pool_mode(pool
);
3921 if (mode
== PM_OUT_OF_DATA_SPACE
)
3922 DMEMIT("out_of_data_space ");
3923 else if (is_read_only_pool_mode(mode
))
3928 if (!pool
->pf
.discard_enabled
)
3929 DMEMIT("ignore_discard ");
3930 else if (pool
->pf
.discard_passdown
)
3931 DMEMIT("discard_passdown ");
3933 DMEMIT("no_discard_passdown ");
3935 if (pool
->pf
.error_if_no_space
)
3936 DMEMIT("error_if_no_space ");
3938 DMEMIT("queue_if_no_space ");
3940 if (dm_pool_metadata_needs_check(pool
->pmd
))
3941 DMEMIT("needs_check ");
3947 case STATUSTYPE_TABLE
:
3948 DMEMIT("%s %s %lu %llu ",
3949 format_dev_t(buf
, pt
->metadata_dev
->bdev
->bd_dev
),
3950 format_dev_t(buf2
, pt
->data_dev
->bdev
->bd_dev
),
3951 (unsigned long)pool
->sectors_per_block
,
3952 (unsigned long long)pt
->low_water_blocks
);
3953 emit_flags(&pt
->requested_pf
, result
, sz
, maxlen
);
3962 static int pool_iterate_devices(struct dm_target
*ti
,
3963 iterate_devices_callout_fn fn
, void *data
)
3965 struct pool_c
*pt
= ti
->private;
3967 return fn(ti
, pt
->data_dev
, 0, ti
->len
, data
);
3970 static void pool_io_hints(struct dm_target
*ti
, struct queue_limits
*limits
)
3972 struct pool_c
*pt
= ti
->private;
3973 struct pool
*pool
= pt
->pool
;
3974 sector_t io_opt_sectors
= limits
->io_opt
>> SECTOR_SHIFT
;
3977 * If max_sectors is smaller than pool->sectors_per_block adjust it
3978 * to the highest possible power-of-2 factor of pool->sectors_per_block.
3979 * This is especially beneficial when the pool's data device is a RAID
3980 * device that has a full stripe width that matches pool->sectors_per_block
3981 * -- because even though partial RAID stripe-sized IOs will be issued to a
3982 * single RAID stripe; when aggregated they will end on a full RAID stripe
3983 * boundary.. which avoids additional partial RAID stripe writes cascading
3985 if (limits
->max_sectors
< pool
->sectors_per_block
) {
3986 while (!is_factor(pool
->sectors_per_block
, limits
->max_sectors
)) {
3987 if ((limits
->max_sectors
& (limits
->max_sectors
- 1)) == 0)
3988 limits
->max_sectors
--;
3989 limits
->max_sectors
= rounddown_pow_of_two(limits
->max_sectors
);
3994 * If the system-determined stacked limits are compatible with the
3995 * pool's blocksize (io_opt is a factor) do not override them.
3997 if (io_opt_sectors
< pool
->sectors_per_block
||
3998 !is_factor(io_opt_sectors
, pool
->sectors_per_block
)) {
3999 if (is_factor(pool
->sectors_per_block
, limits
->max_sectors
))
4000 blk_limits_io_min(limits
, limits
->max_sectors
<< SECTOR_SHIFT
);
4002 blk_limits_io_min(limits
, pool
->sectors_per_block
<< SECTOR_SHIFT
);
4003 blk_limits_io_opt(limits
, pool
->sectors_per_block
<< SECTOR_SHIFT
);
4007 * pt->adjusted_pf is a staging area for the actual features to use.
4008 * They get transferred to the live pool in bind_control_target()
4009 * called from pool_preresume().
4011 if (!pt
->adjusted_pf
.discard_enabled
) {
4013 * Must explicitly disallow stacking discard limits otherwise the
4014 * block layer will stack them if pool's data device has support.
4015 * QUEUE_FLAG_DISCARD wouldn't be set but there is no way for the
4016 * user to see that, so make sure to set all discard limits to 0.
4018 limits
->discard_granularity
= 0;
4022 disable_passdown_if_not_supported(pt
);
4025 * The pool uses the same discard limits as the underlying data
4026 * device. DM core has already set this up.
4030 static struct target_type pool_target
= {
4031 .name
= "thin-pool",
4032 .features
= DM_TARGET_SINGLETON
| DM_TARGET_ALWAYS_WRITEABLE
|
4033 DM_TARGET_IMMUTABLE
,
4034 .version
= {1, 19, 0},
4035 .module
= THIS_MODULE
,
4039 .presuspend
= pool_presuspend
,
4040 .presuspend_undo
= pool_presuspend_undo
,
4041 .postsuspend
= pool_postsuspend
,
4042 .preresume
= pool_preresume
,
4043 .resume
= pool_resume
,
4044 .message
= pool_message
,
4045 .status
= pool_status
,
4046 .iterate_devices
= pool_iterate_devices
,
4047 .io_hints
= pool_io_hints
,
4050 /*----------------------------------------------------------------
4051 * Thin target methods
4052 *--------------------------------------------------------------*/
4053 static void thin_get(struct thin_c
*tc
)
4055 atomic_inc(&tc
->refcount
);
4058 static void thin_put(struct thin_c
*tc
)
4060 if (atomic_dec_and_test(&tc
->refcount
))
4061 complete(&tc
->can_destroy
);
4064 static void thin_dtr(struct dm_target
*ti
)
4066 struct thin_c
*tc
= ti
->private;
4067 unsigned long flags
;
4069 spin_lock_irqsave(&tc
->pool
->lock
, flags
);
4070 list_del_rcu(&tc
->list
);
4071 spin_unlock_irqrestore(&tc
->pool
->lock
, flags
);
4075 wait_for_completion(&tc
->can_destroy
);
4077 mutex_lock(&dm_thin_pool_table
.mutex
);
4079 __pool_dec(tc
->pool
);
4080 dm_pool_close_thin_device(tc
->td
);
4081 dm_put_device(ti
, tc
->pool_dev
);
4083 dm_put_device(ti
, tc
->origin_dev
);
4086 mutex_unlock(&dm_thin_pool_table
.mutex
);
4090 * Thin target parameters:
4092 * <pool_dev> <dev_id> [origin_dev]
4094 * pool_dev: the path to the pool (eg, /dev/mapper/my_pool)
4095 * dev_id: the internal device identifier
4096 * origin_dev: a device external to the pool that should act as the origin
4098 * If the pool device has discards disabled, they get disabled for the thin
4101 static int thin_ctr(struct dm_target
*ti
, unsigned argc
, char **argv
)
4105 struct dm_dev
*pool_dev
, *origin_dev
;
4106 struct mapped_device
*pool_md
;
4107 unsigned long flags
;
4109 mutex_lock(&dm_thin_pool_table
.mutex
);
4111 if (argc
!= 2 && argc
!= 3) {
4112 ti
->error
= "Invalid argument count";
4117 tc
= ti
->private = kzalloc(sizeof(*tc
), GFP_KERNEL
);
4119 ti
->error
= "Out of memory";
4123 tc
->thin_md
= dm_table_get_md(ti
->table
);
4124 spin_lock_init(&tc
->lock
);
4125 INIT_LIST_HEAD(&tc
->deferred_cells
);
4126 bio_list_init(&tc
->deferred_bio_list
);
4127 bio_list_init(&tc
->retry_on_resume_list
);
4128 tc
->sort_bio_list
= RB_ROOT
;
4131 r
= dm_get_device(ti
, argv
[2], FMODE_READ
, &origin_dev
);
4133 ti
->error
= "Error opening origin device";
4134 goto bad_origin_dev
;
4136 tc
->origin_dev
= origin_dev
;
4139 r
= dm_get_device(ti
, argv
[0], dm_table_get_mode(ti
->table
), &pool_dev
);
4141 ti
->error
= "Error opening pool device";
4144 tc
->pool_dev
= pool_dev
;
4146 if (read_dev_id(argv
[1], (unsigned long long *)&tc
->dev_id
, 0)) {
4147 ti
->error
= "Invalid device id";
4152 pool_md
= dm_get_md(tc
->pool_dev
->bdev
->bd_dev
);
4154 ti
->error
= "Couldn't get pool mapped device";
4159 tc
->pool
= __pool_table_lookup(pool_md
);
4161 ti
->error
= "Couldn't find pool object";
4163 goto bad_pool_lookup
;
4165 __pool_inc(tc
->pool
);
4167 if (get_pool_mode(tc
->pool
) == PM_FAIL
) {
4168 ti
->error
= "Couldn't open thin device, Pool is in fail mode";
4173 r
= dm_pool_open_thin_device(tc
->pool
->pmd
, tc
->dev_id
, &tc
->td
);
4175 ti
->error
= "Couldn't open thin internal device";
4179 r
= dm_set_target_max_io_len(ti
, tc
->pool
->sectors_per_block
);
4183 ti
->num_flush_bios
= 1;
4184 ti
->flush_supported
= true;
4185 ti
->per_io_data_size
= sizeof(struct dm_thin_endio_hook
);
4187 /* In case the pool supports discards, pass them on. */
4188 if (tc
->pool
->pf
.discard_enabled
) {
4189 ti
->discards_supported
= true;
4190 ti
->num_discard_bios
= 1;
4191 ti
->split_discard_bios
= false;
4194 mutex_unlock(&dm_thin_pool_table
.mutex
);
4196 spin_lock_irqsave(&tc
->pool
->lock
, flags
);
4197 if (tc
->pool
->suspended
) {
4198 spin_unlock_irqrestore(&tc
->pool
->lock
, flags
);
4199 mutex_lock(&dm_thin_pool_table
.mutex
); /* reacquire for __pool_dec */
4200 ti
->error
= "Unable to activate thin device while pool is suspended";
4204 atomic_set(&tc
->refcount
, 1);
4205 init_completion(&tc
->can_destroy
);
4206 list_add_tail_rcu(&tc
->list
, &tc
->pool
->active_thins
);
4207 spin_unlock_irqrestore(&tc
->pool
->lock
, flags
);
4209 * This synchronize_rcu() call is needed here otherwise we risk a
4210 * wake_worker() call finding no bios to process (because the newly
4211 * added tc isn't yet visible). So this reduces latency since we
4212 * aren't then dependent on the periodic commit to wake_worker().
4221 dm_pool_close_thin_device(tc
->td
);
4223 __pool_dec(tc
->pool
);
4227 dm_put_device(ti
, tc
->pool_dev
);
4230 dm_put_device(ti
, tc
->origin_dev
);
4234 mutex_unlock(&dm_thin_pool_table
.mutex
);
4239 static int thin_map(struct dm_target
*ti
, struct bio
*bio
)
4241 bio
->bi_iter
.bi_sector
= dm_target_offset(ti
, bio
->bi_iter
.bi_sector
);
4243 return thin_bio_map(ti
, bio
);
4246 static int thin_endio(struct dm_target
*ti
, struct bio
*bio
,
4249 unsigned long flags
;
4250 struct dm_thin_endio_hook
*h
= dm_per_bio_data(bio
, sizeof(struct dm_thin_endio_hook
));
4251 struct list_head work
;
4252 struct dm_thin_new_mapping
*m
, *tmp
;
4253 struct pool
*pool
= h
->tc
->pool
;
4255 if (h
->shared_read_entry
) {
4256 INIT_LIST_HEAD(&work
);
4257 dm_deferred_entry_dec(h
->shared_read_entry
, &work
);
4259 spin_lock_irqsave(&pool
->lock
, flags
);
4260 list_for_each_entry_safe(m
, tmp
, &work
, list
) {
4262 __complete_mapping_preparation(m
);
4264 spin_unlock_irqrestore(&pool
->lock
, flags
);
4267 if (h
->all_io_entry
) {
4268 INIT_LIST_HEAD(&work
);
4269 dm_deferred_entry_dec(h
->all_io_entry
, &work
);
4270 if (!list_empty(&work
)) {
4271 spin_lock_irqsave(&pool
->lock
, flags
);
4272 list_for_each_entry_safe(m
, tmp
, &work
, list
)
4273 list_add_tail(&m
->list
, &pool
->prepared_discards
);
4274 spin_unlock_irqrestore(&pool
->lock
, flags
);
4280 cell_defer_no_holder(h
->tc
, h
->cell
);
4282 return DM_ENDIO_DONE
;
4285 static void thin_presuspend(struct dm_target
*ti
)
4287 struct thin_c
*tc
= ti
->private;
4289 if (dm_noflush_suspending(ti
))
4290 noflush_work(tc
, do_noflush_start
);
4293 static void thin_postsuspend(struct dm_target
*ti
)
4295 struct thin_c
*tc
= ti
->private;
4298 * The dm_noflush_suspending flag has been cleared by now, so
4299 * unfortunately we must always run this.
4301 noflush_work(tc
, do_noflush_stop
);
4304 static int thin_preresume(struct dm_target
*ti
)
4306 struct thin_c
*tc
= ti
->private;
4309 tc
->origin_size
= get_dev_size(tc
->origin_dev
->bdev
);
4315 * <nr mapped sectors> <highest mapped sector>
4317 static void thin_status(struct dm_target
*ti
, status_type_t type
,
4318 unsigned status_flags
, char *result
, unsigned maxlen
)
4322 dm_block_t mapped
, highest
;
4323 char buf
[BDEVNAME_SIZE
];
4324 struct thin_c
*tc
= ti
->private;
4326 if (get_pool_mode(tc
->pool
) == PM_FAIL
) {
4335 case STATUSTYPE_INFO
:
4336 r
= dm_thin_get_mapped_count(tc
->td
, &mapped
);
4338 DMERR("dm_thin_get_mapped_count returned %d", r
);
4342 r
= dm_thin_get_highest_mapped_block(tc
->td
, &highest
);
4344 DMERR("dm_thin_get_highest_mapped_block returned %d", r
);
4348 DMEMIT("%llu ", mapped
* tc
->pool
->sectors_per_block
);
4350 DMEMIT("%llu", ((highest
+ 1) *
4351 tc
->pool
->sectors_per_block
) - 1);
4356 case STATUSTYPE_TABLE
:
4358 format_dev_t(buf
, tc
->pool_dev
->bdev
->bd_dev
),
4359 (unsigned long) tc
->dev_id
);
4361 DMEMIT(" %s", format_dev_t(buf
, tc
->origin_dev
->bdev
->bd_dev
));
4372 static int thin_iterate_devices(struct dm_target
*ti
,
4373 iterate_devices_callout_fn fn
, void *data
)
4376 struct thin_c
*tc
= ti
->private;
4377 struct pool
*pool
= tc
->pool
;
4380 * We can't call dm_pool_get_data_dev_size() since that blocks. So
4381 * we follow a more convoluted path through to the pool's target.
4384 return 0; /* nothing is bound */
4386 blocks
= pool
->ti
->len
;
4387 (void) sector_div(blocks
, pool
->sectors_per_block
);
4389 return fn(ti
, tc
->pool_dev
, 0, pool
->sectors_per_block
* blocks
, data
);
4394 static void thin_io_hints(struct dm_target
*ti
, struct queue_limits
*limits
)
4396 struct thin_c
*tc
= ti
->private;
4397 struct pool
*pool
= tc
->pool
;
4399 if (!pool
->pf
.discard_enabled
)
4402 limits
->discard_granularity
= pool
->sectors_per_block
<< SECTOR_SHIFT
;
4403 limits
->max_discard_sectors
= 2048 * 1024 * 16; /* 16G */
4406 static struct target_type thin_target
= {
4408 .version
= {1, 19, 0},
4409 .module
= THIS_MODULE
,
4413 .end_io
= thin_endio
,
4414 .preresume
= thin_preresume
,
4415 .presuspend
= thin_presuspend
,
4416 .postsuspend
= thin_postsuspend
,
4417 .status
= thin_status
,
4418 .iterate_devices
= thin_iterate_devices
,
4419 .io_hints
= thin_io_hints
,
4422 /*----------------------------------------------------------------*/
4424 static int __init
dm_thin_init(void)
4430 _new_mapping_cache
= KMEM_CACHE(dm_thin_new_mapping
, 0);
4431 if (!_new_mapping_cache
)
4434 r
= dm_register_target(&thin_target
);
4436 goto bad_new_mapping_cache
;
4438 r
= dm_register_target(&pool_target
);
4440 goto bad_thin_target
;
4445 dm_unregister_target(&thin_target
);
4446 bad_new_mapping_cache
:
4447 kmem_cache_destroy(_new_mapping_cache
);
4452 static void dm_thin_exit(void)
4454 dm_unregister_target(&thin_target
);
4455 dm_unregister_target(&pool_target
);
4457 kmem_cache_destroy(_new_mapping_cache
);
4460 module_init(dm_thin_init
);
4461 module_exit(dm_thin_exit
);
4463 module_param_named(no_space_timeout
, no_space_timeout_secs
, uint
, S_IRUGO
| S_IWUSR
);
4464 MODULE_PARM_DESC(no_space_timeout
, "Out of data space queue IO timeout in seconds");
4466 MODULE_DESCRIPTION(DM_NAME
" thin provisioning target");
4467 MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
4468 MODULE_LICENSE("GPL");