4 * Copyright (C) 2013 Proxmox Server Solutions
5 * Copyright (c) 2019 Virtuozzo International GmbH.
8 * Dietmar Maurer (dietmar@proxmox.com)
9 * Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
11 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12 * See the COPYING file in the top-level directory.
15 #include "qemu/osdep.h"
18 #include "qapi/error.h"
19 #include "block/block-copy.h"
20 #include "sysemu/block-backend.h"
21 #include "qemu/units.h"
22 #include "qemu/coroutine.h"
23 #include "block/aio_task.h"
24 #include "qemu/error-report.h"
26 #define BLOCK_COPY_MAX_COPY_RANGE (16 * MiB)
27 #define BLOCK_COPY_MAX_BUFFER (1 * MiB)
28 #define BLOCK_COPY_MAX_MEM (128 * MiB)
29 #define BLOCK_COPY_MAX_WORKERS 64
30 #define BLOCK_COPY_SLICE_TIME 100000000ULL /* ns */
31 #define BLOCK_COPY_CLUSTER_SIZE_DEFAULT (1 << 16)
34 COPY_READ_WRITE_CLUSTER
,
41 static coroutine_fn
int block_copy_task_entry(AioTask
*task
);
43 typedef struct BlockCopyCallState
{
44 /* Fields initialized in block_copy_async() and never changed. */
50 bool ignore_ratelimit
;
51 BlockCopyAsyncCallbackFunc cb
;
53 /* Coroutine where async block-copy is running */
56 /* Fields whose state changes throughout the execution */
57 bool finished
; /* atomic */
58 QemuCoSleep sleep
; /* TODO: protect API with a lock */
59 bool cancelled
; /* atomic */
60 /* To reference all call states from BlockCopyState */
61 QLIST_ENTRY(BlockCopyCallState
) list
;
64 * Fields that report information about return values and erros.
65 * Protected by lock in BlockCopyState.
69 * @ret is set concurrently by tasks under mutex. Only set once by first
70 * failed task (and untouched if no task failed).
71 * After finishing (call_state->finished is true), it is not modified
72 * anymore and may be safely read without mutex.
77 typedef struct BlockCopyTask
{
81 * Fields initialized in block_copy_task_create()
85 BlockCopyCallState
*call_state
;
88 * @method can also be set again in the while loop of
89 * block_copy_dirty_clusters(), but it is never accessed concurrently
90 * because the only other function that reads it is
91 * block_copy_task_entry() and it is invoked afterwards in the same
94 BlockCopyMethod method
;
97 * Fields whose state changes throughout the execution
98 * Protected by lock in BlockCopyState.
100 CoQueue wait_queue
; /* coroutines blocked on this task */
102 * Only protect the case of parallel read while updating @bytes
103 * value in block_copy_task_shrink().
106 QLIST_ENTRY(BlockCopyTask
) list
;
109 static int64_t task_end(BlockCopyTask
*task
)
111 return task
->offset
+ task
->bytes
;
114 typedef struct BlockCopyState
{
116 * BdrvChild objects are not owned or managed by block-copy. They are
117 * provided by block-copy user and user is responsible for appropriate
118 * permissions on these children.
124 * Fields initialized in block_copy_state_new()
127 int64_t cluster_size
;
128 int64_t max_transfer
;
130 BdrvRequestFlags write_flags
;
133 * Fields whose state changes throughout the execution
137 int64_t in_flight_bytes
;
138 BlockCopyMethod method
;
139 QLIST_HEAD(, BlockCopyTask
) tasks
; /* All tasks from all block-copy calls */
140 QLIST_HEAD(, BlockCopyCallState
) calls
;
144 * Used by sync=top jobs, which first scan the source node for unallocated
145 * areas and clear them in the copy_bitmap. During this process, the bitmap
146 * is thus not fully initialized: It may still have bits set for areas that
147 * are unallocated and should actually not be copied.
149 * This is indicated by skip_unallocated.
151 * In this case, block_copy() will query the source’s allocation status,
152 * skip unallocated regions, clear them in the copy_bitmap, and invoke
153 * block_copy_reset_unallocated() every time it does.
155 bool skip_unallocated
; /* atomic */
156 /* State fields that use a thread-safe API */
157 BdrvDirtyBitmap
*copy_bitmap
;
158 ProgressMeter
*progress
;
160 RateLimit rate_limit
;
163 /* Called with lock held */
164 static BlockCopyTask
*find_conflicting_task(BlockCopyState
*s
,
165 int64_t offset
, int64_t bytes
)
169 QLIST_FOREACH(t
, &s
->tasks
, list
) {
170 if (offset
+ bytes
> t
->offset
&& offset
< t
->offset
+ t
->bytes
) {
179 * If there are no intersecting tasks return false. Otherwise, wait for the
180 * first found intersecting tasks to finish and return true.
182 * Called with lock held. May temporary release the lock.
183 * Return value of 0 proves that lock was NOT released.
185 static bool coroutine_fn
block_copy_wait_one(BlockCopyState
*s
, int64_t offset
,
188 BlockCopyTask
*task
= find_conflicting_task(s
, offset
, bytes
);
194 qemu_co_queue_wait(&task
->wait_queue
, &s
->lock
);
199 /* Called with lock held */
200 static int64_t block_copy_chunk_size(BlockCopyState
*s
)
203 case COPY_READ_WRITE_CLUSTER
:
204 return s
->cluster_size
;
205 case COPY_READ_WRITE
:
206 case COPY_RANGE_SMALL
:
207 return MIN(MAX(s
->cluster_size
, BLOCK_COPY_MAX_BUFFER
),
209 case COPY_RANGE_FULL
:
210 return MIN(MAX(s
->cluster_size
, BLOCK_COPY_MAX_COPY_RANGE
),
213 /* Cannot have COPY_WRITE_ZEROES here. */
219 * Search for the first dirty area in offset/bytes range and create task at
220 * the beginning of it.
222 static coroutine_fn BlockCopyTask
*
223 block_copy_task_create(BlockCopyState
*s
, BlockCopyCallState
*call_state
,
224 int64_t offset
, int64_t bytes
)
229 QEMU_LOCK_GUARD(&s
->lock
);
230 max_chunk
= MIN_NON_ZERO(block_copy_chunk_size(s
), call_state
->max_chunk
);
231 if (!bdrv_dirty_bitmap_next_dirty_area(s
->copy_bitmap
,
232 offset
, offset
+ bytes
,
233 max_chunk
, &offset
, &bytes
))
238 assert(QEMU_IS_ALIGNED(offset
, s
->cluster_size
));
239 bytes
= QEMU_ALIGN_UP(bytes
, s
->cluster_size
);
241 /* region is dirty, so no existent tasks possible in it */
242 assert(!find_conflicting_task(s
, offset
, bytes
));
244 bdrv_reset_dirty_bitmap(s
->copy_bitmap
, offset
, bytes
);
245 s
->in_flight_bytes
+= bytes
;
247 task
= g_new(BlockCopyTask
, 1);
248 *task
= (BlockCopyTask
) {
249 .task
.func
= block_copy_task_entry
,
251 .call_state
= call_state
,
256 qemu_co_queue_init(&task
->wait_queue
);
257 QLIST_INSERT_HEAD(&s
->tasks
, task
, list
);
263 * block_copy_task_shrink
265 * Drop the tail of the task to be handled later. Set dirty bits back and
266 * wake up all tasks waiting for us (may be some of them are not intersecting
269 static void coroutine_fn
block_copy_task_shrink(BlockCopyTask
*task
,
272 QEMU_LOCK_GUARD(&task
->s
->lock
);
273 if (new_bytes
== task
->bytes
) {
277 assert(new_bytes
> 0 && new_bytes
< task
->bytes
);
279 task
->s
->in_flight_bytes
-= task
->bytes
- new_bytes
;
280 bdrv_set_dirty_bitmap(task
->s
->copy_bitmap
,
281 task
->offset
+ new_bytes
, task
->bytes
- new_bytes
);
283 task
->bytes
= new_bytes
;
284 qemu_co_queue_restart_all(&task
->wait_queue
);
287 static void coroutine_fn
block_copy_task_end(BlockCopyTask
*task
, int ret
)
289 QEMU_LOCK_GUARD(&task
->s
->lock
);
290 task
->s
->in_flight_bytes
-= task
->bytes
;
292 bdrv_set_dirty_bitmap(task
->s
->copy_bitmap
, task
->offset
, task
->bytes
);
294 QLIST_REMOVE(task
, list
);
295 if (task
->s
->progress
) {
296 progress_set_remaining(task
->s
->progress
,
297 bdrv_get_dirty_count(task
->s
->copy_bitmap
) +
298 task
->s
->in_flight_bytes
);
300 qemu_co_queue_restart_all(&task
->wait_queue
);
303 void block_copy_state_free(BlockCopyState
*s
)
309 ratelimit_destroy(&s
->rate_limit
);
310 bdrv_release_dirty_bitmap(s
->copy_bitmap
);
311 shres_destroy(s
->mem
);
315 static uint32_t block_copy_max_transfer(BdrvChild
*source
, BdrvChild
*target
)
317 return MIN_NON_ZERO(INT_MAX
,
318 MIN_NON_ZERO(source
->bs
->bl
.max_transfer
,
319 target
->bs
->bl
.max_transfer
));
322 void block_copy_set_copy_opts(BlockCopyState
*s
, bool use_copy_range
,
325 /* Keep BDRV_REQ_SERIALISING set (or not set) in block_copy_state_new() */
326 s
->write_flags
= (s
->write_flags
& BDRV_REQ_SERIALISING
) |
327 (compress
? BDRV_REQ_WRITE_COMPRESSED
: 0);
329 if (s
->max_transfer
< s
->cluster_size
) {
331 * copy_range does not respect max_transfer. We don't want to bother
332 * with requests smaller than block-copy cluster size, so fallback to
333 * buffered copying (read and write respect max_transfer on their
336 s
->method
= COPY_READ_WRITE_CLUSTER
;
337 } else if (compress
) {
338 /* Compression supports only cluster-size writes and no copy-range. */
339 s
->method
= COPY_READ_WRITE_CLUSTER
;
342 * If copy range enabled, start with COPY_RANGE_SMALL, until first
343 * successful copy_range (look at block_copy_do_copy).
345 s
->method
= use_copy_range
? COPY_RANGE_SMALL
: COPY_READ_WRITE
;
349 static int64_t block_copy_calculate_cluster_size(BlockDriverState
*target
,
354 bool target_does_cow
= bdrv_backing_chain_next(target
);
357 * If there is no backing file on the target, we cannot rely on COW if our
358 * backup cluster size is smaller than the target cluster size. Even for
359 * targets with a backing file, try to avoid COW if possible.
361 ret
= bdrv_get_info(target
, &bdi
);
362 if (ret
== -ENOTSUP
&& !target_does_cow
) {
363 /* Cluster size is not defined */
364 warn_report("The target block device doesn't provide "
365 "information about the block size and it doesn't have a "
366 "backing file. The default block size of %u bytes is "
367 "used. If the actual block size of the target exceeds "
368 "this default, the backup may be unusable",
369 BLOCK_COPY_CLUSTER_SIZE_DEFAULT
);
370 return BLOCK_COPY_CLUSTER_SIZE_DEFAULT
;
371 } else if (ret
< 0 && !target_does_cow
) {
372 error_setg_errno(errp
, -ret
,
373 "Couldn't determine the cluster size of the target image, "
374 "which has no backing file");
375 error_append_hint(errp
,
376 "Aborting, since this may create an unusable destination image\n");
378 } else if (ret
< 0 && target_does_cow
) {
379 /* Not fatal; just trudge on ahead. */
380 return BLOCK_COPY_CLUSTER_SIZE_DEFAULT
;
383 return MAX(BLOCK_COPY_CLUSTER_SIZE_DEFAULT
, bdi
.cluster_size
);
386 BlockCopyState
*block_copy_state_new(BdrvChild
*source
, BdrvChild
*target
,
387 const BdrvDirtyBitmap
*bitmap
,
392 int64_t cluster_size
;
393 BdrvDirtyBitmap
*copy_bitmap
;
396 cluster_size
= block_copy_calculate_cluster_size(target
->bs
, errp
);
397 if (cluster_size
< 0) {
401 copy_bitmap
= bdrv_create_dirty_bitmap(source
->bs
, cluster_size
, NULL
,
406 bdrv_disable_dirty_bitmap(copy_bitmap
);
408 if (!bdrv_merge_dirty_bitmap(copy_bitmap
, bitmap
, NULL
, errp
)) {
409 error_prepend(errp
, "Failed to merge bitmap '%s' to internal "
410 "copy-bitmap: ", bdrv_dirty_bitmap_name(bitmap
));
411 bdrv_release_dirty_bitmap(copy_bitmap
);
415 bdrv_set_dirty_bitmap(copy_bitmap
, 0,
416 bdrv_dirty_bitmap_size(copy_bitmap
));
420 * If source is in backing chain of target assume that target is going to be
421 * used for "image fleecing", i.e. it should represent a kind of snapshot of
422 * source at backup-start point in time. And target is going to be read by
423 * somebody (for example, used as NBD export) during backup job.
425 * In this case, we need to add BDRV_REQ_SERIALISING write flag to avoid
426 * intersection of backup writes and third party reads from target,
427 * otherwise reading from target we may occasionally read already updated by
430 * For more information see commit f8d59dfb40bb and test
431 * tests/qemu-iotests/222
433 is_fleecing
= bdrv_chain_contains(target
->bs
, source
->bs
);
435 s
= g_new(BlockCopyState
, 1);
436 *s
= (BlockCopyState
) {
439 .copy_bitmap
= copy_bitmap
,
440 .cluster_size
= cluster_size
,
441 .len
= bdrv_dirty_bitmap_size(copy_bitmap
),
442 .write_flags
= (is_fleecing
? BDRV_REQ_SERIALISING
: 0),
443 .mem
= shres_create(BLOCK_COPY_MAX_MEM
),
444 .max_transfer
= QEMU_ALIGN_DOWN(
445 block_copy_max_transfer(source
, target
),
449 block_copy_set_copy_opts(s
, false, false);
451 ratelimit_init(&s
->rate_limit
);
452 qemu_co_mutex_init(&s
->lock
);
453 QLIST_INIT(&s
->tasks
);
454 QLIST_INIT(&s
->calls
);
459 /* Only set before running the job, no need for locking. */
460 void block_copy_set_progress_meter(BlockCopyState
*s
, ProgressMeter
*pm
)
466 * Takes ownership of @task
468 * If pool is NULL directly run the task, otherwise schedule it into the pool.
470 * Returns: task.func return code if pool is NULL
471 * otherwise -ECANCELED if pool status is bad
472 * otherwise 0 (successfully scheduled)
474 static coroutine_fn
int block_copy_task_run(AioTaskPool
*pool
,
478 int ret
= task
->task
.func(&task
->task
);
484 aio_task_pool_wait_slot(pool
);
485 if (aio_task_pool_status(pool
) < 0) {
486 co_put_to_shres(task
->s
->mem
, task
->bytes
);
487 block_copy_task_end(task
, -ECANCELED
);
492 aio_task_pool_start_task(pool
, &task
->task
);
500 * Do copy of cluster-aligned chunk. Requested region is allowed to exceed
501 * s->len only to cover last cluster when s->len is not aligned to clusters.
503 * No sync here: nor bitmap neighter intersecting requests handling, only copy.
505 * @method is an in-out argument, so that copy_range can be either extended to
506 * a full-size buffer or disabled if the copy_range attempt fails. The output
507 * value of @method should be used for subsequent tasks.
508 * Returns 0 on success.
510 static int coroutine_fn
block_copy_do_copy(BlockCopyState
*s
,
511 int64_t offset
, int64_t bytes
,
512 BlockCopyMethod
*method
,
516 int64_t nbytes
= MIN(offset
+ bytes
, s
->len
) - offset
;
517 void *bounce_buffer
= NULL
;
519 assert(offset
>= 0 && bytes
> 0 && INT64_MAX
- offset
>= bytes
);
520 assert(QEMU_IS_ALIGNED(offset
, s
->cluster_size
));
521 assert(QEMU_IS_ALIGNED(bytes
, s
->cluster_size
));
522 assert(offset
< s
->len
);
523 assert(offset
+ bytes
<= s
->len
||
524 offset
+ bytes
== QEMU_ALIGN_UP(s
->len
, s
->cluster_size
));
525 assert(nbytes
< INT_MAX
);
528 case COPY_WRITE_ZEROES
:
529 ret
= bdrv_co_pwrite_zeroes(s
->target
, offset
, nbytes
, s
->write_flags
&
530 ~BDRV_REQ_WRITE_COMPRESSED
);
532 trace_block_copy_write_zeroes_fail(s
, offset
, ret
);
533 *error_is_read
= false;
537 case COPY_RANGE_SMALL
:
538 case COPY_RANGE_FULL
:
539 ret
= bdrv_co_copy_range(s
->source
, offset
, s
->target
, offset
, nbytes
,
542 /* Successful copy-range, increase chunk size. */
543 *method
= COPY_RANGE_FULL
;
547 trace_block_copy_copy_range_fail(s
, offset
, ret
);
548 *method
= COPY_READ_WRITE
;
549 /* Fall through to read+write with allocated buffer */
551 case COPY_READ_WRITE_CLUSTER
:
552 case COPY_READ_WRITE
:
554 * In case of failed copy_range request above, we may proceed with
555 * buffered request larger than BLOCK_COPY_MAX_BUFFER.
556 * Still, further requests will be properly limited, so don't care too
557 * much. Moreover the most likely case (copy_range is unsupported for
558 * the configuration, so the very first copy_range request fails)
559 * is handled by setting large copy_size only after first successful
563 bounce_buffer
= qemu_blockalign(s
->source
->bs
, nbytes
);
565 ret
= bdrv_co_pread(s
->source
, offset
, nbytes
, bounce_buffer
, 0);
567 trace_block_copy_read_fail(s
, offset
, ret
);
568 *error_is_read
= true;
572 ret
= bdrv_co_pwrite(s
->target
, offset
, nbytes
, bounce_buffer
,
575 trace_block_copy_write_fail(s
, offset
, ret
);
576 *error_is_read
= false;
581 qemu_vfree(bounce_buffer
);
591 static coroutine_fn
int block_copy_task_entry(AioTask
*task
)
593 BlockCopyTask
*t
= container_of(task
, BlockCopyTask
, task
);
594 BlockCopyState
*s
= t
->s
;
595 bool error_is_read
= false;
596 BlockCopyMethod method
= t
->method
;
599 ret
= block_copy_do_copy(s
, t
->offset
, t
->bytes
, &method
, &error_is_read
);
601 WITH_QEMU_LOCK_GUARD(&s
->lock
) {
602 if (s
->method
== t
->method
) {
607 if (!t
->call_state
->ret
) {
608 t
->call_state
->ret
= ret
;
609 t
->call_state
->error_is_read
= error_is_read
;
611 } else if (s
->progress
) {
612 progress_work_done(s
->progress
, t
->bytes
);
615 co_put_to_shres(s
->mem
, t
->bytes
);
616 block_copy_task_end(t
, ret
);
621 static int block_copy_block_status(BlockCopyState
*s
, int64_t offset
,
622 int64_t bytes
, int64_t *pnum
)
625 BlockDriverState
*base
;
628 if (qatomic_read(&s
->skip_unallocated
)) {
629 base
= bdrv_backing_chain_next(s
->source
->bs
);
634 ret
= bdrv_block_status_above(s
->source
->bs
, base
, offset
, bytes
, &num
,
636 if (ret
< 0 || num
< s
->cluster_size
) {
638 * On error or if failed to obtain large enough chunk just fallback to
641 num
= s
->cluster_size
;
642 ret
= BDRV_BLOCK_ALLOCATED
| BDRV_BLOCK_DATA
;
643 } else if (offset
+ num
== s
->len
) {
644 num
= QEMU_ALIGN_UP(num
, s
->cluster_size
);
646 num
= QEMU_ALIGN_DOWN(num
, s
->cluster_size
);
654 * Check if the cluster starting at offset is allocated or not.
655 * return via pnum the number of contiguous clusters sharing this allocation.
657 static int block_copy_is_cluster_allocated(BlockCopyState
*s
, int64_t offset
,
660 BlockDriverState
*bs
= s
->source
->bs
;
661 int64_t count
, total_count
= 0;
662 int64_t bytes
= s
->len
- offset
;
665 assert(QEMU_IS_ALIGNED(offset
, s
->cluster_size
));
668 ret
= bdrv_is_allocated(bs
, offset
, bytes
, &count
);
673 total_count
+= count
;
675 if (ret
|| count
== 0) {
677 * ret: partial segment(s) are considered allocated.
678 * otherwise: unallocated tail is treated as an entire segment.
680 *pnum
= DIV_ROUND_UP(total_count
, s
->cluster_size
);
684 /* Unallocated segment(s) with uncertain following segment(s) */
685 if (total_count
>= s
->cluster_size
) {
686 *pnum
= total_count
/ s
->cluster_size
;
695 void block_copy_reset(BlockCopyState
*s
, int64_t offset
, int64_t bytes
)
697 QEMU_LOCK_GUARD(&s
->lock
);
699 bdrv_reset_dirty_bitmap(s
->copy_bitmap
, offset
, bytes
);
701 progress_set_remaining(s
->progress
,
702 bdrv_get_dirty_count(s
->copy_bitmap
) +
708 * Reset bits in copy_bitmap starting at offset if they represent unallocated
709 * data in the image. May reset subsequent contiguous bits.
710 * @return 0 when the cluster at @offset was unallocated,
711 * 1 otherwise, and -ret on error.
713 int64_t block_copy_reset_unallocated(BlockCopyState
*s
,
714 int64_t offset
, int64_t *count
)
717 int64_t clusters
, bytes
;
719 ret
= block_copy_is_cluster_allocated(s
, offset
, &clusters
);
724 bytes
= clusters
* s
->cluster_size
;
727 block_copy_reset(s
, offset
, bytes
);
735 * block_copy_dirty_clusters
737 * Copy dirty clusters in @offset/@bytes range.
738 * Returns 1 if dirty clusters found and successfully copied, 0 if no dirty
739 * clusters found and -errno on failure.
741 static int coroutine_fn
742 block_copy_dirty_clusters(BlockCopyCallState
*call_state
)
744 BlockCopyState
*s
= call_state
->s
;
745 int64_t offset
= call_state
->offset
;
746 int64_t bytes
= call_state
->bytes
;
749 bool found_dirty
= false;
750 int64_t end
= offset
+ bytes
;
751 AioTaskPool
*aio
= NULL
;
754 * block_copy() user is responsible for keeping source and target in same
757 assert(bdrv_get_aio_context(s
->source
->bs
) ==
758 bdrv_get_aio_context(s
->target
->bs
));
760 assert(QEMU_IS_ALIGNED(offset
, s
->cluster_size
));
761 assert(QEMU_IS_ALIGNED(bytes
, s
->cluster_size
));
763 while (bytes
&& aio_task_pool_status(aio
) == 0 &&
764 !qatomic_read(&call_state
->cancelled
)) {
766 int64_t status_bytes
;
768 task
= block_copy_task_create(s
, call_state
, offset
, bytes
);
770 /* No more dirty bits in the bitmap */
771 trace_block_copy_skip_range(s
, offset
, bytes
);
774 if (task
->offset
> offset
) {
775 trace_block_copy_skip_range(s
, offset
, task
->offset
- offset
);
780 ret
= block_copy_block_status(s
, task
->offset
, task
->bytes
,
782 assert(ret
>= 0); /* never fail */
783 if (status_bytes
< task
->bytes
) {
784 block_copy_task_shrink(task
, status_bytes
);
786 if (qatomic_read(&s
->skip_unallocated
) &&
787 !(ret
& BDRV_BLOCK_ALLOCATED
)) {
788 block_copy_task_end(task
, 0);
789 trace_block_copy_skip_range(s
, task
->offset
, task
->bytes
);
790 offset
= task_end(task
);
791 bytes
= end
- offset
;
795 if (ret
& BDRV_BLOCK_ZERO
) {
796 task
->method
= COPY_WRITE_ZEROES
;
799 if (!call_state
->ignore_ratelimit
) {
800 uint64_t ns
= ratelimit_calculate_delay(&s
->rate_limit
, 0);
802 block_copy_task_end(task
, -EAGAIN
);
804 qemu_co_sleep_ns_wakeable(&call_state
->sleep
,
805 QEMU_CLOCK_REALTIME
, ns
);
810 ratelimit_calculate_delay(&s
->rate_limit
, task
->bytes
);
812 trace_block_copy_process(s
, task
->offset
);
814 co_get_from_shres(s
->mem
, task
->bytes
);
816 offset
= task_end(task
);
817 bytes
= end
- offset
;
820 aio
= aio_task_pool_new(call_state
->max_workers
);
823 ret
= block_copy_task_run(aio
, task
);
831 aio_task_pool_wait_all(aio
);
834 * We are not really interested in -ECANCELED returned from
835 * block_copy_task_run. If it fails, it means some task already failed
836 * for real reason, let's return first failure.
837 * Still, assert that we don't rewrite failure by success.
839 * Note: ret may be positive here because of block-status result.
841 assert(ret
>= 0 || aio_task_pool_status(aio
) < 0);
842 ret
= aio_task_pool_status(aio
);
844 aio_task_pool_free(aio
);
847 return ret
< 0 ? ret
: found_dirty
;
850 void block_copy_kick(BlockCopyCallState
*call_state
)
852 qemu_co_sleep_wake(&call_state
->sleep
);
858 * Copy requested region, accordingly to dirty bitmap.
859 * Collaborate with parallel block_copy requests: if they succeed it will help
860 * us. If they fail, we will retry not-copied regions. So, if we return error,
861 * it means that some I/O operation failed in context of _this_ block_copy call,
862 * not some parallel operation.
864 static int coroutine_fn
block_copy_common(BlockCopyCallState
*call_state
)
867 BlockCopyState
*s
= call_state
->s
;
869 qemu_co_mutex_lock(&s
->lock
);
870 QLIST_INSERT_HEAD(&s
->calls
, call_state
, list
);
871 qemu_co_mutex_unlock(&s
->lock
);
874 ret
= block_copy_dirty_clusters(call_state
);
876 if (ret
== 0 && !qatomic_read(&call_state
->cancelled
)) {
877 WITH_QEMU_LOCK_GUARD(&s
->lock
) {
879 * Check that there is no task we still need to
882 ret
= block_copy_wait_one(s
, call_state
->offset
,
886 * No pending tasks, but check again the bitmap in this
887 * same critical section, since a task might have failed
888 * between this and the critical section in
889 * block_copy_dirty_clusters().
891 * block_copy_wait_one return value 0 also means that it
892 * didn't release the lock. So, we are still in the same
893 * critical section, not interrupted by any concurrent
896 ret
= bdrv_dirty_bitmap_next_dirty(s
->copy_bitmap
,
898 call_state
->bytes
) >= 0;
904 * We retry in two cases:
905 * 1. Some progress done
906 * Something was copied, which means that there were yield points
907 * and some new dirty bits may have appeared (due to failed parallel
908 * block-copy requests).
909 * 2. We have waited for some intersecting block-copy request
910 * It may have failed and produced new dirty bits.
912 } while (ret
> 0 && !qatomic_read(&call_state
->cancelled
));
914 qatomic_store_release(&call_state
->finished
, true);
916 if (call_state
->cb
) {
917 call_state
->cb(call_state
->cb_opaque
);
920 qemu_co_mutex_lock(&s
->lock
);
921 QLIST_REMOVE(call_state
, list
);
922 qemu_co_mutex_unlock(&s
->lock
);
927 int coroutine_fn
block_copy(BlockCopyState
*s
, int64_t start
, int64_t bytes
,
928 bool ignore_ratelimit
)
930 BlockCopyCallState call_state
= {
934 .ignore_ratelimit
= ignore_ratelimit
,
935 .max_workers
= BLOCK_COPY_MAX_WORKERS
,
938 return block_copy_common(&call_state
);
941 static void coroutine_fn
block_copy_async_co_entry(void *opaque
)
943 block_copy_common(opaque
);
946 BlockCopyCallState
*block_copy_async(BlockCopyState
*s
,
947 int64_t offset
, int64_t bytes
,
948 int max_workers
, int64_t max_chunk
,
949 BlockCopyAsyncCallbackFunc cb
,
952 BlockCopyCallState
*call_state
= g_new(BlockCopyCallState
, 1);
954 *call_state
= (BlockCopyCallState
) {
958 .max_workers
= max_workers
,
959 .max_chunk
= max_chunk
,
961 .cb_opaque
= cb_opaque
,
963 .co
= qemu_coroutine_create(block_copy_async_co_entry
, call_state
),
966 qemu_coroutine_enter(call_state
->co
);
971 void block_copy_call_free(BlockCopyCallState
*call_state
)
977 assert(qatomic_read(&call_state
->finished
));
981 bool block_copy_call_finished(BlockCopyCallState
*call_state
)
983 return qatomic_read(&call_state
->finished
);
986 bool block_copy_call_succeeded(BlockCopyCallState
*call_state
)
988 return qatomic_load_acquire(&call_state
->finished
) &&
989 !qatomic_read(&call_state
->cancelled
) &&
990 call_state
->ret
== 0;
993 bool block_copy_call_failed(BlockCopyCallState
*call_state
)
995 return qatomic_load_acquire(&call_state
->finished
) &&
996 !qatomic_read(&call_state
->cancelled
) &&
1000 bool block_copy_call_cancelled(BlockCopyCallState
*call_state
)
1002 return qatomic_read(&call_state
->cancelled
);
1005 int block_copy_call_status(BlockCopyCallState
*call_state
, bool *error_is_read
)
1007 assert(qatomic_load_acquire(&call_state
->finished
));
1008 if (error_is_read
) {
1009 *error_is_read
= call_state
->error_is_read
;
1011 return call_state
->ret
;
1015 * Note that cancelling and finishing are racy.
1016 * User can cancel a block-copy that is already finished.
1018 void block_copy_call_cancel(BlockCopyCallState
*call_state
)
1020 qatomic_set(&call_state
->cancelled
, true);
1021 block_copy_kick(call_state
);
1024 BdrvDirtyBitmap
*block_copy_dirty_bitmap(BlockCopyState
*s
)
1026 return s
->copy_bitmap
;
1029 int64_t block_copy_cluster_size(BlockCopyState
*s
)
1031 return s
->cluster_size
;
1034 void block_copy_set_skip_unallocated(BlockCopyState
*s
, bool skip
)
1036 qatomic_set(&s
->skip_unallocated
, skip
);
1039 void block_copy_set_speed(BlockCopyState
*s
, uint64_t speed
)
1041 ratelimit_set_speed(&s
->rate_limit
, speed
, BLOCK_COPY_SLICE_TIME
);
1044 * Note: it's good to kick all call states from here, but it should be done
1045 * only from a coroutine, to not crash if s->calls list changed while
1046 * entering one call. So for now, the only user of this function kicks its
1047 * only one call_state by hand.