4 * Copyright (C) 2013 Proxmox Server Solutions
5 * Copyright (c) 2019 Virtuozzo International GmbH.
8 * Dietmar Maurer (dietmar@proxmox.com)
9 * Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
11 * This work is licensed under the terms of the GNU GPL, version 2 or later.
12 * See the COPYING file in the top-level directory.
15 #include "qemu/osdep.h"
18 #include "qapi/error.h"
19 #include "block/block-copy.h"
20 #include "block/block_int-io.h"
21 #include "block/dirty-bitmap.h"
22 #include "block/reqlist.h"
23 #include "sysemu/block-backend.h"
24 #include "qemu/units.h"
25 #include "qemu/co-shared-resource.h"
26 #include "qemu/coroutine.h"
27 #include "qemu/ratelimit.h"
28 #include "block/aio_task.h"
29 #include "qemu/error-report.h"
30 #include "qemu/memalign.h"
32 #define BLOCK_COPY_MAX_COPY_RANGE (16 * MiB)
33 #define BLOCK_COPY_MAX_BUFFER (1 * MiB)
34 #define BLOCK_COPY_MAX_MEM (128 * MiB)
35 #define BLOCK_COPY_MAX_WORKERS 64
36 #define BLOCK_COPY_SLICE_TIME 100000000ULL /* ns */
37 #define BLOCK_COPY_CLUSTER_SIZE_DEFAULT (1 << 16)
40 COPY_READ_WRITE_CLUSTER
,
47 static coroutine_fn
int block_copy_task_entry(AioTask
*task
);
49 typedef struct BlockCopyCallState
{
50 /* Fields initialized in block_copy_async() and never changed. */
56 bool ignore_ratelimit
;
57 BlockCopyAsyncCallbackFunc cb
;
59 /* Coroutine where async block-copy is running */
62 /* Fields whose state changes throughout the execution */
63 bool finished
; /* atomic */
64 QemuCoSleep sleep
; /* TODO: protect API with a lock */
65 bool cancelled
; /* atomic */
66 /* To reference all call states from BlockCopyState */
67 QLIST_ENTRY(BlockCopyCallState
) list
;
70 * Fields that report information about return values and errors.
71 * Protected by lock in BlockCopyState.
75 * @ret is set concurrently by tasks under mutex. Only set once by first
76 * failed task (and untouched if no task failed).
77 * After finishing (call_state->finished is true), it is not modified
78 * anymore and may be safely read without mutex.
83 typedef struct BlockCopyTask
{
87 * Fields initialized in block_copy_task_create()
91 BlockCopyCallState
*call_state
;
93 * @method can also be set again in the while loop of
94 * block_copy_dirty_clusters(), but it is never accessed concurrently
95 * because the only other function that reads it is
96 * block_copy_task_entry() and it is invoked afterwards in the same
99 BlockCopyMethod method
;
102 * Generally, req is protected by lock in BlockCopyState, Still req.offset
103 * is only set on task creation, so may be read concurrently after creation.
104 * req.bytes is changed at most once, and need only protecting the case of
105 * parallel read while updating @bytes value in block_copy_task_shrink().
110 static int64_t task_end(BlockCopyTask
*task
)
112 return task
->req
.offset
+ task
->req
.bytes
;
115 typedef struct BlockCopyState
{
117 * BdrvChild objects are not owned or managed by block-copy. They are
118 * provided by block-copy user and user is responsible for appropriate
119 * permissions on these children.
125 * Fields initialized in block_copy_state_new()
128 int64_t cluster_size
;
129 int64_t max_transfer
;
131 BdrvRequestFlags write_flags
;
134 * Fields whose state changes throughout the execution
138 int64_t in_flight_bytes
;
139 BlockCopyMethod method
;
141 QLIST_HEAD(, BlockCopyCallState
) calls
;
145 * Used by sync=top jobs, which first scan the source node for unallocated
146 * areas and clear them in the copy_bitmap. During this process, the bitmap
147 * is thus not fully initialized: It may still have bits set for areas that
148 * are unallocated and should actually not be copied.
150 * This is indicated by skip_unallocated.
152 * In this case, block_copy() will query the source’s allocation status,
153 * skip unallocated regions, clear them in the copy_bitmap, and invoke
154 * block_copy_reset_unallocated() every time it does.
156 bool skip_unallocated
; /* atomic */
157 /* State fields that use a thread-safe API */
158 BdrvDirtyBitmap
*copy_bitmap
;
159 ProgressMeter
*progress
;
161 RateLimit rate_limit
;
164 /* Called with lock held */
165 static int64_t block_copy_chunk_size(BlockCopyState
*s
)
168 case COPY_READ_WRITE_CLUSTER
:
169 return s
->cluster_size
;
170 case COPY_READ_WRITE
:
171 case COPY_RANGE_SMALL
:
172 return MIN(MAX(s
->cluster_size
, BLOCK_COPY_MAX_BUFFER
),
174 case COPY_RANGE_FULL
:
175 return MIN(MAX(s
->cluster_size
, BLOCK_COPY_MAX_COPY_RANGE
),
178 /* Cannot have COPY_WRITE_ZEROES here. */
184 * Search for the first dirty area in offset/bytes range and create task at
185 * the beginning of it.
187 static coroutine_fn BlockCopyTask
*
188 block_copy_task_create(BlockCopyState
*s
, BlockCopyCallState
*call_state
,
189 int64_t offset
, int64_t bytes
)
194 QEMU_LOCK_GUARD(&s
->lock
);
195 max_chunk
= MIN_NON_ZERO(block_copy_chunk_size(s
), call_state
->max_chunk
);
196 if (!bdrv_dirty_bitmap_next_dirty_area(s
->copy_bitmap
,
197 offset
, offset
+ bytes
,
198 max_chunk
, &offset
, &bytes
))
203 assert(QEMU_IS_ALIGNED(offset
, s
->cluster_size
));
204 bytes
= QEMU_ALIGN_UP(bytes
, s
->cluster_size
);
206 /* region is dirty, so no existent tasks possible in it */
207 assert(!reqlist_find_conflict(&s
->reqs
, offset
, bytes
));
209 bdrv_reset_dirty_bitmap(s
->copy_bitmap
, offset
, bytes
);
210 s
->in_flight_bytes
+= bytes
;
212 task
= g_new(BlockCopyTask
, 1);
213 *task
= (BlockCopyTask
) {
214 .task
.func
= block_copy_task_entry
,
216 .call_state
= call_state
,
219 reqlist_init_req(&s
->reqs
, &task
->req
, offset
, bytes
);
225 * block_copy_task_shrink
227 * Drop the tail of the task to be handled later. Set dirty bits back and
228 * wake up all tasks waiting for us (may be some of them are not intersecting
231 static void coroutine_fn
block_copy_task_shrink(BlockCopyTask
*task
,
234 QEMU_LOCK_GUARD(&task
->s
->lock
);
235 if (new_bytes
== task
->req
.bytes
) {
239 assert(new_bytes
> 0 && new_bytes
< task
->req
.bytes
);
241 task
->s
->in_flight_bytes
-= task
->req
.bytes
- new_bytes
;
242 bdrv_set_dirty_bitmap(task
->s
->copy_bitmap
,
243 task
->req
.offset
+ new_bytes
,
244 task
->req
.bytes
- new_bytes
);
246 reqlist_shrink_req(&task
->req
, new_bytes
);
249 static void coroutine_fn
block_copy_task_end(BlockCopyTask
*task
, int ret
)
251 QEMU_LOCK_GUARD(&task
->s
->lock
);
252 task
->s
->in_flight_bytes
-= task
->req
.bytes
;
254 bdrv_set_dirty_bitmap(task
->s
->copy_bitmap
, task
->req
.offset
,
257 if (task
->s
->progress
) {
258 progress_set_remaining(task
->s
->progress
,
259 bdrv_get_dirty_count(task
->s
->copy_bitmap
) +
260 task
->s
->in_flight_bytes
);
262 reqlist_remove_req(&task
->req
);
265 void block_copy_state_free(BlockCopyState
*s
)
271 ratelimit_destroy(&s
->rate_limit
);
272 bdrv_release_dirty_bitmap(s
->copy_bitmap
);
273 shres_destroy(s
->mem
);
277 static uint32_t block_copy_max_transfer(BdrvChild
*source
, BdrvChild
*target
)
279 return MIN_NON_ZERO(INT_MAX
,
280 MIN_NON_ZERO(source
->bs
->bl
.max_transfer
,
281 target
->bs
->bl
.max_transfer
));
284 void block_copy_set_copy_opts(BlockCopyState
*s
, bool use_copy_range
,
287 /* Keep BDRV_REQ_SERIALISING set (or not set) in block_copy_state_new() */
288 s
->write_flags
= (s
->write_flags
& BDRV_REQ_SERIALISING
) |
289 (compress
? BDRV_REQ_WRITE_COMPRESSED
: 0);
291 if (s
->max_transfer
< s
->cluster_size
) {
293 * copy_range does not respect max_transfer. We don't want to bother
294 * with requests smaller than block-copy cluster size, so fallback to
295 * buffered copying (read and write respect max_transfer on their
298 s
->method
= COPY_READ_WRITE_CLUSTER
;
299 } else if (compress
) {
300 /* Compression supports only cluster-size writes and no copy-range. */
301 s
->method
= COPY_READ_WRITE_CLUSTER
;
304 * If copy range enabled, start with COPY_RANGE_SMALL, until first
305 * successful copy_range (look at block_copy_do_copy).
307 s
->method
= use_copy_range
? COPY_RANGE_SMALL
: COPY_READ_WRITE
;
311 static int64_t block_copy_calculate_cluster_size(BlockDriverState
*target
,
316 bool target_does_cow
;
319 GRAPH_RDLOCK_GUARD_MAINLOOP();
321 target_does_cow
= bdrv_backing_chain_next(target
);
324 * If there is no backing file on the target, we cannot rely on COW if our
325 * backup cluster size is smaller than the target cluster size. Even for
326 * targets with a backing file, try to avoid COW if possible.
328 ret
= bdrv_get_info(target
, &bdi
);
329 if (ret
== -ENOTSUP
&& !target_does_cow
) {
330 /* Cluster size is not defined */
331 warn_report("The target block device doesn't provide "
332 "information about the block size and it doesn't have a "
333 "backing file. The default block size of %u bytes is "
334 "used. If the actual block size of the target exceeds "
335 "this default, the backup may be unusable",
336 BLOCK_COPY_CLUSTER_SIZE_DEFAULT
);
337 return BLOCK_COPY_CLUSTER_SIZE_DEFAULT
;
338 } else if (ret
< 0 && !target_does_cow
) {
339 error_setg_errno(errp
, -ret
,
340 "Couldn't determine the cluster size of the target image, "
341 "which has no backing file");
342 error_append_hint(errp
,
343 "Aborting, since this may create an unusable destination image\n");
345 } else if (ret
< 0 && target_does_cow
) {
346 /* Not fatal; just trudge on ahead. */
347 return BLOCK_COPY_CLUSTER_SIZE_DEFAULT
;
350 return MAX(BLOCK_COPY_CLUSTER_SIZE_DEFAULT
, bdi
.cluster_size
);
353 BlockCopyState
*block_copy_state_new(BdrvChild
*source
, BdrvChild
*target
,
354 const BdrvDirtyBitmap
*bitmap
,
359 int64_t cluster_size
;
360 BdrvDirtyBitmap
*copy_bitmap
;
365 cluster_size
= block_copy_calculate_cluster_size(target
->bs
, errp
);
366 if (cluster_size
< 0) {
370 copy_bitmap
= bdrv_create_dirty_bitmap(source
->bs
, cluster_size
, NULL
,
375 bdrv_disable_dirty_bitmap(copy_bitmap
);
377 if (!bdrv_merge_dirty_bitmap(copy_bitmap
, bitmap
, NULL
, errp
)) {
378 error_prepend(errp
, "Failed to merge bitmap '%s' to internal "
379 "copy-bitmap: ", bdrv_dirty_bitmap_name(bitmap
));
380 bdrv_release_dirty_bitmap(copy_bitmap
);
384 bdrv_set_dirty_bitmap(copy_bitmap
, 0,
385 bdrv_dirty_bitmap_size(copy_bitmap
));
389 * If source is in backing chain of target assume that target is going to be
390 * used for "image fleecing", i.e. it should represent a kind of snapshot of
391 * source at backup-start point in time. And target is going to be read by
392 * somebody (for example, used as NBD export) during backup job.
394 * In this case, we need to add BDRV_REQ_SERIALISING write flag to avoid
395 * intersection of backup writes and third party reads from target,
396 * otherwise reading from target we may occasionally read already updated by
399 * For more information see commit f8d59dfb40bb and test
400 * tests/qemu-iotests/222
402 is_fleecing
= bdrv_chain_contains(target
->bs
, source
->bs
);
404 s
= g_new(BlockCopyState
, 1);
405 *s
= (BlockCopyState
) {
408 .copy_bitmap
= copy_bitmap
,
409 .cluster_size
= cluster_size
,
410 .len
= bdrv_dirty_bitmap_size(copy_bitmap
),
411 .write_flags
= (is_fleecing
? BDRV_REQ_SERIALISING
: 0),
412 .mem
= shres_create(BLOCK_COPY_MAX_MEM
),
413 .max_transfer
= QEMU_ALIGN_DOWN(
414 block_copy_max_transfer(source
, target
),
418 block_copy_set_copy_opts(s
, false, false);
420 ratelimit_init(&s
->rate_limit
);
421 qemu_co_mutex_init(&s
->lock
);
422 QLIST_INIT(&s
->reqs
);
423 QLIST_INIT(&s
->calls
);
428 /* Only set before running the job, no need for locking. */
429 void block_copy_set_progress_meter(BlockCopyState
*s
, ProgressMeter
*pm
)
435 * Takes ownership of @task
437 * If pool is NULL directly run the task, otherwise schedule it into the pool.
439 * Returns: task.func return code if pool is NULL
440 * otherwise -ECANCELED if pool status is bad
441 * otherwise 0 (successfully scheduled)
443 static coroutine_fn
int block_copy_task_run(AioTaskPool
*pool
,
447 int ret
= task
->task
.func(&task
->task
);
453 aio_task_pool_wait_slot(pool
);
454 if (aio_task_pool_status(pool
) < 0) {
455 co_put_to_shres(task
->s
->mem
, task
->req
.bytes
);
456 block_copy_task_end(task
, -ECANCELED
);
461 aio_task_pool_start_task(pool
, &task
->task
);
469 * Do copy of cluster-aligned chunk. Requested region is allowed to exceed
470 * s->len only to cover last cluster when s->len is not aligned to clusters.
472 * No sync here: neither bitmap nor intersecting requests handling, only copy.
474 * @method is an in-out argument, so that copy_range can be either extended to
475 * a full-size buffer or disabled if the copy_range attempt fails. The output
476 * value of @method should be used for subsequent tasks.
477 * Returns 0 on success.
479 static int coroutine_fn GRAPH_RDLOCK
480 block_copy_do_copy(BlockCopyState
*s
, int64_t offset
, int64_t bytes
,
481 BlockCopyMethod
*method
, bool *error_is_read
)
484 int64_t nbytes
= MIN(offset
+ bytes
, s
->len
) - offset
;
485 void *bounce_buffer
= NULL
;
487 assert(offset
>= 0 && bytes
> 0 && INT64_MAX
- offset
>= bytes
);
488 assert(QEMU_IS_ALIGNED(offset
, s
->cluster_size
));
489 assert(QEMU_IS_ALIGNED(bytes
, s
->cluster_size
));
490 assert(offset
< s
->len
);
491 assert(offset
+ bytes
<= s
->len
||
492 offset
+ bytes
== QEMU_ALIGN_UP(s
->len
, s
->cluster_size
));
493 assert(nbytes
< INT_MAX
);
496 case COPY_WRITE_ZEROES
:
497 ret
= bdrv_co_pwrite_zeroes(s
->target
, offset
, nbytes
, s
->write_flags
&
498 ~BDRV_REQ_WRITE_COMPRESSED
);
500 trace_block_copy_write_zeroes_fail(s
, offset
, ret
);
501 *error_is_read
= false;
505 case COPY_RANGE_SMALL
:
506 case COPY_RANGE_FULL
:
507 ret
= bdrv_co_copy_range(s
->source
, offset
, s
->target
, offset
, nbytes
,
510 /* Successful copy-range, increase chunk size. */
511 *method
= COPY_RANGE_FULL
;
515 trace_block_copy_copy_range_fail(s
, offset
, ret
);
516 *method
= COPY_READ_WRITE
;
517 /* Fall through to read+write with allocated buffer */
519 case COPY_READ_WRITE_CLUSTER
:
520 case COPY_READ_WRITE
:
522 * In case of failed copy_range request above, we may proceed with
523 * buffered request larger than BLOCK_COPY_MAX_BUFFER.
524 * Still, further requests will be properly limited, so don't care too
525 * much. Moreover the most likely case (copy_range is unsupported for
526 * the configuration, so the very first copy_range request fails)
527 * is handled by setting large copy_size only after first successful
531 bounce_buffer
= qemu_blockalign(s
->source
->bs
, nbytes
);
533 ret
= bdrv_co_pread(s
->source
, offset
, nbytes
, bounce_buffer
, 0);
535 trace_block_copy_read_fail(s
, offset
, ret
);
536 *error_is_read
= true;
540 ret
= bdrv_co_pwrite(s
->target
, offset
, nbytes
, bounce_buffer
,
543 trace_block_copy_write_fail(s
, offset
, ret
);
544 *error_is_read
= false;
549 qemu_vfree(bounce_buffer
);
559 static coroutine_fn
int block_copy_task_entry(AioTask
*task
)
561 BlockCopyTask
*t
= container_of(task
, BlockCopyTask
, task
);
562 BlockCopyState
*s
= t
->s
;
563 bool error_is_read
= false;
564 BlockCopyMethod method
= t
->method
;
567 WITH_GRAPH_RDLOCK_GUARD() {
568 ret
= block_copy_do_copy(s
, t
->req
.offset
, t
->req
.bytes
, &method
,
572 WITH_QEMU_LOCK_GUARD(&s
->lock
) {
573 if (s
->method
== t
->method
) {
578 if (!t
->call_state
->ret
) {
579 t
->call_state
->ret
= ret
;
580 t
->call_state
->error_is_read
= error_is_read
;
582 } else if (s
->progress
) {
583 progress_work_done(s
->progress
, t
->req
.bytes
);
586 co_put_to_shres(s
->mem
, t
->req
.bytes
);
587 block_copy_task_end(t
, ret
);
592 static coroutine_fn GRAPH_RDLOCK
593 int block_copy_block_status(BlockCopyState
*s
, int64_t offset
, int64_t bytes
,
597 BlockDriverState
*base
;
600 if (qatomic_read(&s
->skip_unallocated
)) {
601 base
= bdrv_backing_chain_next(s
->source
->bs
);
606 ret
= bdrv_co_block_status_above(s
->source
->bs
, base
, offset
, bytes
, &num
,
608 if (ret
< 0 || num
< s
->cluster_size
) {
610 * On error or if failed to obtain large enough chunk just fallback to
613 num
= s
->cluster_size
;
614 ret
= BDRV_BLOCK_ALLOCATED
| BDRV_BLOCK_DATA
;
615 } else if (offset
+ num
== s
->len
) {
616 num
= QEMU_ALIGN_UP(num
, s
->cluster_size
);
618 num
= QEMU_ALIGN_DOWN(num
, s
->cluster_size
);
626 * Check if the cluster starting at offset is allocated or not.
627 * return via pnum the number of contiguous clusters sharing this allocation.
629 static int coroutine_fn GRAPH_RDLOCK
630 block_copy_is_cluster_allocated(BlockCopyState
*s
, int64_t offset
,
633 BlockDriverState
*bs
= s
->source
->bs
;
634 int64_t count
, total_count
= 0;
635 int64_t bytes
= s
->len
- offset
;
638 assert(QEMU_IS_ALIGNED(offset
, s
->cluster_size
));
641 /* protected in backup_run() */
642 ret
= bdrv_co_is_allocated(bs
, offset
, bytes
, &count
);
647 total_count
+= count
;
649 if (ret
|| count
== 0) {
651 * ret: partial segment(s) are considered allocated.
652 * otherwise: unallocated tail is treated as an entire segment.
654 *pnum
= DIV_ROUND_UP(total_count
, s
->cluster_size
);
658 /* Unallocated segment(s) with uncertain following segment(s) */
659 if (total_count
>= s
->cluster_size
) {
660 *pnum
= total_count
/ s
->cluster_size
;
669 void block_copy_reset(BlockCopyState
*s
, int64_t offset
, int64_t bytes
)
671 QEMU_LOCK_GUARD(&s
->lock
);
673 bdrv_reset_dirty_bitmap(s
->copy_bitmap
, offset
, bytes
);
675 progress_set_remaining(s
->progress
,
676 bdrv_get_dirty_count(s
->copy_bitmap
) +
682 * Reset bits in copy_bitmap starting at offset if they represent unallocated
683 * data in the image. May reset subsequent contiguous bits.
684 * @return 0 when the cluster at @offset was unallocated,
685 * 1 otherwise, and -ret on error.
687 int64_t coroutine_fn
block_copy_reset_unallocated(BlockCopyState
*s
,
692 int64_t clusters
, bytes
;
694 ret
= block_copy_is_cluster_allocated(s
, offset
, &clusters
);
699 bytes
= clusters
* s
->cluster_size
;
702 block_copy_reset(s
, offset
, bytes
);
710 * block_copy_dirty_clusters
712 * Copy dirty clusters in @offset/@bytes range.
713 * Returns 1 if dirty clusters found and successfully copied, 0 if no dirty
714 * clusters found and -errno on failure.
716 static int coroutine_fn GRAPH_RDLOCK
717 block_copy_dirty_clusters(BlockCopyCallState
*call_state
)
719 BlockCopyState
*s
= call_state
->s
;
720 int64_t offset
= call_state
->offset
;
721 int64_t bytes
= call_state
->bytes
;
724 bool found_dirty
= false;
725 int64_t end
= offset
+ bytes
;
726 AioTaskPool
*aio
= NULL
;
729 * block_copy() user is responsible for keeping source and target in same
732 assert(bdrv_get_aio_context(s
->source
->bs
) ==
733 bdrv_get_aio_context(s
->target
->bs
));
735 assert(QEMU_IS_ALIGNED(offset
, s
->cluster_size
));
736 assert(QEMU_IS_ALIGNED(bytes
, s
->cluster_size
));
738 while (bytes
&& aio_task_pool_status(aio
) == 0 &&
739 !qatomic_read(&call_state
->cancelled
)) {
741 int64_t status_bytes
;
743 task
= block_copy_task_create(s
, call_state
, offset
, bytes
);
745 /* No more dirty bits in the bitmap */
746 trace_block_copy_skip_range(s
, offset
, bytes
);
749 if (task
->req
.offset
> offset
) {
750 trace_block_copy_skip_range(s
, offset
, task
->req
.offset
- offset
);
755 ret
= block_copy_block_status(s
, task
->req
.offset
, task
->req
.bytes
,
757 assert(ret
>= 0); /* never fail */
758 if (status_bytes
< task
->req
.bytes
) {
759 block_copy_task_shrink(task
, status_bytes
);
761 if (qatomic_read(&s
->skip_unallocated
) &&
762 !(ret
& BDRV_BLOCK_ALLOCATED
)) {
763 block_copy_task_end(task
, 0);
764 trace_block_copy_skip_range(s
, task
->req
.offset
, task
->req
.bytes
);
765 offset
= task_end(task
);
766 bytes
= end
- offset
;
770 if (ret
& BDRV_BLOCK_ZERO
) {
771 task
->method
= COPY_WRITE_ZEROES
;
774 if (!call_state
->ignore_ratelimit
) {
775 uint64_t ns
= ratelimit_calculate_delay(&s
->rate_limit
, 0);
777 block_copy_task_end(task
, -EAGAIN
);
779 qemu_co_sleep_ns_wakeable(&call_state
->sleep
,
780 QEMU_CLOCK_REALTIME
, ns
);
785 ratelimit_calculate_delay(&s
->rate_limit
, task
->req
.bytes
);
787 trace_block_copy_process(s
, task
->req
.offset
);
789 co_get_from_shres(s
->mem
, task
->req
.bytes
);
791 offset
= task_end(task
);
792 bytes
= end
- offset
;
795 aio
= aio_task_pool_new(call_state
->max_workers
);
798 ret
= block_copy_task_run(aio
, task
);
806 aio_task_pool_wait_all(aio
);
809 * We are not really interested in -ECANCELED returned from
810 * block_copy_task_run. If it fails, it means some task already failed
811 * for real reason, let's return first failure.
812 * Still, assert that we don't rewrite failure by success.
814 * Note: ret may be positive here because of block-status result.
816 assert(ret
>= 0 || aio_task_pool_status(aio
) < 0);
817 ret
= aio_task_pool_status(aio
);
819 aio_task_pool_free(aio
);
822 return ret
< 0 ? ret
: found_dirty
;
825 void block_copy_kick(BlockCopyCallState
*call_state
)
827 qemu_co_sleep_wake(&call_state
->sleep
);
833 * Copy requested region, accordingly to dirty bitmap.
834 * Collaborate with parallel block_copy requests: if they succeed it will help
835 * us. If they fail, we will retry not-copied regions. So, if we return error,
836 * it means that some I/O operation failed in context of _this_ block_copy call,
837 * not some parallel operation.
839 static int coroutine_fn GRAPH_RDLOCK
840 block_copy_common(BlockCopyCallState
*call_state
)
843 BlockCopyState
*s
= call_state
->s
;
845 qemu_co_mutex_lock(&s
->lock
);
846 QLIST_INSERT_HEAD(&s
->calls
, call_state
, list
);
847 qemu_co_mutex_unlock(&s
->lock
);
850 ret
= block_copy_dirty_clusters(call_state
);
852 if (ret
== 0 && !qatomic_read(&call_state
->cancelled
)) {
853 WITH_QEMU_LOCK_GUARD(&s
->lock
) {
855 * Check that there is no task we still need to
858 ret
= reqlist_wait_one(&s
->reqs
, call_state
->offset
,
859 call_state
->bytes
, &s
->lock
);
862 * No pending tasks, but check again the bitmap in this
863 * same critical section, since a task might have failed
864 * between this and the critical section in
865 * block_copy_dirty_clusters().
867 * reqlist_wait_one return value 0 also means that it
868 * didn't release the lock. So, we are still in the same
869 * critical section, not interrupted by any concurrent
872 ret
= bdrv_dirty_bitmap_next_dirty(s
->copy_bitmap
,
874 call_state
->bytes
) >= 0;
880 * We retry in two cases:
881 * 1. Some progress done
882 * Something was copied, which means that there were yield points
883 * and some new dirty bits may have appeared (due to failed parallel
884 * block-copy requests).
885 * 2. We have waited for some intersecting block-copy request
886 * It may have failed and produced new dirty bits.
888 } while (ret
> 0 && !qatomic_read(&call_state
->cancelled
));
890 qatomic_store_release(&call_state
->finished
, true);
892 if (call_state
->cb
) {
893 call_state
->cb(call_state
->cb_opaque
);
896 qemu_co_mutex_lock(&s
->lock
);
897 QLIST_REMOVE(call_state
, list
);
898 qemu_co_mutex_unlock(&s
->lock
);
903 static void coroutine_fn
block_copy_async_co_entry(void *opaque
)
905 GRAPH_RDLOCK_GUARD();
906 block_copy_common(opaque
);
909 int coroutine_fn
block_copy(BlockCopyState
*s
, int64_t start
, int64_t bytes
,
910 bool ignore_ratelimit
, uint64_t timeout_ns
,
911 BlockCopyAsyncCallbackFunc cb
,
915 BlockCopyCallState
*call_state
= g_new(BlockCopyCallState
, 1);
917 *call_state
= (BlockCopyCallState
) {
921 .ignore_ratelimit
= ignore_ratelimit
,
922 .max_workers
= BLOCK_COPY_MAX_WORKERS
,
924 .cb_opaque
= cb_opaque
,
927 ret
= qemu_co_timeout(block_copy_async_co_entry
, call_state
, timeout_ns
,
930 assert(ret
== -ETIMEDOUT
);
931 block_copy_call_cancel(call_state
);
932 /* call_state will be freed by running coroutine. */
936 ret
= call_state
->ret
;
942 BlockCopyCallState
*block_copy_async(BlockCopyState
*s
,
943 int64_t offset
, int64_t bytes
,
944 int max_workers
, int64_t max_chunk
,
945 BlockCopyAsyncCallbackFunc cb
,
948 BlockCopyCallState
*call_state
= g_new(BlockCopyCallState
, 1);
950 *call_state
= (BlockCopyCallState
) {
954 .max_workers
= max_workers
,
955 .max_chunk
= max_chunk
,
957 .cb_opaque
= cb_opaque
,
959 .co
= qemu_coroutine_create(block_copy_async_co_entry
, call_state
),
962 qemu_coroutine_enter(call_state
->co
);
967 void block_copy_call_free(BlockCopyCallState
*call_state
)
973 assert(qatomic_read(&call_state
->finished
));
977 bool block_copy_call_finished(BlockCopyCallState
*call_state
)
979 return qatomic_read(&call_state
->finished
);
982 bool block_copy_call_succeeded(BlockCopyCallState
*call_state
)
984 return qatomic_load_acquire(&call_state
->finished
) &&
985 !qatomic_read(&call_state
->cancelled
) &&
986 call_state
->ret
== 0;
989 bool block_copy_call_failed(BlockCopyCallState
*call_state
)
991 return qatomic_load_acquire(&call_state
->finished
) &&
992 !qatomic_read(&call_state
->cancelled
) &&
996 bool block_copy_call_cancelled(BlockCopyCallState
*call_state
)
998 return qatomic_read(&call_state
->cancelled
);
1001 int block_copy_call_status(BlockCopyCallState
*call_state
, bool *error_is_read
)
1003 assert(qatomic_load_acquire(&call_state
->finished
));
1004 if (error_is_read
) {
1005 *error_is_read
= call_state
->error_is_read
;
1007 return call_state
->ret
;
1011 * Note that cancelling and finishing are racy.
1012 * User can cancel a block-copy that is already finished.
1014 void block_copy_call_cancel(BlockCopyCallState
*call_state
)
1016 qatomic_set(&call_state
->cancelled
, true);
1017 block_copy_kick(call_state
);
1020 BdrvDirtyBitmap
*block_copy_dirty_bitmap(BlockCopyState
*s
)
1022 return s
->copy_bitmap
;
1025 int64_t block_copy_cluster_size(BlockCopyState
*s
)
1027 return s
->cluster_size
;
1030 void block_copy_set_skip_unallocated(BlockCopyState
*s
, bool skip
)
1032 qatomic_set(&s
->skip_unallocated
, skip
);
1035 void block_copy_set_speed(BlockCopyState
*s
, uint64_t speed
)
1037 ratelimit_set_speed(&s
->rate_limit
, speed
, BLOCK_COPY_SLICE_TIME
);
1040 * Note: it's good to kick all call states from here, but it should be done
1041 * only from a coroutine, to not crash if s->calls list changed while
1042 * entering one call. So for now, the only user of this function kicks its
1043 * only one call_state by hand.