4 * Copyright (C) 2013 Proxmox Server Solutions
7 * Dietmar Maurer (dietmar@proxmox.com)
9 * This work is licensed under the terms of the GNU GPL, version 2 or later.
10 * See the COPYING file in the top-level directory.
14 #include "qemu/osdep.h"
17 #include "block/block.h"
18 #include "block/block_int.h"
19 #include "block/blockjob_int.h"
20 #include "block/block_backup.h"
21 #include "qapi/error.h"
22 #include "qapi/qmp/qerror.h"
23 #include "qemu/ratelimit.h"
24 #include "qemu/cutils.h"
25 #include "sysemu/block-backend.h"
26 #include "qemu/bitmap.h"
27 #include "qemu/error-report.h"
29 #define BACKUP_CLUSTER_SIZE_DEFAULT (1 << 16)
31 typedef struct CowRequest
{
34 QLIST_ENTRY(CowRequest
) list
;
35 CoQueue wait_queue
; /* coroutines blocked on this request */
38 typedef struct BackupBlockJob
{
42 BdrvDirtyBitmap
*sync_bitmap
;
43 BdrvDirtyBitmap
*copy_bitmap
;
45 MirrorSyncMode sync_mode
;
46 BitmapSyncMode bitmap_mode
;
47 BlockdevOnError on_source_error
;
48 BlockdevOnError on_target_error
;
49 CoRwlock flush_rwlock
;
54 NotifierWithReturn before_write
;
55 QLIST_HEAD(, CowRequest
) inflight_reqs
;
58 int64_t copy_range_size
;
60 bool serialize_target_writes
;
61 bool initializing_bitmap
;
64 static const BlockJobDriver backup_job_driver
;
66 /* See if in-flight requests overlap and wait for them to complete */
67 static void coroutine_fn
wait_for_overlapping_requests(BackupBlockJob
*job
,
76 QLIST_FOREACH(req
, &job
->inflight_reqs
, list
) {
77 if (end
> req
->start_byte
&& start
< req
->end_byte
) {
78 qemu_co_queue_wait(&req
->wait_queue
, NULL
);
86 /* Keep track of an in-flight request */
87 static void cow_request_begin(CowRequest
*req
, BackupBlockJob
*job
,
88 int64_t start
, int64_t end
)
90 req
->start_byte
= start
;
92 qemu_co_queue_init(&req
->wait_queue
);
93 QLIST_INSERT_HEAD(&job
->inflight_reqs
, req
, list
);
96 /* Forget about a completed request */
97 static void cow_request_end(CowRequest
*req
)
99 QLIST_REMOVE(req
, list
);
100 qemu_co_queue_restart_all(&req
->wait_queue
);
103 /* Copy range to target with a bounce buffer and return the bytes copied. If
104 * error occurred, return a negative error number */
105 static int coroutine_fn
backup_cow_with_bounce_buffer(BackupBlockJob
*job
,
108 bool is_write_notifier
,
110 void **bounce_buffer
)
113 BlockBackend
*blk
= job
->common
.blk
;
115 int read_flags
= is_write_notifier
? BDRV_REQ_NO_SERIALISING
: 0;
116 int write_flags
= job
->serialize_target_writes
? BDRV_REQ_SERIALISING
: 0;
118 assert(QEMU_IS_ALIGNED(start
, job
->cluster_size
));
119 bdrv_reset_dirty_bitmap(job
->copy_bitmap
, start
, job
->cluster_size
);
120 nbytes
= MIN(job
->cluster_size
, job
->len
- start
);
121 if (!*bounce_buffer
) {
122 *bounce_buffer
= blk_blockalign(blk
, job
->cluster_size
);
125 ret
= blk_co_pread(blk
, start
, nbytes
, *bounce_buffer
, read_flags
);
127 trace_backup_do_cow_read_fail(job
, start
, ret
);
129 *error_is_read
= true;
134 if (buffer_is_zero(*bounce_buffer
, nbytes
)) {
135 ret
= blk_co_pwrite_zeroes(job
->target
, start
,
136 nbytes
, write_flags
| BDRV_REQ_MAY_UNMAP
);
138 ret
= blk_co_pwrite(job
->target
, start
,
139 nbytes
, *bounce_buffer
, write_flags
|
140 (job
->compress
? BDRV_REQ_WRITE_COMPRESSED
: 0));
143 trace_backup_do_cow_write_fail(job
, start
, ret
);
145 *error_is_read
= false;
152 bdrv_set_dirty_bitmap(job
->copy_bitmap
, start
, job
->cluster_size
);
157 /* Copy range to target and return the bytes copied. If error occurred, return a
158 * negative error number. */
159 static int coroutine_fn
backup_cow_with_offload(BackupBlockJob
*job
,
162 bool is_write_notifier
)
166 BlockBackend
*blk
= job
->common
.blk
;
168 int read_flags
= is_write_notifier
? BDRV_REQ_NO_SERIALISING
: 0;
169 int write_flags
= job
->serialize_target_writes
? BDRV_REQ_SERIALISING
: 0;
171 assert(QEMU_IS_ALIGNED(job
->copy_range_size
, job
->cluster_size
));
172 assert(QEMU_IS_ALIGNED(start
, job
->cluster_size
));
173 nbytes
= MIN(job
->copy_range_size
, end
- start
);
174 nr_clusters
= DIV_ROUND_UP(nbytes
, job
->cluster_size
);
175 bdrv_reset_dirty_bitmap(job
->copy_bitmap
, start
,
176 job
->cluster_size
* nr_clusters
);
177 ret
= blk_co_copy_range(blk
, start
, job
->target
, start
, nbytes
,
178 read_flags
, write_flags
);
180 trace_backup_do_cow_copy_range_fail(job
, start
, ret
);
181 bdrv_set_dirty_bitmap(job
->copy_bitmap
, start
,
182 job
->cluster_size
* nr_clusters
);
190 * Check if the cluster starting at offset is allocated or not.
191 * return via pnum the number of contiguous clusters sharing this allocation.
193 static int backup_is_cluster_allocated(BackupBlockJob
*s
, int64_t offset
,
196 BlockDriverState
*bs
= blk_bs(s
->common
.blk
);
197 int64_t count
, total_count
= 0;
198 int64_t bytes
= s
->len
- offset
;
201 assert(QEMU_IS_ALIGNED(offset
, s
->cluster_size
));
204 ret
= bdrv_is_allocated(bs
, offset
, bytes
, &count
);
209 total_count
+= count
;
211 if (ret
|| count
== 0) {
213 * ret: partial segment(s) are considered allocated.
214 * otherwise: unallocated tail is treated as an entire segment.
216 *pnum
= DIV_ROUND_UP(total_count
, s
->cluster_size
);
220 /* Unallocated segment(s) with uncertain following segment(s) */
221 if (total_count
>= s
->cluster_size
) {
222 *pnum
= total_count
/ s
->cluster_size
;
232 * Reset bits in copy_bitmap starting at offset if they represent unallocated
233 * data in the image. May reset subsequent contiguous bits.
234 * @return 0 when the cluster at @offset was unallocated,
235 * 1 otherwise, and -ret on error.
237 static int64_t backup_bitmap_reset_unallocated(BackupBlockJob
*s
,
238 int64_t offset
, int64_t *count
)
241 int64_t clusters
, bytes
, estimate
;
243 ret
= backup_is_cluster_allocated(s
, offset
, &clusters
);
248 bytes
= clusters
* s
->cluster_size
;
251 bdrv_reset_dirty_bitmap(s
->copy_bitmap
, offset
, bytes
);
252 estimate
= bdrv_get_dirty_count(s
->copy_bitmap
);
253 job_progress_set_remaining(&s
->common
.job
, estimate
);
260 static int coroutine_fn
backup_do_cow(BackupBlockJob
*job
,
261 int64_t offset
, uint64_t bytes
,
263 bool is_write_notifier
)
265 CowRequest cow_request
;
267 int64_t start
, end
; /* bytes */
268 void *bounce_buffer
= NULL
;
269 int64_t status_bytes
;
271 qemu_co_rwlock_rdlock(&job
->flush_rwlock
);
273 start
= QEMU_ALIGN_DOWN(offset
, job
->cluster_size
);
274 end
= QEMU_ALIGN_UP(bytes
+ offset
, job
->cluster_size
);
276 trace_backup_do_cow_enter(job
, start
, offset
, bytes
);
278 wait_for_overlapping_requests(job
, start
, end
);
279 cow_request_begin(&cow_request
, job
, start
, end
);
281 while (start
< end
) {
284 if (!bdrv_dirty_bitmap_get(job
->copy_bitmap
, start
)) {
285 trace_backup_do_cow_skip(job
, start
);
286 start
+= job
->cluster_size
;
287 continue; /* already copied */
290 dirty_end
= bdrv_dirty_bitmap_next_zero(job
->copy_bitmap
, start
,
296 if (job
->initializing_bitmap
) {
297 ret
= backup_bitmap_reset_unallocated(job
, start
, &status_bytes
);
299 trace_backup_do_cow_skip_range(job
, start
, status_bytes
);
300 start
+= status_bytes
;
303 /* Clamp to known allocated region */
304 dirty_end
= MIN(dirty_end
, start
+ status_bytes
);
307 trace_backup_do_cow_process(job
, start
);
309 if (job
->use_copy_range
) {
310 ret
= backup_cow_with_offload(job
, start
, dirty_end
,
313 job
->use_copy_range
= false;
316 if (!job
->use_copy_range
) {
317 ret
= backup_cow_with_bounce_buffer(job
, start
, dirty_end
,
319 error_is_read
, &bounce_buffer
);
325 /* Publish progress, guest I/O counts as progress too. Note that the
326 * offset field is an opaque progress value, it is not a disk offset.
329 job
->bytes_read
+= ret
;
330 job_progress_update(&job
->common
.job
, ret
);
335 qemu_vfree(bounce_buffer
);
338 cow_request_end(&cow_request
);
340 trace_backup_do_cow_return(job
, offset
, bytes
, ret
);
342 qemu_co_rwlock_unlock(&job
->flush_rwlock
);
347 static int coroutine_fn
backup_before_write_notify(
348 NotifierWithReturn
*notifier
,
351 BackupBlockJob
*job
= container_of(notifier
, BackupBlockJob
, before_write
);
352 BdrvTrackedRequest
*req
= opaque
;
354 assert(req
->bs
== blk_bs(job
->common
.blk
));
355 assert(QEMU_IS_ALIGNED(req
->offset
, BDRV_SECTOR_SIZE
));
356 assert(QEMU_IS_ALIGNED(req
->bytes
, BDRV_SECTOR_SIZE
));
358 return backup_do_cow(job
, req
->offset
, req
->bytes
, NULL
, true);
361 static void backup_cleanup_sync_bitmap(BackupBlockJob
*job
, int ret
)
364 BlockDriverState
*bs
= blk_bs(job
->common
.blk
);
365 bool sync
= (((ret
== 0) || (job
->bitmap_mode
== BITMAP_SYNC_MODE_ALWAYS
)) \
366 && (job
->bitmap_mode
!= BITMAP_SYNC_MODE_NEVER
));
370 * We succeeded, or we always intended to sync the bitmap.
371 * Delete this bitmap and install the child.
373 bm
= bdrv_dirty_bitmap_abdicate(bs
, job
->sync_bitmap
, NULL
);
376 * We failed, or we never intended to sync the bitmap anyway.
377 * Merge the successor back into the parent, keeping all data.
379 bm
= bdrv_reclaim_dirty_bitmap(bs
, job
->sync_bitmap
, NULL
);
384 if (ret
< 0 && job
->bitmap_mode
== BITMAP_SYNC_MODE_ALWAYS
) {
385 /* If we failed and synced, merge in the bits we didn't copy: */
386 bdrv_dirty_bitmap_merge_internal(bm
, job
->copy_bitmap
,
391 static void backup_commit(Job
*job
)
393 BackupBlockJob
*s
= container_of(job
, BackupBlockJob
, common
.job
);
394 if (s
->sync_bitmap
) {
395 backup_cleanup_sync_bitmap(s
, 0);
399 static void backup_abort(Job
*job
)
401 BackupBlockJob
*s
= container_of(job
, BackupBlockJob
, common
.job
);
402 if (s
->sync_bitmap
) {
403 backup_cleanup_sync_bitmap(s
, -1);
407 static void backup_clean(Job
*job
)
409 BackupBlockJob
*s
= container_of(job
, BackupBlockJob
, common
.job
);
410 BlockDriverState
*bs
= blk_bs(s
->common
.blk
);
412 if (s
->copy_bitmap
) {
413 bdrv_release_dirty_bitmap(bs
, s
->copy_bitmap
);
414 s
->copy_bitmap
= NULL
;
418 blk_unref(s
->target
);
422 void backup_do_checkpoint(BlockJob
*job
, Error
**errp
)
424 BackupBlockJob
*backup_job
= container_of(job
, BackupBlockJob
, common
);
426 assert(block_job_driver(job
) == &backup_job_driver
);
428 if (backup_job
->sync_mode
!= MIRROR_SYNC_MODE_NONE
) {
429 error_setg(errp
, "The backup job only supports block checkpoint in"
434 bdrv_set_dirty_bitmap(backup_job
->copy_bitmap
, 0, backup_job
->len
);
437 static void backup_drain(BlockJob
*job
)
439 BackupBlockJob
*s
= container_of(job
, BackupBlockJob
, common
);
441 /* Need to keep a reference in case blk_drain triggers execution
442 * of backup_complete...
445 BlockBackend
*target
= s
->target
;
452 static BlockErrorAction
backup_error_action(BackupBlockJob
*job
,
453 bool read
, int error
)
456 return block_job_error_action(&job
->common
, job
->on_source_error
,
459 return block_job_error_action(&job
->common
, job
->on_target_error
,
464 static bool coroutine_fn
yield_and_check(BackupBlockJob
*job
)
468 if (job_is_cancelled(&job
->common
.job
)) {
472 /* We need to yield even for delay_ns = 0 so that bdrv_drain_all() can
473 * return. Without a yield, the VM would not reboot. */
474 delay_ns
= block_job_ratelimit_get_delay(&job
->common
, job
->bytes_read
);
476 job_sleep_ns(&job
->common
.job
, delay_ns
);
478 if (job_is_cancelled(&job
->common
.job
)) {
485 static int coroutine_fn
backup_loop(BackupBlockJob
*job
)
489 BdrvDirtyBitmapIter
*bdbi
;
492 bdbi
= bdrv_dirty_iter_new(job
->copy_bitmap
);
493 while ((offset
= bdrv_dirty_iter_next(bdbi
)) != -1) {
495 if (yield_and_check(job
)) {
498 ret
= backup_do_cow(job
, offset
,
499 job
->cluster_size
, &error_is_read
, false);
500 if (ret
< 0 && backup_error_action(job
, error_is_read
, -ret
) ==
501 BLOCK_ERROR_ACTION_REPORT
)
509 bdrv_dirty_iter_free(bdbi
);
513 static void backup_init_copy_bitmap(BackupBlockJob
*job
)
518 if (job
->sync_mode
== MIRROR_SYNC_MODE_BITMAP
) {
519 ret
= bdrv_dirty_bitmap_merge_internal(job
->copy_bitmap
,
524 if (job
->sync_mode
== MIRROR_SYNC_MODE_TOP
) {
526 * We can't hog the coroutine to initialize this thoroughly.
527 * Set a flag and resume work when we are able to yield safely.
529 job
->initializing_bitmap
= true;
531 bdrv_set_dirty_bitmap(job
->copy_bitmap
, 0, job
->len
);
534 estimate
= bdrv_get_dirty_count(job
->copy_bitmap
);
535 job_progress_set_remaining(&job
->common
.job
, estimate
);
538 static int coroutine_fn
backup_run(Job
*job
, Error
**errp
)
540 BackupBlockJob
*s
= container_of(job
, BackupBlockJob
, common
.job
);
541 BlockDriverState
*bs
= blk_bs(s
->common
.blk
);
544 QLIST_INIT(&s
->inflight_reqs
);
545 qemu_co_rwlock_init(&s
->flush_rwlock
);
547 backup_init_copy_bitmap(s
);
549 s
->before_write
.notify
= backup_before_write_notify
;
550 bdrv_add_before_write_notifier(bs
, &s
->before_write
);
552 if (s
->sync_mode
== MIRROR_SYNC_MODE_TOP
) {
556 for (offset
= 0; offset
< s
->len
; ) {
557 if (yield_and_check(s
)) {
562 ret
= backup_bitmap_reset_unallocated(s
, offset
, &count
);
569 s
->initializing_bitmap
= false;
572 if (s
->sync_mode
== MIRROR_SYNC_MODE_NONE
) {
573 /* All bits are set in copy_bitmap to allow any cluster to be copied.
574 * This does not actually require them to be copied. */
575 while (!job_is_cancelled(job
)) {
576 /* Yield until the job is cancelled. We just let our before_write
577 * notify callback service CoW requests. */
581 ret
= backup_loop(s
);
585 notifier_with_return_remove(&s
->before_write
);
587 /* wait until pending backup_do_cow() calls have completed */
588 qemu_co_rwlock_wrlock(&s
->flush_rwlock
);
589 qemu_co_rwlock_unlock(&s
->flush_rwlock
);
594 static const BlockJobDriver backup_job_driver
= {
596 .instance_size
= sizeof(BackupBlockJob
),
597 .job_type
= JOB_TYPE_BACKUP
,
598 .free
= block_job_free
,
599 .user_resume
= block_job_user_resume
,
600 .drain
= block_job_drain
,
602 .commit
= backup_commit
,
603 .abort
= backup_abort
,
604 .clean
= backup_clean
,
606 .drain
= backup_drain
,
609 static int64_t backup_calculate_cluster_size(BlockDriverState
*target
,
616 * If there is no backing file on the target, we cannot rely on COW if our
617 * backup cluster size is smaller than the target cluster size. Even for
618 * targets with a backing file, try to avoid COW if possible.
620 ret
= bdrv_get_info(target
, &bdi
);
621 if (ret
== -ENOTSUP
&& !target
->backing
) {
622 /* Cluster size is not defined */
623 warn_report("The target block device doesn't provide "
624 "information about the block size and it doesn't have a "
625 "backing file. The default block size of %u bytes is "
626 "used. If the actual block size of the target exceeds "
627 "this default, the backup may be unusable",
628 BACKUP_CLUSTER_SIZE_DEFAULT
);
629 return BACKUP_CLUSTER_SIZE_DEFAULT
;
630 } else if (ret
< 0 && !target
->backing
) {
631 error_setg_errno(errp
, -ret
,
632 "Couldn't determine the cluster size of the target image, "
633 "which has no backing file");
634 error_append_hint(errp
,
635 "Aborting, since this may create an unusable destination image\n");
637 } else if (ret
< 0 && target
->backing
) {
638 /* Not fatal; just trudge on ahead. */
639 return BACKUP_CLUSTER_SIZE_DEFAULT
;
642 return MAX(BACKUP_CLUSTER_SIZE_DEFAULT
, bdi
.cluster_size
);
645 BlockJob
*backup_job_create(const char *job_id
, BlockDriverState
*bs
,
646 BlockDriverState
*target
, int64_t speed
,
647 MirrorSyncMode sync_mode
, BdrvDirtyBitmap
*sync_bitmap
,
648 BitmapSyncMode bitmap_mode
,
650 BlockdevOnError on_source_error
,
651 BlockdevOnError on_target_error
,
653 BlockCompletionFunc
*cb
, void *opaque
,
654 JobTxn
*txn
, Error
**errp
)
657 BackupBlockJob
*job
= NULL
;
659 int64_t cluster_size
;
660 BdrvDirtyBitmap
*copy_bitmap
= NULL
;
665 /* QMP interface protects us from these cases */
666 assert(sync_mode
!= MIRROR_SYNC_MODE_INCREMENTAL
);
667 assert(sync_bitmap
|| sync_mode
!= MIRROR_SYNC_MODE_BITMAP
);
670 error_setg(errp
, "Source and target cannot be the same");
674 if (!bdrv_is_inserted(bs
)) {
675 error_setg(errp
, "Device is not inserted: %s",
676 bdrv_get_device_name(bs
));
680 if (!bdrv_is_inserted(target
)) {
681 error_setg(errp
, "Device is not inserted: %s",
682 bdrv_get_device_name(target
));
686 if (compress
&& target
->drv
->bdrv_co_pwritev_compressed
== NULL
) {
687 error_setg(errp
, "Compression is not supported for this drive %s",
688 bdrv_get_device_name(target
));
692 if (bdrv_op_is_blocked(bs
, BLOCK_OP_TYPE_BACKUP_SOURCE
, errp
)) {
696 if (bdrv_op_is_blocked(target
, BLOCK_OP_TYPE_BACKUP_TARGET
, errp
)) {
701 /* If we need to write to this bitmap, check that we can: */
702 if (bitmap_mode
!= BITMAP_SYNC_MODE_NEVER
&&
703 bdrv_dirty_bitmap_check(sync_bitmap
, BDRV_BITMAP_DEFAULT
, errp
)) {
707 /* Create a new bitmap, and freeze/disable this one. */
708 if (bdrv_dirty_bitmap_create_successor(bs
, sync_bitmap
, errp
) < 0) {
713 len
= bdrv_getlength(bs
);
715 error_setg_errno(errp
, -len
, "unable to get length for '%s'",
716 bdrv_get_device_name(bs
));
720 cluster_size
= backup_calculate_cluster_size(target
, errp
);
721 if (cluster_size
< 0) {
725 copy_bitmap
= bdrv_create_dirty_bitmap(bs
, cluster_size
, NULL
, errp
);
729 bdrv_disable_dirty_bitmap(copy_bitmap
);
731 /* job->len is fixed, so we can't allow resize */
732 job
= block_job_create(job_id
, &backup_job_driver
, txn
, bs
,
733 BLK_PERM_CONSISTENT_READ
,
734 BLK_PERM_CONSISTENT_READ
| BLK_PERM_WRITE
|
735 BLK_PERM_WRITE_UNCHANGED
| BLK_PERM_GRAPH_MOD
,
736 speed
, creation_flags
, cb
, opaque
, errp
);
741 /* The target must match the source in size, so no resize here either */
742 job
->target
= blk_new(job
->common
.job
.aio_context
,
744 BLK_PERM_CONSISTENT_READ
| BLK_PERM_WRITE
|
745 BLK_PERM_WRITE_UNCHANGED
| BLK_PERM_GRAPH_MOD
);
746 ret
= blk_insert_bs(job
->target
, target
, errp
);
750 blk_set_disable_request_queuing(job
->target
, true);
752 job
->on_source_error
= on_source_error
;
753 job
->on_target_error
= on_target_error
;
754 job
->sync_mode
= sync_mode
;
755 job
->sync_bitmap
= sync_bitmap
;
756 job
->bitmap_mode
= bitmap_mode
;
757 job
->compress
= compress
;
759 /* Detect image-fleecing (and similar) schemes */
760 job
->serialize_target_writes
= bdrv_chain_contains(target
, bs
);
761 job
->cluster_size
= cluster_size
;
762 job
->copy_bitmap
= copy_bitmap
;
764 job
->use_copy_range
= !compress
; /* compression isn't supported for it */
765 job
->copy_range_size
= MIN_NON_ZERO(blk_get_max_transfer(job
->common
.blk
),
766 blk_get_max_transfer(job
->target
));
767 job
->copy_range_size
= MAX(job
->cluster_size
,
768 QEMU_ALIGN_UP(job
->copy_range_size
,
771 /* Required permissions are already taken with target's blk_new() */
772 block_job_add_bdrv(&job
->common
, "target", target
, 0, BLK_PERM_ALL
,
780 assert(!job
|| !job
->copy_bitmap
);
781 bdrv_release_dirty_bitmap(bs
, copy_bitmap
);
784 bdrv_reclaim_dirty_bitmap(bs
, sync_bitmap
, NULL
);
787 backup_clean(&job
->common
.job
);
788 job_early_fail(&job
->common
.job
);