]> git.proxmox.com Git - mirror_qemu.git/blob - block/io.c
block/snapshot: stress that we fallback to primary child
[mirror_qemu.git] / block / io.c
1 /*
2 * Block layer I/O functions
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 #include "qemu/osdep.h"
26 #include "trace.h"
27 #include "sysemu/block-backend.h"
28 #include "block/aio-wait.h"
29 #include "block/blockjob.h"
30 #include "block/blockjob_int.h"
31 #include "block/block_int.h"
32 #include "block/coroutines.h"
33 #include "block/write-threshold.h"
34 #include "qemu/cutils.h"
35 #include "qemu/memalign.h"
36 #include "qapi/error.h"
37 #include "qemu/error-report.h"
38 #include "qemu/main-loop.h"
39 #include "sysemu/replay.h"
40
41 /* Maximum bounce buffer for copy-on-read and write zeroes, in bytes */
42 #define MAX_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS)
43
44 static void bdrv_parent_cb_resize(BlockDriverState *bs);
45 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
46 int64_t offset, int64_t bytes, BdrvRequestFlags flags);
47
48 static void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore,
49 bool ignore_bds_parents)
50 {
51 BdrvChild *c, *next;
52
53 QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
54 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) {
55 continue;
56 }
57 bdrv_parent_drained_begin_single(c, false);
58 }
59 }
60
61 static void bdrv_parent_drained_end_single_no_poll(BdrvChild *c,
62 int *drained_end_counter)
63 {
64 assert(c->parent_quiesce_counter > 0);
65 c->parent_quiesce_counter--;
66 if (c->klass->drained_end) {
67 c->klass->drained_end(c, drained_end_counter);
68 }
69 }
70
71 void bdrv_parent_drained_end_single(BdrvChild *c)
72 {
73 int drained_end_counter = 0;
74 IO_OR_GS_CODE();
75 bdrv_parent_drained_end_single_no_poll(c, &drained_end_counter);
76 BDRV_POLL_WHILE(c->bs, qatomic_read(&drained_end_counter) > 0);
77 }
78
79 static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
80 bool ignore_bds_parents,
81 int *drained_end_counter)
82 {
83 BdrvChild *c;
84
85 QLIST_FOREACH(c, &bs->parents, next_parent) {
86 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) {
87 continue;
88 }
89 bdrv_parent_drained_end_single_no_poll(c, drained_end_counter);
90 }
91 }
92
93 static bool bdrv_parent_drained_poll_single(BdrvChild *c)
94 {
95 if (c->klass->drained_poll) {
96 return c->klass->drained_poll(c);
97 }
98 return false;
99 }
100
101 static bool bdrv_parent_drained_poll(BlockDriverState *bs, BdrvChild *ignore,
102 bool ignore_bds_parents)
103 {
104 BdrvChild *c, *next;
105 bool busy = false;
106
107 QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
108 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) {
109 continue;
110 }
111 busy |= bdrv_parent_drained_poll_single(c);
112 }
113
114 return busy;
115 }
116
117 void bdrv_parent_drained_begin_single(BdrvChild *c, bool poll)
118 {
119 IO_OR_GS_CODE();
120 c->parent_quiesce_counter++;
121 if (c->klass->drained_begin) {
122 c->klass->drained_begin(c);
123 }
124 if (poll) {
125 BDRV_POLL_WHILE(c->bs, bdrv_parent_drained_poll_single(c));
126 }
127 }
128
129 static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src)
130 {
131 dst->pdiscard_alignment = MAX(dst->pdiscard_alignment,
132 src->pdiscard_alignment);
133 dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer);
134 dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer);
135 dst->max_hw_transfer = MIN_NON_ZERO(dst->max_hw_transfer,
136 src->max_hw_transfer);
137 dst->opt_mem_alignment = MAX(dst->opt_mem_alignment,
138 src->opt_mem_alignment);
139 dst->min_mem_alignment = MAX(dst->min_mem_alignment,
140 src->min_mem_alignment);
141 dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov);
142 dst->max_hw_iov = MIN_NON_ZERO(dst->max_hw_iov, src->max_hw_iov);
143 }
144
145 typedef struct BdrvRefreshLimitsState {
146 BlockDriverState *bs;
147 BlockLimits old_bl;
148 } BdrvRefreshLimitsState;
149
150 static void bdrv_refresh_limits_abort(void *opaque)
151 {
152 BdrvRefreshLimitsState *s = opaque;
153
154 s->bs->bl = s->old_bl;
155 }
156
157 static TransactionActionDrv bdrv_refresh_limits_drv = {
158 .abort = bdrv_refresh_limits_abort,
159 .clean = g_free,
160 };
161
162 /* @tran is allowed to be NULL, in this case no rollback is possible. */
163 void bdrv_refresh_limits(BlockDriverState *bs, Transaction *tran, Error **errp)
164 {
165 ERRP_GUARD();
166 BlockDriver *drv = bs->drv;
167 BdrvChild *c;
168 bool have_limits;
169
170 GLOBAL_STATE_CODE();
171
172 if (tran) {
173 BdrvRefreshLimitsState *s = g_new(BdrvRefreshLimitsState, 1);
174 *s = (BdrvRefreshLimitsState) {
175 .bs = bs,
176 .old_bl = bs->bl,
177 };
178 tran_add(tran, &bdrv_refresh_limits_drv, s);
179 }
180
181 memset(&bs->bl, 0, sizeof(bs->bl));
182
183 if (!drv) {
184 return;
185 }
186
187 /* Default alignment based on whether driver has byte interface */
188 bs->bl.request_alignment = (drv->bdrv_co_preadv ||
189 drv->bdrv_aio_preadv ||
190 drv->bdrv_co_preadv_part) ? 1 : 512;
191
192 /* Take some limits from the children as a default */
193 have_limits = false;
194 QLIST_FOREACH(c, &bs->children, next) {
195 if (c->role & (BDRV_CHILD_DATA | BDRV_CHILD_FILTERED | BDRV_CHILD_COW))
196 {
197 bdrv_merge_limits(&bs->bl, &c->bs->bl);
198 have_limits = true;
199 }
200 }
201
202 if (!have_limits) {
203 bs->bl.min_mem_alignment = 512;
204 bs->bl.opt_mem_alignment = qemu_real_host_page_size();
205
206 /* Safe default since most protocols use readv()/writev()/etc */
207 bs->bl.max_iov = IOV_MAX;
208 }
209
210 /* Then let the driver override it */
211 if (drv->bdrv_refresh_limits) {
212 drv->bdrv_refresh_limits(bs, errp);
213 if (*errp) {
214 return;
215 }
216 }
217
218 if (bs->bl.request_alignment > BDRV_MAX_ALIGNMENT) {
219 error_setg(errp, "Driver requires too large request alignment");
220 }
221 }
222
223 /**
224 * The copy-on-read flag is actually a reference count so multiple users may
225 * use the feature without worrying about clobbering its previous state.
226 * Copy-on-read stays enabled until all users have called to disable it.
227 */
228 void bdrv_enable_copy_on_read(BlockDriverState *bs)
229 {
230 IO_CODE();
231 qatomic_inc(&bs->copy_on_read);
232 }
233
234 void bdrv_disable_copy_on_read(BlockDriverState *bs)
235 {
236 int old = qatomic_fetch_dec(&bs->copy_on_read);
237 IO_CODE();
238 assert(old >= 1);
239 }
240
241 typedef struct {
242 Coroutine *co;
243 BlockDriverState *bs;
244 bool done;
245 bool begin;
246 bool recursive;
247 bool poll;
248 BdrvChild *parent;
249 bool ignore_bds_parents;
250 int *drained_end_counter;
251 } BdrvCoDrainData;
252
253 static void coroutine_fn bdrv_drain_invoke_entry(void *opaque)
254 {
255 BdrvCoDrainData *data = opaque;
256 BlockDriverState *bs = data->bs;
257
258 if (data->begin) {
259 bs->drv->bdrv_co_drain_begin(bs);
260 } else {
261 bs->drv->bdrv_co_drain_end(bs);
262 }
263
264 /* Set data->done and decrement drained_end_counter before bdrv_wakeup() */
265 qatomic_mb_set(&data->done, true);
266 if (!data->begin) {
267 qatomic_dec(data->drained_end_counter);
268 }
269 bdrv_dec_in_flight(bs);
270
271 g_free(data);
272 }
273
274 /* Recursively call BlockDriver.bdrv_co_drain_begin/end callbacks */
275 static void bdrv_drain_invoke(BlockDriverState *bs, bool begin,
276 int *drained_end_counter)
277 {
278 BdrvCoDrainData *data;
279
280 if (!bs->drv || (begin && !bs->drv->bdrv_co_drain_begin) ||
281 (!begin && !bs->drv->bdrv_co_drain_end)) {
282 return;
283 }
284
285 data = g_new(BdrvCoDrainData, 1);
286 *data = (BdrvCoDrainData) {
287 .bs = bs,
288 .done = false,
289 .begin = begin,
290 .drained_end_counter = drained_end_counter,
291 };
292
293 if (!begin) {
294 qatomic_inc(drained_end_counter);
295 }
296
297 /* Make sure the driver callback completes during the polling phase for
298 * drain_begin. */
299 bdrv_inc_in_flight(bs);
300 data->co = qemu_coroutine_create(bdrv_drain_invoke_entry, data);
301 aio_co_schedule(bdrv_get_aio_context(bs), data->co);
302 }
303
304 /* Returns true if BDRV_POLL_WHILE() should go into a blocking aio_poll() */
305 bool bdrv_drain_poll(BlockDriverState *bs, bool recursive,
306 BdrvChild *ignore_parent, bool ignore_bds_parents)
307 {
308 BdrvChild *child, *next;
309 IO_OR_GS_CODE();
310
311 if (bdrv_parent_drained_poll(bs, ignore_parent, ignore_bds_parents)) {
312 return true;
313 }
314
315 if (qatomic_read(&bs->in_flight)) {
316 return true;
317 }
318
319 if (recursive) {
320 assert(!ignore_bds_parents);
321 QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
322 if (bdrv_drain_poll(child->bs, recursive, child, false)) {
323 return true;
324 }
325 }
326 }
327
328 return false;
329 }
330
331 static bool bdrv_drain_poll_top_level(BlockDriverState *bs, bool recursive,
332 BdrvChild *ignore_parent)
333 {
334 return bdrv_drain_poll(bs, recursive, ignore_parent, false);
335 }
336
337 static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
338 BdrvChild *parent, bool ignore_bds_parents,
339 bool poll);
340 static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
341 BdrvChild *parent, bool ignore_bds_parents,
342 int *drained_end_counter);
343
344 static void bdrv_co_drain_bh_cb(void *opaque)
345 {
346 BdrvCoDrainData *data = opaque;
347 Coroutine *co = data->co;
348 BlockDriverState *bs = data->bs;
349
350 if (bs) {
351 AioContext *ctx = bdrv_get_aio_context(bs);
352 aio_context_acquire(ctx);
353 bdrv_dec_in_flight(bs);
354 if (data->begin) {
355 assert(!data->drained_end_counter);
356 bdrv_do_drained_begin(bs, data->recursive, data->parent,
357 data->ignore_bds_parents, data->poll);
358 } else {
359 assert(!data->poll);
360 bdrv_do_drained_end(bs, data->recursive, data->parent,
361 data->ignore_bds_parents,
362 data->drained_end_counter);
363 }
364 aio_context_release(ctx);
365 } else {
366 assert(data->begin);
367 bdrv_drain_all_begin();
368 }
369
370 data->done = true;
371 aio_co_wake(co);
372 }
373
374 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
375 bool begin, bool recursive,
376 BdrvChild *parent,
377 bool ignore_bds_parents,
378 bool poll,
379 int *drained_end_counter)
380 {
381 BdrvCoDrainData data;
382 Coroutine *self = qemu_coroutine_self();
383 AioContext *ctx = bdrv_get_aio_context(bs);
384 AioContext *co_ctx = qemu_coroutine_get_aio_context(self);
385
386 /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
387 * other coroutines run if they were queued by aio_co_enter(). */
388
389 assert(qemu_in_coroutine());
390 data = (BdrvCoDrainData) {
391 .co = self,
392 .bs = bs,
393 .done = false,
394 .begin = begin,
395 .recursive = recursive,
396 .parent = parent,
397 .ignore_bds_parents = ignore_bds_parents,
398 .poll = poll,
399 .drained_end_counter = drained_end_counter,
400 };
401
402 if (bs) {
403 bdrv_inc_in_flight(bs);
404 }
405
406 /*
407 * Temporarily drop the lock across yield or we would get deadlocks.
408 * bdrv_co_drain_bh_cb() reaquires the lock as needed.
409 *
410 * When we yield below, the lock for the current context will be
411 * released, so if this is actually the lock that protects bs, don't drop
412 * it a second time.
413 */
414 if (ctx != co_ctx) {
415 aio_context_release(ctx);
416 }
417 replay_bh_schedule_oneshot_event(ctx, bdrv_co_drain_bh_cb, &data);
418
419 qemu_coroutine_yield();
420 /* If we are resumed from some other event (such as an aio completion or a
421 * timer callback), it is a bug in the caller that should be fixed. */
422 assert(data.done);
423
424 /* Reaquire the AioContext of bs if we dropped it */
425 if (ctx != co_ctx) {
426 aio_context_acquire(ctx);
427 }
428 }
429
430 void bdrv_do_drained_begin_quiesce(BlockDriverState *bs,
431 BdrvChild *parent, bool ignore_bds_parents)
432 {
433 IO_OR_GS_CODE();
434 assert(!qemu_in_coroutine());
435
436 /* Stop things in parent-to-child order */
437 if (qatomic_fetch_inc(&bs->quiesce_counter) == 0) {
438 aio_disable_external(bdrv_get_aio_context(bs));
439 }
440
441 bdrv_parent_drained_begin(bs, parent, ignore_bds_parents);
442 bdrv_drain_invoke(bs, true, NULL);
443 }
444
445 static void bdrv_do_drained_begin(BlockDriverState *bs, bool recursive,
446 BdrvChild *parent, bool ignore_bds_parents,
447 bool poll)
448 {
449 BdrvChild *child, *next;
450
451 if (qemu_in_coroutine()) {
452 bdrv_co_yield_to_drain(bs, true, recursive, parent, ignore_bds_parents,
453 poll, NULL);
454 return;
455 }
456
457 bdrv_do_drained_begin_quiesce(bs, parent, ignore_bds_parents);
458
459 if (recursive) {
460 assert(!ignore_bds_parents);
461 bs->recursive_quiesce_counter++;
462 QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
463 bdrv_do_drained_begin(child->bs, true, child, ignore_bds_parents,
464 false);
465 }
466 }
467
468 /*
469 * Wait for drained requests to finish.
470 *
471 * Calling BDRV_POLL_WHILE() only once for the top-level node is okay: The
472 * call is needed so things in this AioContext can make progress even
473 * though we don't return to the main AioContext loop - this automatically
474 * includes other nodes in the same AioContext and therefore all child
475 * nodes.
476 */
477 if (poll) {
478 assert(!ignore_bds_parents);
479 BDRV_POLL_WHILE(bs, bdrv_drain_poll_top_level(bs, recursive, parent));
480 }
481 }
482
483 void bdrv_drained_begin(BlockDriverState *bs)
484 {
485 IO_OR_GS_CODE();
486 bdrv_do_drained_begin(bs, false, NULL, false, true);
487 }
488
489 void bdrv_subtree_drained_begin(BlockDriverState *bs)
490 {
491 IO_OR_GS_CODE();
492 bdrv_do_drained_begin(bs, true, NULL, false, true);
493 }
494
495 /**
496 * This function does not poll, nor must any of its recursively called
497 * functions. The *drained_end_counter pointee will be incremented
498 * once for every background operation scheduled, and decremented once
499 * the operation settles. Therefore, the pointer must remain valid
500 * until the pointee reaches 0. That implies that whoever sets up the
501 * pointee has to poll until it is 0.
502 *
503 * We use atomic operations to access *drained_end_counter, because
504 * (1) when called from bdrv_set_aio_context_ignore(), the subgraph of
505 * @bs may contain nodes in different AioContexts,
506 * (2) bdrv_drain_all_end() uses the same counter for all nodes,
507 * regardless of which AioContext they are in.
508 */
509 static void bdrv_do_drained_end(BlockDriverState *bs, bool recursive,
510 BdrvChild *parent, bool ignore_bds_parents,
511 int *drained_end_counter)
512 {
513 BdrvChild *child;
514 int old_quiesce_counter;
515
516 assert(drained_end_counter != NULL);
517
518 if (qemu_in_coroutine()) {
519 bdrv_co_yield_to_drain(bs, false, recursive, parent, ignore_bds_parents,
520 false, drained_end_counter);
521 return;
522 }
523 assert(bs->quiesce_counter > 0);
524
525 /* Re-enable things in child-to-parent order */
526 bdrv_drain_invoke(bs, false, drained_end_counter);
527 bdrv_parent_drained_end(bs, parent, ignore_bds_parents,
528 drained_end_counter);
529
530 old_quiesce_counter = qatomic_fetch_dec(&bs->quiesce_counter);
531 if (old_quiesce_counter == 1) {
532 aio_enable_external(bdrv_get_aio_context(bs));
533 }
534
535 if (recursive) {
536 assert(!ignore_bds_parents);
537 bs->recursive_quiesce_counter--;
538 QLIST_FOREACH(child, &bs->children, next) {
539 bdrv_do_drained_end(child->bs, true, child, ignore_bds_parents,
540 drained_end_counter);
541 }
542 }
543 }
544
545 void bdrv_drained_end(BlockDriverState *bs)
546 {
547 int drained_end_counter = 0;
548 IO_OR_GS_CODE();
549 bdrv_do_drained_end(bs, false, NULL, false, &drained_end_counter);
550 BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0);
551 }
552
553 void bdrv_drained_end_no_poll(BlockDriverState *bs, int *drained_end_counter)
554 {
555 IO_CODE();
556 bdrv_do_drained_end(bs, false, NULL, false, drained_end_counter);
557 }
558
559 void bdrv_subtree_drained_end(BlockDriverState *bs)
560 {
561 int drained_end_counter = 0;
562 IO_OR_GS_CODE();
563 bdrv_do_drained_end(bs, true, NULL, false, &drained_end_counter);
564 BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0);
565 }
566
567 void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent)
568 {
569 int i;
570 IO_OR_GS_CODE();
571
572 for (i = 0; i < new_parent->recursive_quiesce_counter; i++) {
573 bdrv_do_drained_begin(child->bs, true, child, false, true);
574 }
575 }
576
577 void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent)
578 {
579 int drained_end_counter = 0;
580 int i;
581 IO_OR_GS_CODE();
582
583 for (i = 0; i < old_parent->recursive_quiesce_counter; i++) {
584 bdrv_do_drained_end(child->bs, true, child, false,
585 &drained_end_counter);
586 }
587
588 BDRV_POLL_WHILE(child->bs, qatomic_read(&drained_end_counter) > 0);
589 }
590
591 void bdrv_drain(BlockDriverState *bs)
592 {
593 IO_OR_GS_CODE();
594 bdrv_drained_begin(bs);
595 bdrv_drained_end(bs);
596 }
597
598 static void bdrv_drain_assert_idle(BlockDriverState *bs)
599 {
600 BdrvChild *child, *next;
601
602 assert(qatomic_read(&bs->in_flight) == 0);
603 QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
604 bdrv_drain_assert_idle(child->bs);
605 }
606 }
607
608 unsigned int bdrv_drain_all_count = 0;
609
610 static bool bdrv_drain_all_poll(void)
611 {
612 BlockDriverState *bs = NULL;
613 bool result = false;
614 GLOBAL_STATE_CODE();
615
616 /* bdrv_drain_poll() can't make changes to the graph and we are holding the
617 * main AioContext lock, so iterating bdrv_next_all_states() is safe. */
618 while ((bs = bdrv_next_all_states(bs))) {
619 AioContext *aio_context = bdrv_get_aio_context(bs);
620 aio_context_acquire(aio_context);
621 result |= bdrv_drain_poll(bs, false, NULL, true);
622 aio_context_release(aio_context);
623 }
624
625 return result;
626 }
627
628 /*
629 * Wait for pending requests to complete across all BlockDriverStates
630 *
631 * This function does not flush data to disk, use bdrv_flush_all() for that
632 * after calling this function.
633 *
634 * This pauses all block jobs and disables external clients. It must
635 * be paired with bdrv_drain_all_end().
636 *
637 * NOTE: no new block jobs or BlockDriverStates can be created between
638 * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls.
639 */
640 void bdrv_drain_all_begin(void)
641 {
642 BlockDriverState *bs = NULL;
643 GLOBAL_STATE_CODE();
644
645 if (qemu_in_coroutine()) {
646 bdrv_co_yield_to_drain(NULL, true, false, NULL, true, true, NULL);
647 return;
648 }
649
650 /*
651 * bdrv queue is managed by record/replay,
652 * waiting for finishing the I/O requests may
653 * be infinite
654 */
655 if (replay_events_enabled()) {
656 return;
657 }
658
659 /* AIO_WAIT_WHILE() with a NULL context can only be called from the main
660 * loop AioContext, so make sure we're in the main context. */
661 assert(qemu_get_current_aio_context() == qemu_get_aio_context());
662 assert(bdrv_drain_all_count < INT_MAX);
663 bdrv_drain_all_count++;
664
665 /* Quiesce all nodes, without polling in-flight requests yet. The graph
666 * cannot change during this loop. */
667 while ((bs = bdrv_next_all_states(bs))) {
668 AioContext *aio_context = bdrv_get_aio_context(bs);
669
670 aio_context_acquire(aio_context);
671 bdrv_do_drained_begin(bs, false, NULL, true, false);
672 aio_context_release(aio_context);
673 }
674
675 /* Now poll the in-flight requests */
676 AIO_WAIT_WHILE(NULL, bdrv_drain_all_poll());
677
678 while ((bs = bdrv_next_all_states(bs))) {
679 bdrv_drain_assert_idle(bs);
680 }
681 }
682
683 void bdrv_drain_all_end_quiesce(BlockDriverState *bs)
684 {
685 int drained_end_counter = 0;
686 GLOBAL_STATE_CODE();
687
688 g_assert(bs->quiesce_counter > 0);
689 g_assert(!bs->refcnt);
690
691 while (bs->quiesce_counter) {
692 bdrv_do_drained_end(bs, false, NULL, true, &drained_end_counter);
693 }
694 BDRV_POLL_WHILE(bs, qatomic_read(&drained_end_counter) > 0);
695 }
696
697 void bdrv_drain_all_end(void)
698 {
699 BlockDriverState *bs = NULL;
700 int drained_end_counter = 0;
701 GLOBAL_STATE_CODE();
702
703 /*
704 * bdrv queue is managed by record/replay,
705 * waiting for finishing the I/O requests may
706 * be endless
707 */
708 if (replay_events_enabled()) {
709 return;
710 }
711
712 while ((bs = bdrv_next_all_states(bs))) {
713 AioContext *aio_context = bdrv_get_aio_context(bs);
714
715 aio_context_acquire(aio_context);
716 bdrv_do_drained_end(bs, false, NULL, true, &drained_end_counter);
717 aio_context_release(aio_context);
718 }
719
720 assert(qemu_get_current_aio_context() == qemu_get_aio_context());
721 AIO_WAIT_WHILE(NULL, qatomic_read(&drained_end_counter) > 0);
722
723 assert(bdrv_drain_all_count > 0);
724 bdrv_drain_all_count--;
725 }
726
727 void bdrv_drain_all(void)
728 {
729 GLOBAL_STATE_CODE();
730 bdrv_drain_all_begin();
731 bdrv_drain_all_end();
732 }
733
734 /**
735 * Remove an active request from the tracked requests list
736 *
737 * This function should be called when a tracked request is completing.
738 */
739 static void coroutine_fn tracked_request_end(BdrvTrackedRequest *req)
740 {
741 if (req->serialising) {
742 qatomic_dec(&req->bs->serialising_in_flight);
743 }
744
745 qemu_co_mutex_lock(&req->bs->reqs_lock);
746 QLIST_REMOVE(req, list);
747 qemu_co_queue_restart_all(&req->wait_queue);
748 qemu_co_mutex_unlock(&req->bs->reqs_lock);
749 }
750
751 /**
752 * Add an active request to the tracked requests list
753 */
754 static void coroutine_fn tracked_request_begin(BdrvTrackedRequest *req,
755 BlockDriverState *bs,
756 int64_t offset,
757 int64_t bytes,
758 enum BdrvTrackedRequestType type)
759 {
760 bdrv_check_request(offset, bytes, &error_abort);
761
762 *req = (BdrvTrackedRequest){
763 .bs = bs,
764 .offset = offset,
765 .bytes = bytes,
766 .type = type,
767 .co = qemu_coroutine_self(),
768 .serialising = false,
769 .overlap_offset = offset,
770 .overlap_bytes = bytes,
771 };
772
773 qemu_co_queue_init(&req->wait_queue);
774
775 qemu_co_mutex_lock(&bs->reqs_lock);
776 QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
777 qemu_co_mutex_unlock(&bs->reqs_lock);
778 }
779
780 static bool tracked_request_overlaps(BdrvTrackedRequest *req,
781 int64_t offset, int64_t bytes)
782 {
783 bdrv_check_request(offset, bytes, &error_abort);
784
785 /* aaaa bbbb */
786 if (offset >= req->overlap_offset + req->overlap_bytes) {
787 return false;
788 }
789 /* bbbb aaaa */
790 if (req->overlap_offset >= offset + bytes) {
791 return false;
792 }
793 return true;
794 }
795
796 /* Called with self->bs->reqs_lock held */
797 static coroutine_fn BdrvTrackedRequest *
798 bdrv_find_conflicting_request(BdrvTrackedRequest *self)
799 {
800 BdrvTrackedRequest *req;
801
802 QLIST_FOREACH(req, &self->bs->tracked_requests, list) {
803 if (req == self || (!req->serialising && !self->serialising)) {
804 continue;
805 }
806 if (tracked_request_overlaps(req, self->overlap_offset,
807 self->overlap_bytes))
808 {
809 /*
810 * Hitting this means there was a reentrant request, for
811 * example, a block driver issuing nested requests. This must
812 * never happen since it means deadlock.
813 */
814 assert(qemu_coroutine_self() != req->co);
815
816 /*
817 * If the request is already (indirectly) waiting for us, or
818 * will wait for us as soon as it wakes up, then just go on
819 * (instead of producing a deadlock in the former case).
820 */
821 if (!req->waiting_for) {
822 return req;
823 }
824 }
825 }
826
827 return NULL;
828 }
829
830 /* Called with self->bs->reqs_lock held */
831 static void coroutine_fn
832 bdrv_wait_serialising_requests_locked(BdrvTrackedRequest *self)
833 {
834 BdrvTrackedRequest *req;
835
836 while ((req = bdrv_find_conflicting_request(self))) {
837 self->waiting_for = req;
838 qemu_co_queue_wait(&req->wait_queue, &self->bs->reqs_lock);
839 self->waiting_for = NULL;
840 }
841 }
842
843 /* Called with req->bs->reqs_lock held */
844 static void tracked_request_set_serialising(BdrvTrackedRequest *req,
845 uint64_t align)
846 {
847 int64_t overlap_offset = req->offset & ~(align - 1);
848 int64_t overlap_bytes =
849 ROUND_UP(req->offset + req->bytes, align) - overlap_offset;
850
851 bdrv_check_request(req->offset, req->bytes, &error_abort);
852
853 if (!req->serialising) {
854 qatomic_inc(&req->bs->serialising_in_flight);
855 req->serialising = true;
856 }
857
858 req->overlap_offset = MIN(req->overlap_offset, overlap_offset);
859 req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes);
860 }
861
862 /**
863 * Return the tracked request on @bs for the current coroutine, or
864 * NULL if there is none.
865 */
866 BdrvTrackedRequest *coroutine_fn bdrv_co_get_self_request(BlockDriverState *bs)
867 {
868 BdrvTrackedRequest *req;
869 Coroutine *self = qemu_coroutine_self();
870 IO_CODE();
871
872 QLIST_FOREACH(req, &bs->tracked_requests, list) {
873 if (req->co == self) {
874 return req;
875 }
876 }
877
878 return NULL;
879 }
880
881 /**
882 * Round a region to cluster boundaries
883 */
884 void bdrv_round_to_clusters(BlockDriverState *bs,
885 int64_t offset, int64_t bytes,
886 int64_t *cluster_offset,
887 int64_t *cluster_bytes)
888 {
889 BlockDriverInfo bdi;
890 IO_CODE();
891 if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
892 *cluster_offset = offset;
893 *cluster_bytes = bytes;
894 } else {
895 int64_t c = bdi.cluster_size;
896 *cluster_offset = QEMU_ALIGN_DOWN(offset, c);
897 *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c);
898 }
899 }
900
901 static int bdrv_get_cluster_size(BlockDriverState *bs)
902 {
903 BlockDriverInfo bdi;
904 int ret;
905
906 ret = bdrv_get_info(bs, &bdi);
907 if (ret < 0 || bdi.cluster_size == 0) {
908 return bs->bl.request_alignment;
909 } else {
910 return bdi.cluster_size;
911 }
912 }
913
914 void bdrv_inc_in_flight(BlockDriverState *bs)
915 {
916 IO_CODE();
917 qatomic_inc(&bs->in_flight);
918 }
919
920 void bdrv_wakeup(BlockDriverState *bs)
921 {
922 IO_CODE();
923 aio_wait_kick();
924 }
925
926 void bdrv_dec_in_flight(BlockDriverState *bs)
927 {
928 IO_CODE();
929 qatomic_dec(&bs->in_flight);
930 bdrv_wakeup(bs);
931 }
932
933 static void coroutine_fn
934 bdrv_wait_serialising_requests(BdrvTrackedRequest *self)
935 {
936 BlockDriverState *bs = self->bs;
937
938 if (!qatomic_read(&bs->serialising_in_flight)) {
939 return;
940 }
941
942 qemu_co_mutex_lock(&bs->reqs_lock);
943 bdrv_wait_serialising_requests_locked(self);
944 qemu_co_mutex_unlock(&bs->reqs_lock);
945 }
946
947 void coroutine_fn bdrv_make_request_serialising(BdrvTrackedRequest *req,
948 uint64_t align)
949 {
950 IO_CODE();
951
952 qemu_co_mutex_lock(&req->bs->reqs_lock);
953
954 tracked_request_set_serialising(req, align);
955 bdrv_wait_serialising_requests_locked(req);
956
957 qemu_co_mutex_unlock(&req->bs->reqs_lock);
958 }
959
960 int bdrv_check_qiov_request(int64_t offset, int64_t bytes,
961 QEMUIOVector *qiov, size_t qiov_offset,
962 Error **errp)
963 {
964 /*
965 * Check generic offset/bytes correctness
966 */
967
968 if (offset < 0) {
969 error_setg(errp, "offset is negative: %" PRIi64, offset);
970 return -EIO;
971 }
972
973 if (bytes < 0) {
974 error_setg(errp, "bytes is negative: %" PRIi64, bytes);
975 return -EIO;
976 }
977
978 if (bytes > BDRV_MAX_LENGTH) {
979 error_setg(errp, "bytes(%" PRIi64 ") exceeds maximum(%" PRIi64 ")",
980 bytes, BDRV_MAX_LENGTH);
981 return -EIO;
982 }
983
984 if (offset > BDRV_MAX_LENGTH) {
985 error_setg(errp, "offset(%" PRIi64 ") exceeds maximum(%" PRIi64 ")",
986 offset, BDRV_MAX_LENGTH);
987 return -EIO;
988 }
989
990 if (offset > BDRV_MAX_LENGTH - bytes) {
991 error_setg(errp, "sum of offset(%" PRIi64 ") and bytes(%" PRIi64 ") "
992 "exceeds maximum(%" PRIi64 ")", offset, bytes,
993 BDRV_MAX_LENGTH);
994 return -EIO;
995 }
996
997 if (!qiov) {
998 return 0;
999 }
1000
1001 /*
1002 * Check qiov and qiov_offset
1003 */
1004
1005 if (qiov_offset > qiov->size) {
1006 error_setg(errp, "qiov_offset(%zu) overflow io vector size(%zu)",
1007 qiov_offset, qiov->size);
1008 return -EIO;
1009 }
1010
1011 if (bytes > qiov->size - qiov_offset) {
1012 error_setg(errp, "bytes(%" PRIi64 ") + qiov_offset(%zu) overflow io "
1013 "vector size(%zu)", bytes, qiov_offset, qiov->size);
1014 return -EIO;
1015 }
1016
1017 return 0;
1018 }
1019
1020 int bdrv_check_request(int64_t offset, int64_t bytes, Error **errp)
1021 {
1022 return bdrv_check_qiov_request(offset, bytes, NULL, 0, errp);
1023 }
1024
1025 static int bdrv_check_request32(int64_t offset, int64_t bytes,
1026 QEMUIOVector *qiov, size_t qiov_offset)
1027 {
1028 int ret = bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, NULL);
1029 if (ret < 0) {
1030 return ret;
1031 }
1032
1033 if (bytes > BDRV_REQUEST_MAX_BYTES) {
1034 return -EIO;
1035 }
1036
1037 return 0;
1038 }
1039
1040 /*
1041 * Completely zero out a block device with the help of bdrv_pwrite_zeroes.
1042 * The operation is sped up by checking the block status and only writing
1043 * zeroes to the device if they currently do not return zeroes. Optional
1044 * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
1045 * BDRV_REQ_FUA).
1046 *
1047 * Returns < 0 on error, 0 on success. For error codes see bdrv_pwrite().
1048 */
1049 int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags)
1050 {
1051 int ret;
1052 int64_t target_size, bytes, offset = 0;
1053 BlockDriverState *bs = child->bs;
1054 IO_CODE();
1055
1056 target_size = bdrv_getlength(bs);
1057 if (target_size < 0) {
1058 return target_size;
1059 }
1060
1061 for (;;) {
1062 bytes = MIN(target_size - offset, BDRV_REQUEST_MAX_BYTES);
1063 if (bytes <= 0) {
1064 return 0;
1065 }
1066 ret = bdrv_block_status(bs, offset, bytes, &bytes, NULL, NULL);
1067 if (ret < 0) {
1068 return ret;
1069 }
1070 if (ret & BDRV_BLOCK_ZERO) {
1071 offset += bytes;
1072 continue;
1073 }
1074 ret = bdrv_pwrite_zeroes(child, offset, bytes, flags);
1075 if (ret < 0) {
1076 return ret;
1077 }
1078 offset += bytes;
1079 }
1080 }
1081
1082 /*
1083 * Writes to the file and ensures that no writes are reordered across this
1084 * request (acts as a barrier)
1085 *
1086 * Returns 0 on success, -errno in error cases.
1087 */
1088 int coroutine_fn bdrv_co_pwrite_sync(BdrvChild *child, int64_t offset,
1089 int64_t bytes, const void *buf,
1090 BdrvRequestFlags flags)
1091 {
1092 int ret;
1093 IO_CODE();
1094
1095 ret = bdrv_co_pwrite(child, offset, bytes, buf, flags);
1096 if (ret < 0) {
1097 return ret;
1098 }
1099
1100 ret = bdrv_co_flush(child->bs);
1101 if (ret < 0) {
1102 return ret;
1103 }
1104
1105 return 0;
1106 }
1107
1108 typedef struct CoroutineIOCompletion {
1109 Coroutine *coroutine;
1110 int ret;
1111 } CoroutineIOCompletion;
1112
1113 static void bdrv_co_io_em_complete(void *opaque, int ret)
1114 {
1115 CoroutineIOCompletion *co = opaque;
1116
1117 co->ret = ret;
1118 aio_co_wake(co->coroutine);
1119 }
1120
1121 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
1122 int64_t offset, int64_t bytes,
1123 QEMUIOVector *qiov,
1124 size_t qiov_offset, int flags)
1125 {
1126 BlockDriver *drv = bs->drv;
1127 int64_t sector_num;
1128 unsigned int nb_sectors;
1129 QEMUIOVector local_qiov;
1130 int ret;
1131
1132 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1133 assert(!(flags & ~BDRV_REQ_MASK));
1134 assert(!(flags & BDRV_REQ_NO_FALLBACK));
1135
1136 if (!drv) {
1137 return -ENOMEDIUM;
1138 }
1139
1140 if (drv->bdrv_co_preadv_part) {
1141 return drv->bdrv_co_preadv_part(bs, offset, bytes, qiov, qiov_offset,
1142 flags);
1143 }
1144
1145 if (qiov_offset > 0 || bytes != qiov->size) {
1146 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1147 qiov = &local_qiov;
1148 }
1149
1150 if (drv->bdrv_co_preadv) {
1151 ret = drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags);
1152 goto out;
1153 }
1154
1155 if (drv->bdrv_aio_preadv) {
1156 BlockAIOCB *acb;
1157 CoroutineIOCompletion co = {
1158 .coroutine = qemu_coroutine_self(),
1159 };
1160
1161 acb = drv->bdrv_aio_preadv(bs, offset, bytes, qiov, flags,
1162 bdrv_co_io_em_complete, &co);
1163 if (acb == NULL) {
1164 ret = -EIO;
1165 goto out;
1166 } else {
1167 qemu_coroutine_yield();
1168 ret = co.ret;
1169 goto out;
1170 }
1171 }
1172
1173 sector_num = offset >> BDRV_SECTOR_BITS;
1174 nb_sectors = bytes >> BDRV_SECTOR_BITS;
1175
1176 assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
1177 assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
1178 assert(bytes <= BDRV_REQUEST_MAX_BYTES);
1179 assert(drv->bdrv_co_readv);
1180
1181 ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
1182
1183 out:
1184 if (qiov == &local_qiov) {
1185 qemu_iovec_destroy(&local_qiov);
1186 }
1187
1188 return ret;
1189 }
1190
1191 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,
1192 int64_t offset, int64_t bytes,
1193 QEMUIOVector *qiov,
1194 size_t qiov_offset,
1195 BdrvRequestFlags flags)
1196 {
1197 BlockDriver *drv = bs->drv;
1198 int64_t sector_num;
1199 unsigned int nb_sectors;
1200 QEMUIOVector local_qiov;
1201 int ret;
1202
1203 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1204 assert(!(flags & ~BDRV_REQ_MASK));
1205 assert(!(flags & BDRV_REQ_NO_FALLBACK));
1206
1207 if (!drv) {
1208 return -ENOMEDIUM;
1209 }
1210
1211 if (drv->bdrv_co_pwritev_part) {
1212 ret = drv->bdrv_co_pwritev_part(bs, offset, bytes, qiov, qiov_offset,
1213 flags & bs->supported_write_flags);
1214 flags &= ~bs->supported_write_flags;
1215 goto emulate_flags;
1216 }
1217
1218 if (qiov_offset > 0 || bytes != qiov->size) {
1219 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1220 qiov = &local_qiov;
1221 }
1222
1223 if (drv->bdrv_co_pwritev) {
1224 ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov,
1225 flags & bs->supported_write_flags);
1226 flags &= ~bs->supported_write_flags;
1227 goto emulate_flags;
1228 }
1229
1230 if (drv->bdrv_aio_pwritev) {
1231 BlockAIOCB *acb;
1232 CoroutineIOCompletion co = {
1233 .coroutine = qemu_coroutine_self(),
1234 };
1235
1236 acb = drv->bdrv_aio_pwritev(bs, offset, bytes, qiov,
1237 flags & bs->supported_write_flags,
1238 bdrv_co_io_em_complete, &co);
1239 flags &= ~bs->supported_write_flags;
1240 if (acb == NULL) {
1241 ret = -EIO;
1242 } else {
1243 qemu_coroutine_yield();
1244 ret = co.ret;
1245 }
1246 goto emulate_flags;
1247 }
1248
1249 sector_num = offset >> BDRV_SECTOR_BITS;
1250 nb_sectors = bytes >> BDRV_SECTOR_BITS;
1251
1252 assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
1253 assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
1254 assert(bytes <= BDRV_REQUEST_MAX_BYTES);
1255
1256 assert(drv->bdrv_co_writev);
1257 ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov,
1258 flags & bs->supported_write_flags);
1259 flags &= ~bs->supported_write_flags;
1260
1261 emulate_flags:
1262 if (ret == 0 && (flags & BDRV_REQ_FUA)) {
1263 ret = bdrv_co_flush(bs);
1264 }
1265
1266 if (qiov == &local_qiov) {
1267 qemu_iovec_destroy(&local_qiov);
1268 }
1269
1270 return ret;
1271 }
1272
1273 static int coroutine_fn
1274 bdrv_driver_pwritev_compressed(BlockDriverState *bs, int64_t offset,
1275 int64_t bytes, QEMUIOVector *qiov,
1276 size_t qiov_offset)
1277 {
1278 BlockDriver *drv = bs->drv;
1279 QEMUIOVector local_qiov;
1280 int ret;
1281
1282 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1283
1284 if (!drv) {
1285 return -ENOMEDIUM;
1286 }
1287
1288 if (!block_driver_can_compress(drv)) {
1289 return -ENOTSUP;
1290 }
1291
1292 if (drv->bdrv_co_pwritev_compressed_part) {
1293 return drv->bdrv_co_pwritev_compressed_part(bs, offset, bytes,
1294 qiov, qiov_offset);
1295 }
1296
1297 if (qiov_offset == 0) {
1298 return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov);
1299 }
1300
1301 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1302 ret = drv->bdrv_co_pwritev_compressed(bs, offset, bytes, &local_qiov);
1303 qemu_iovec_destroy(&local_qiov);
1304
1305 return ret;
1306 }
1307
1308 static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child,
1309 int64_t offset, int64_t bytes, QEMUIOVector *qiov,
1310 size_t qiov_offset, int flags)
1311 {
1312 BlockDriverState *bs = child->bs;
1313
1314 /* Perform I/O through a temporary buffer so that users who scribble over
1315 * their read buffer while the operation is in progress do not end up
1316 * modifying the image file. This is critical for zero-copy guest I/O
1317 * where anything might happen inside guest memory.
1318 */
1319 void *bounce_buffer = NULL;
1320
1321 BlockDriver *drv = bs->drv;
1322 int64_t cluster_offset;
1323 int64_t cluster_bytes;
1324 int64_t skip_bytes;
1325 int ret;
1326 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer,
1327 BDRV_REQUEST_MAX_BYTES);
1328 int64_t progress = 0;
1329 bool skip_write;
1330
1331 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1332
1333 if (!drv) {
1334 return -ENOMEDIUM;
1335 }
1336
1337 /*
1338 * Do not write anything when the BDS is inactive. That is not
1339 * allowed, and it would not help.
1340 */
1341 skip_write = (bs->open_flags & BDRV_O_INACTIVE);
1342
1343 /* FIXME We cannot require callers to have write permissions when all they
1344 * are doing is a read request. If we did things right, write permissions
1345 * would be obtained anyway, but internally by the copy-on-read code. As
1346 * long as it is implemented here rather than in a separate filter driver,
1347 * the copy-on-read code doesn't have its own BdrvChild, however, for which
1348 * it could request permissions. Therefore we have to bypass the permission
1349 * system for the moment. */
1350 // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE));
1351
1352 /* Cover entire cluster so no additional backing file I/O is required when
1353 * allocating cluster in the image file. Note that this value may exceed
1354 * BDRV_REQUEST_MAX_BYTES (even when the original read did not), which
1355 * is one reason we loop rather than doing it all at once.
1356 */
1357 bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes);
1358 skip_bytes = offset - cluster_offset;
1359
1360 trace_bdrv_co_do_copy_on_readv(bs, offset, bytes,
1361 cluster_offset, cluster_bytes);
1362
1363 while (cluster_bytes) {
1364 int64_t pnum;
1365
1366 if (skip_write) {
1367 ret = 1; /* "already allocated", so nothing will be copied */
1368 pnum = MIN(cluster_bytes, max_transfer);
1369 } else {
1370 ret = bdrv_is_allocated(bs, cluster_offset,
1371 MIN(cluster_bytes, max_transfer), &pnum);
1372 if (ret < 0) {
1373 /*
1374 * Safe to treat errors in querying allocation as if
1375 * unallocated; we'll probably fail again soon on the
1376 * read, but at least that will set a decent errno.
1377 */
1378 pnum = MIN(cluster_bytes, max_transfer);
1379 }
1380
1381 /* Stop at EOF if the image ends in the middle of the cluster */
1382 if (ret == 0 && pnum == 0) {
1383 assert(progress >= bytes);
1384 break;
1385 }
1386
1387 assert(skip_bytes < pnum);
1388 }
1389
1390 if (ret <= 0) {
1391 QEMUIOVector local_qiov;
1392
1393 /* Must copy-on-read; use the bounce buffer */
1394 pnum = MIN(pnum, MAX_BOUNCE_BUFFER);
1395 if (!bounce_buffer) {
1396 int64_t max_we_need = MAX(pnum, cluster_bytes - pnum);
1397 int64_t max_allowed = MIN(max_transfer, MAX_BOUNCE_BUFFER);
1398 int64_t bounce_buffer_len = MIN(max_we_need, max_allowed);
1399
1400 bounce_buffer = qemu_try_blockalign(bs, bounce_buffer_len);
1401 if (!bounce_buffer) {
1402 ret = -ENOMEM;
1403 goto err;
1404 }
1405 }
1406 qemu_iovec_init_buf(&local_qiov, bounce_buffer, pnum);
1407
1408 ret = bdrv_driver_preadv(bs, cluster_offset, pnum,
1409 &local_qiov, 0, 0);
1410 if (ret < 0) {
1411 goto err;
1412 }
1413
1414 bdrv_debug_event(bs, BLKDBG_COR_WRITE);
1415 if (drv->bdrv_co_pwrite_zeroes &&
1416 buffer_is_zero(bounce_buffer, pnum)) {
1417 /* FIXME: Should we (perhaps conditionally) be setting
1418 * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy
1419 * that still correctly reads as zero? */
1420 ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, pnum,
1421 BDRV_REQ_WRITE_UNCHANGED);
1422 } else {
1423 /* This does not change the data on the disk, it is not
1424 * necessary to flush even in cache=writethrough mode.
1425 */
1426 ret = bdrv_driver_pwritev(bs, cluster_offset, pnum,
1427 &local_qiov, 0,
1428 BDRV_REQ_WRITE_UNCHANGED);
1429 }
1430
1431 if (ret < 0) {
1432 /* It might be okay to ignore write errors for guest
1433 * requests. If this is a deliberate copy-on-read
1434 * then we don't want to ignore the error. Simply
1435 * report it in all cases.
1436 */
1437 goto err;
1438 }
1439
1440 if (!(flags & BDRV_REQ_PREFETCH)) {
1441 qemu_iovec_from_buf(qiov, qiov_offset + progress,
1442 bounce_buffer + skip_bytes,
1443 MIN(pnum - skip_bytes, bytes - progress));
1444 }
1445 } else if (!(flags & BDRV_REQ_PREFETCH)) {
1446 /* Read directly into the destination */
1447 ret = bdrv_driver_preadv(bs, offset + progress,
1448 MIN(pnum - skip_bytes, bytes - progress),
1449 qiov, qiov_offset + progress, 0);
1450 if (ret < 0) {
1451 goto err;
1452 }
1453 }
1454
1455 cluster_offset += pnum;
1456 cluster_bytes -= pnum;
1457 progress += pnum - skip_bytes;
1458 skip_bytes = 0;
1459 }
1460 ret = 0;
1461
1462 err:
1463 qemu_vfree(bounce_buffer);
1464 return ret;
1465 }
1466
1467 /*
1468 * Forwards an already correctly aligned request to the BlockDriver. This
1469 * handles copy on read, zeroing after EOF, and fragmentation of large
1470 * reads; any other features must be implemented by the caller.
1471 */
1472 static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child,
1473 BdrvTrackedRequest *req, int64_t offset, int64_t bytes,
1474 int64_t align, QEMUIOVector *qiov, size_t qiov_offset, int flags)
1475 {
1476 BlockDriverState *bs = child->bs;
1477 int64_t total_bytes, max_bytes;
1478 int ret = 0;
1479 int64_t bytes_remaining = bytes;
1480 int max_transfer;
1481
1482 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1483 assert(is_power_of_2(align));
1484 assert((offset & (align - 1)) == 0);
1485 assert((bytes & (align - 1)) == 0);
1486 assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1487 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
1488 align);
1489
1490 /* TODO: We would need a per-BDS .supported_read_flags and
1491 * potential fallback support, if we ever implement any read flags
1492 * to pass through to drivers. For now, there aren't any
1493 * passthrough flags. */
1494 assert(!(flags & ~(BDRV_REQ_COPY_ON_READ | BDRV_REQ_PREFETCH)));
1495
1496 /* Handle Copy on Read and associated serialisation */
1497 if (flags & BDRV_REQ_COPY_ON_READ) {
1498 /* If we touch the same cluster it counts as an overlap. This
1499 * guarantees that allocating writes will be serialized and not race
1500 * with each other for the same cluster. For example, in copy-on-read
1501 * it ensures that the CoR read and write operations are atomic and
1502 * guest writes cannot interleave between them. */
1503 bdrv_make_request_serialising(req, bdrv_get_cluster_size(bs));
1504 } else {
1505 bdrv_wait_serialising_requests(req);
1506 }
1507
1508 if (flags & BDRV_REQ_COPY_ON_READ) {
1509 int64_t pnum;
1510
1511 /* The flag BDRV_REQ_COPY_ON_READ has reached its addressee */
1512 flags &= ~BDRV_REQ_COPY_ON_READ;
1513
1514 ret = bdrv_is_allocated(bs, offset, bytes, &pnum);
1515 if (ret < 0) {
1516 goto out;
1517 }
1518
1519 if (!ret || pnum != bytes) {
1520 ret = bdrv_co_do_copy_on_readv(child, offset, bytes,
1521 qiov, qiov_offset, flags);
1522 goto out;
1523 } else if (flags & BDRV_REQ_PREFETCH) {
1524 goto out;
1525 }
1526 }
1527
1528 /* Forward the request to the BlockDriver, possibly fragmenting it */
1529 total_bytes = bdrv_getlength(bs);
1530 if (total_bytes < 0) {
1531 ret = total_bytes;
1532 goto out;
1533 }
1534
1535 assert(!(flags & ~bs->supported_read_flags));
1536
1537 max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align);
1538 if (bytes <= max_bytes && bytes <= max_transfer) {
1539 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, qiov_offset, flags);
1540 goto out;
1541 }
1542
1543 while (bytes_remaining) {
1544 int64_t num;
1545
1546 if (max_bytes) {
1547 num = MIN(bytes_remaining, MIN(max_bytes, max_transfer));
1548 assert(num);
1549
1550 ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining,
1551 num, qiov,
1552 qiov_offset + bytes - bytes_remaining,
1553 flags);
1554 max_bytes -= num;
1555 } else {
1556 num = bytes_remaining;
1557 ret = qemu_iovec_memset(qiov, qiov_offset + bytes - bytes_remaining,
1558 0, bytes_remaining);
1559 }
1560 if (ret < 0) {
1561 goto out;
1562 }
1563 bytes_remaining -= num;
1564 }
1565
1566 out:
1567 return ret < 0 ? ret : 0;
1568 }
1569
1570 /*
1571 * Request padding
1572 *
1573 * |<---- align ----->| |<----- align ---->|
1574 * |<- head ->|<------------- bytes ------------->|<-- tail -->|
1575 * | | | | | |
1576 * -*----------$-------*-------- ... --------*-----$------------*---
1577 * | | | | | |
1578 * | offset | | end |
1579 * ALIGN_DOWN(offset) ALIGN_UP(offset) ALIGN_DOWN(end) ALIGN_UP(end)
1580 * [buf ... ) [tail_buf )
1581 *
1582 * @buf is an aligned allocation needed to store @head and @tail paddings. @head
1583 * is placed at the beginning of @buf and @tail at the @end.
1584 *
1585 * @tail_buf is a pointer to sub-buffer, corresponding to align-sized chunk
1586 * around tail, if tail exists.
1587 *
1588 * @merge_reads is true for small requests,
1589 * if @buf_len == @head + bytes + @tail. In this case it is possible that both
1590 * head and tail exist but @buf_len == align and @tail_buf == @buf.
1591 */
1592 typedef struct BdrvRequestPadding {
1593 uint8_t *buf;
1594 size_t buf_len;
1595 uint8_t *tail_buf;
1596 size_t head;
1597 size_t tail;
1598 bool merge_reads;
1599 QEMUIOVector local_qiov;
1600 } BdrvRequestPadding;
1601
1602 static bool bdrv_init_padding(BlockDriverState *bs,
1603 int64_t offset, int64_t bytes,
1604 BdrvRequestPadding *pad)
1605 {
1606 int64_t align = bs->bl.request_alignment;
1607 int64_t sum;
1608
1609 bdrv_check_request(offset, bytes, &error_abort);
1610 assert(align <= INT_MAX); /* documented in block/block_int.h */
1611 assert(align <= SIZE_MAX / 2); /* so we can allocate the buffer */
1612
1613 memset(pad, 0, sizeof(*pad));
1614
1615 pad->head = offset & (align - 1);
1616 pad->tail = ((offset + bytes) & (align - 1));
1617 if (pad->tail) {
1618 pad->tail = align - pad->tail;
1619 }
1620
1621 if (!pad->head && !pad->tail) {
1622 return false;
1623 }
1624
1625 assert(bytes); /* Nothing good in aligning zero-length requests */
1626
1627 sum = pad->head + bytes + pad->tail;
1628 pad->buf_len = (sum > align && pad->head && pad->tail) ? 2 * align : align;
1629 pad->buf = qemu_blockalign(bs, pad->buf_len);
1630 pad->merge_reads = sum == pad->buf_len;
1631 if (pad->tail) {
1632 pad->tail_buf = pad->buf + pad->buf_len - align;
1633 }
1634
1635 return true;
1636 }
1637
1638 static coroutine_fn int bdrv_padding_rmw_read(BdrvChild *child,
1639 BdrvTrackedRequest *req,
1640 BdrvRequestPadding *pad,
1641 bool zero_middle)
1642 {
1643 QEMUIOVector local_qiov;
1644 BlockDriverState *bs = child->bs;
1645 uint64_t align = bs->bl.request_alignment;
1646 int ret;
1647
1648 assert(req->serialising && pad->buf);
1649
1650 if (pad->head || pad->merge_reads) {
1651 int64_t bytes = pad->merge_reads ? pad->buf_len : align;
1652
1653 qemu_iovec_init_buf(&local_qiov, pad->buf, bytes);
1654
1655 if (pad->head) {
1656 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1657 }
1658 if (pad->merge_reads && pad->tail) {
1659 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1660 }
1661 ret = bdrv_aligned_preadv(child, req, req->overlap_offset, bytes,
1662 align, &local_qiov, 0, 0);
1663 if (ret < 0) {
1664 return ret;
1665 }
1666 if (pad->head) {
1667 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1668 }
1669 if (pad->merge_reads && pad->tail) {
1670 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1671 }
1672
1673 if (pad->merge_reads) {
1674 goto zero_mem;
1675 }
1676 }
1677
1678 if (pad->tail) {
1679 qemu_iovec_init_buf(&local_qiov, pad->tail_buf, align);
1680
1681 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1682 ret = bdrv_aligned_preadv(
1683 child, req,
1684 req->overlap_offset + req->overlap_bytes - align,
1685 align, align, &local_qiov, 0, 0);
1686 if (ret < 0) {
1687 return ret;
1688 }
1689 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1690 }
1691
1692 zero_mem:
1693 if (zero_middle) {
1694 memset(pad->buf + pad->head, 0, pad->buf_len - pad->head - pad->tail);
1695 }
1696
1697 return 0;
1698 }
1699
1700 static void bdrv_padding_destroy(BdrvRequestPadding *pad)
1701 {
1702 if (pad->buf) {
1703 qemu_vfree(pad->buf);
1704 qemu_iovec_destroy(&pad->local_qiov);
1705 }
1706 memset(pad, 0, sizeof(*pad));
1707 }
1708
1709 /*
1710 * bdrv_pad_request
1711 *
1712 * Exchange request parameters with padded request if needed. Don't include RMW
1713 * read of padding, bdrv_padding_rmw_read() should be called separately if
1714 * needed.
1715 *
1716 * Request parameters (@qiov, &qiov_offset, &offset, &bytes) are in-out:
1717 * - on function start they represent original request
1718 * - on failure or when padding is not needed they are unchanged
1719 * - on success when padding is needed they represent padded request
1720 */
1721 static int bdrv_pad_request(BlockDriverState *bs,
1722 QEMUIOVector **qiov, size_t *qiov_offset,
1723 int64_t *offset, int64_t *bytes,
1724 BdrvRequestPadding *pad, bool *padded)
1725 {
1726 int ret;
1727
1728 bdrv_check_qiov_request(*offset, *bytes, *qiov, *qiov_offset, &error_abort);
1729
1730 if (!bdrv_init_padding(bs, *offset, *bytes, pad)) {
1731 if (padded) {
1732 *padded = false;
1733 }
1734 return 0;
1735 }
1736
1737 ret = qemu_iovec_init_extended(&pad->local_qiov, pad->buf, pad->head,
1738 *qiov, *qiov_offset, *bytes,
1739 pad->buf + pad->buf_len - pad->tail,
1740 pad->tail);
1741 if (ret < 0) {
1742 bdrv_padding_destroy(pad);
1743 return ret;
1744 }
1745 *bytes += pad->head + pad->tail;
1746 *offset -= pad->head;
1747 *qiov = &pad->local_qiov;
1748 *qiov_offset = 0;
1749 if (padded) {
1750 *padded = true;
1751 }
1752
1753 return 0;
1754 }
1755
1756 int coroutine_fn bdrv_co_preadv(BdrvChild *child,
1757 int64_t offset, int64_t bytes, QEMUIOVector *qiov,
1758 BdrvRequestFlags flags)
1759 {
1760 IO_CODE();
1761 return bdrv_co_preadv_part(child, offset, bytes, qiov, 0, flags);
1762 }
1763
1764 int coroutine_fn bdrv_co_preadv_part(BdrvChild *child,
1765 int64_t offset, int64_t bytes,
1766 QEMUIOVector *qiov, size_t qiov_offset,
1767 BdrvRequestFlags flags)
1768 {
1769 BlockDriverState *bs = child->bs;
1770 BdrvTrackedRequest req;
1771 BdrvRequestPadding pad;
1772 int ret;
1773 IO_CODE();
1774
1775 trace_bdrv_co_preadv_part(bs, offset, bytes, flags);
1776
1777 if (!bdrv_is_inserted(bs)) {
1778 return -ENOMEDIUM;
1779 }
1780
1781 ret = bdrv_check_request32(offset, bytes, qiov, qiov_offset);
1782 if (ret < 0) {
1783 return ret;
1784 }
1785
1786 if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) {
1787 /*
1788 * Aligning zero request is nonsense. Even if driver has special meaning
1789 * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass
1790 * it to driver due to request_alignment.
1791 *
1792 * Still, no reason to return an error if someone do unaligned
1793 * zero-length read occasionally.
1794 */
1795 return 0;
1796 }
1797
1798 bdrv_inc_in_flight(bs);
1799
1800 /* Don't do copy-on-read if we read data before write operation */
1801 if (qatomic_read(&bs->copy_on_read)) {
1802 flags |= BDRV_REQ_COPY_ON_READ;
1803 }
1804
1805 ret = bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad,
1806 NULL);
1807 if (ret < 0) {
1808 goto fail;
1809 }
1810
1811 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
1812 ret = bdrv_aligned_preadv(child, &req, offset, bytes,
1813 bs->bl.request_alignment,
1814 qiov, qiov_offset, flags);
1815 tracked_request_end(&req);
1816 bdrv_padding_destroy(&pad);
1817
1818 fail:
1819 bdrv_dec_in_flight(bs);
1820
1821 return ret;
1822 }
1823
1824 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
1825 int64_t offset, int64_t bytes, BdrvRequestFlags flags)
1826 {
1827 BlockDriver *drv = bs->drv;
1828 QEMUIOVector qiov;
1829 void *buf = NULL;
1830 int ret = 0;
1831 bool need_flush = false;
1832 int head = 0;
1833 int tail = 0;
1834
1835 int64_t max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes,
1836 INT64_MAX);
1837 int alignment = MAX(bs->bl.pwrite_zeroes_alignment,
1838 bs->bl.request_alignment);
1839 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, MAX_BOUNCE_BUFFER);
1840
1841 bdrv_check_request(offset, bytes, &error_abort);
1842
1843 if (!drv) {
1844 return -ENOMEDIUM;
1845 }
1846
1847 if ((flags & ~bs->supported_zero_flags) & BDRV_REQ_NO_FALLBACK) {
1848 return -ENOTSUP;
1849 }
1850
1851 /* Invalidate the cached block-status data range if this write overlaps */
1852 bdrv_bsc_invalidate_range(bs, offset, bytes);
1853
1854 assert(alignment % bs->bl.request_alignment == 0);
1855 head = offset % alignment;
1856 tail = (offset + bytes) % alignment;
1857 max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment);
1858 assert(max_write_zeroes >= bs->bl.request_alignment);
1859
1860 while (bytes > 0 && !ret) {
1861 int64_t num = bytes;
1862
1863 /* Align request. Block drivers can expect the "bulk" of the request
1864 * to be aligned, and that unaligned requests do not cross cluster
1865 * boundaries.
1866 */
1867 if (head) {
1868 /* Make a small request up to the first aligned sector. For
1869 * convenience, limit this request to max_transfer even if
1870 * we don't need to fall back to writes. */
1871 num = MIN(MIN(bytes, max_transfer), alignment - head);
1872 head = (head + num) % alignment;
1873 assert(num < max_write_zeroes);
1874 } else if (tail && num > alignment) {
1875 /* Shorten the request to the last aligned sector. */
1876 num -= tail;
1877 }
1878
1879 /* limit request size */
1880 if (num > max_write_zeroes) {
1881 num = max_write_zeroes;
1882 }
1883
1884 ret = -ENOTSUP;
1885 /* First try the efficient write zeroes operation */
1886 if (drv->bdrv_co_pwrite_zeroes) {
1887 ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num,
1888 flags & bs->supported_zero_flags);
1889 if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) &&
1890 !(bs->supported_zero_flags & BDRV_REQ_FUA)) {
1891 need_flush = true;
1892 }
1893 } else {
1894 assert(!bs->supported_zero_flags);
1895 }
1896
1897 if (ret == -ENOTSUP && !(flags & BDRV_REQ_NO_FALLBACK)) {
1898 /* Fall back to bounce buffer if write zeroes is unsupported */
1899 BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE;
1900
1901 if ((flags & BDRV_REQ_FUA) &&
1902 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1903 /* No need for bdrv_driver_pwrite() to do a fallback
1904 * flush on each chunk; use just one at the end */
1905 write_flags &= ~BDRV_REQ_FUA;
1906 need_flush = true;
1907 }
1908 num = MIN(num, max_transfer);
1909 if (buf == NULL) {
1910 buf = qemu_try_blockalign0(bs, num);
1911 if (buf == NULL) {
1912 ret = -ENOMEM;
1913 goto fail;
1914 }
1915 }
1916 qemu_iovec_init_buf(&qiov, buf, num);
1917
1918 ret = bdrv_driver_pwritev(bs, offset, num, &qiov, 0, write_flags);
1919
1920 /* Keep bounce buffer around if it is big enough for all
1921 * all future requests.
1922 */
1923 if (num < max_transfer) {
1924 qemu_vfree(buf);
1925 buf = NULL;
1926 }
1927 }
1928
1929 offset += num;
1930 bytes -= num;
1931 }
1932
1933 fail:
1934 if (ret == 0 && need_flush) {
1935 ret = bdrv_co_flush(bs);
1936 }
1937 qemu_vfree(buf);
1938 return ret;
1939 }
1940
1941 static inline int coroutine_fn
1942 bdrv_co_write_req_prepare(BdrvChild *child, int64_t offset, int64_t bytes,
1943 BdrvTrackedRequest *req, int flags)
1944 {
1945 BlockDriverState *bs = child->bs;
1946
1947 bdrv_check_request(offset, bytes, &error_abort);
1948
1949 if (bdrv_is_read_only(bs)) {
1950 return -EPERM;
1951 }
1952
1953 assert(!(bs->open_flags & BDRV_O_INACTIVE));
1954 assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1955 assert(!(flags & ~BDRV_REQ_MASK));
1956 assert(!((flags & BDRV_REQ_NO_WAIT) && !(flags & BDRV_REQ_SERIALISING)));
1957
1958 if (flags & BDRV_REQ_SERIALISING) {
1959 QEMU_LOCK_GUARD(&bs->reqs_lock);
1960
1961 tracked_request_set_serialising(req, bdrv_get_cluster_size(bs));
1962
1963 if ((flags & BDRV_REQ_NO_WAIT) && bdrv_find_conflicting_request(req)) {
1964 return -EBUSY;
1965 }
1966
1967 bdrv_wait_serialising_requests_locked(req);
1968 } else {
1969 bdrv_wait_serialising_requests(req);
1970 }
1971
1972 assert(req->overlap_offset <= offset);
1973 assert(offset + bytes <= req->overlap_offset + req->overlap_bytes);
1974 assert(offset + bytes <= bs->total_sectors * BDRV_SECTOR_SIZE ||
1975 child->perm & BLK_PERM_RESIZE);
1976
1977 switch (req->type) {
1978 case BDRV_TRACKED_WRITE:
1979 case BDRV_TRACKED_DISCARD:
1980 if (flags & BDRV_REQ_WRITE_UNCHANGED) {
1981 assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE));
1982 } else {
1983 assert(child->perm & BLK_PERM_WRITE);
1984 }
1985 bdrv_write_threshold_check_write(bs, offset, bytes);
1986 return 0;
1987 case BDRV_TRACKED_TRUNCATE:
1988 assert(child->perm & BLK_PERM_RESIZE);
1989 return 0;
1990 default:
1991 abort();
1992 }
1993 }
1994
1995 static inline void coroutine_fn
1996 bdrv_co_write_req_finish(BdrvChild *child, int64_t offset, int64_t bytes,
1997 BdrvTrackedRequest *req, int ret)
1998 {
1999 int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
2000 BlockDriverState *bs = child->bs;
2001
2002 bdrv_check_request(offset, bytes, &error_abort);
2003
2004 qatomic_inc(&bs->write_gen);
2005
2006 /*
2007 * Discard cannot extend the image, but in error handling cases, such as
2008 * when reverting a qcow2 cluster allocation, the discarded range can pass
2009 * the end of image file, so we cannot assert about BDRV_TRACKED_DISCARD
2010 * here. Instead, just skip it, since semantically a discard request
2011 * beyond EOF cannot expand the image anyway.
2012 */
2013 if (ret == 0 &&
2014 (req->type == BDRV_TRACKED_TRUNCATE ||
2015 end_sector > bs->total_sectors) &&
2016 req->type != BDRV_TRACKED_DISCARD) {
2017 bs->total_sectors = end_sector;
2018 bdrv_parent_cb_resize(bs);
2019 bdrv_dirty_bitmap_truncate(bs, end_sector << BDRV_SECTOR_BITS);
2020 }
2021 if (req->bytes) {
2022 switch (req->type) {
2023 case BDRV_TRACKED_WRITE:
2024 stat64_max(&bs->wr_highest_offset, offset + bytes);
2025 /* fall through, to set dirty bits */
2026 case BDRV_TRACKED_DISCARD:
2027 bdrv_set_dirty(bs, offset, bytes);
2028 break;
2029 default:
2030 break;
2031 }
2032 }
2033 }
2034
2035 /*
2036 * Forwards an already correctly aligned write request to the BlockDriver,
2037 * after possibly fragmenting it.
2038 */
2039 static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child,
2040 BdrvTrackedRequest *req, int64_t offset, int64_t bytes,
2041 int64_t align, QEMUIOVector *qiov, size_t qiov_offset,
2042 BdrvRequestFlags flags)
2043 {
2044 BlockDriverState *bs = child->bs;
2045 BlockDriver *drv = bs->drv;
2046 int ret;
2047
2048 int64_t bytes_remaining = bytes;
2049 int max_transfer;
2050
2051 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
2052
2053 if (!drv) {
2054 return -ENOMEDIUM;
2055 }
2056
2057 if (bdrv_has_readonly_bitmaps(bs)) {
2058 return -EPERM;
2059 }
2060
2061 assert(is_power_of_2(align));
2062 assert((offset & (align - 1)) == 0);
2063 assert((bytes & (align - 1)) == 0);
2064 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
2065 align);
2066
2067 ret = bdrv_co_write_req_prepare(child, offset, bytes, req, flags);
2068
2069 if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
2070 !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes &&
2071 qemu_iovec_is_zero(qiov, qiov_offset, bytes)) {
2072 flags |= BDRV_REQ_ZERO_WRITE;
2073 if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
2074 flags |= BDRV_REQ_MAY_UNMAP;
2075 }
2076 }
2077
2078 if (ret < 0) {
2079 /* Do nothing, write notifier decided to fail this request */
2080 } else if (flags & BDRV_REQ_ZERO_WRITE) {
2081 bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO);
2082 ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags);
2083 } else if (flags & BDRV_REQ_WRITE_COMPRESSED) {
2084 ret = bdrv_driver_pwritev_compressed(bs, offset, bytes,
2085 qiov, qiov_offset);
2086 } else if (bytes <= max_transfer) {
2087 bdrv_debug_event(bs, BLKDBG_PWRITEV);
2088 ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, qiov_offset, flags);
2089 } else {
2090 bdrv_debug_event(bs, BLKDBG_PWRITEV);
2091 while (bytes_remaining) {
2092 int num = MIN(bytes_remaining, max_transfer);
2093 int local_flags = flags;
2094
2095 assert(num);
2096 if (num < bytes_remaining && (flags & BDRV_REQ_FUA) &&
2097 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
2098 /* If FUA is going to be emulated by flush, we only
2099 * need to flush on the last iteration */
2100 local_flags &= ~BDRV_REQ_FUA;
2101 }
2102
2103 ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining,
2104 num, qiov,
2105 qiov_offset + bytes - bytes_remaining,
2106 local_flags);
2107 if (ret < 0) {
2108 break;
2109 }
2110 bytes_remaining -= num;
2111 }
2112 }
2113 bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
2114
2115 if (ret >= 0) {
2116 ret = 0;
2117 }
2118 bdrv_co_write_req_finish(child, offset, bytes, req, ret);
2119
2120 return ret;
2121 }
2122
2123 static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child,
2124 int64_t offset,
2125 int64_t bytes,
2126 BdrvRequestFlags flags,
2127 BdrvTrackedRequest *req)
2128 {
2129 BlockDriverState *bs = child->bs;
2130 QEMUIOVector local_qiov;
2131 uint64_t align = bs->bl.request_alignment;
2132 int ret = 0;
2133 bool padding;
2134 BdrvRequestPadding pad;
2135
2136 padding = bdrv_init_padding(bs, offset, bytes, &pad);
2137 if (padding) {
2138 assert(!(flags & BDRV_REQ_NO_WAIT));
2139 bdrv_make_request_serialising(req, align);
2140
2141 bdrv_padding_rmw_read(child, req, &pad, true);
2142
2143 if (pad.head || pad.merge_reads) {
2144 int64_t aligned_offset = offset & ~(align - 1);
2145 int64_t write_bytes = pad.merge_reads ? pad.buf_len : align;
2146
2147 qemu_iovec_init_buf(&local_qiov, pad.buf, write_bytes);
2148 ret = bdrv_aligned_pwritev(child, req, aligned_offset, write_bytes,
2149 align, &local_qiov, 0,
2150 flags & ~BDRV_REQ_ZERO_WRITE);
2151 if (ret < 0 || pad.merge_reads) {
2152 /* Error or all work is done */
2153 goto out;
2154 }
2155 offset += write_bytes - pad.head;
2156 bytes -= write_bytes - pad.head;
2157 }
2158 }
2159
2160 assert(!bytes || (offset & (align - 1)) == 0);
2161 if (bytes >= align) {
2162 /* Write the aligned part in the middle. */
2163 int64_t aligned_bytes = bytes & ~(align - 1);
2164 ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align,
2165 NULL, 0, flags);
2166 if (ret < 0) {
2167 goto out;
2168 }
2169 bytes -= aligned_bytes;
2170 offset += aligned_bytes;
2171 }
2172
2173 assert(!bytes || (offset & (align - 1)) == 0);
2174 if (bytes) {
2175 assert(align == pad.tail + bytes);
2176
2177 qemu_iovec_init_buf(&local_qiov, pad.tail_buf, align);
2178 ret = bdrv_aligned_pwritev(child, req, offset, align, align,
2179 &local_qiov, 0,
2180 flags & ~BDRV_REQ_ZERO_WRITE);
2181 }
2182
2183 out:
2184 bdrv_padding_destroy(&pad);
2185
2186 return ret;
2187 }
2188
2189 /*
2190 * Handle a write request in coroutine context
2191 */
2192 int coroutine_fn bdrv_co_pwritev(BdrvChild *child,
2193 int64_t offset, int64_t bytes, QEMUIOVector *qiov,
2194 BdrvRequestFlags flags)
2195 {
2196 IO_CODE();
2197 return bdrv_co_pwritev_part(child, offset, bytes, qiov, 0, flags);
2198 }
2199
2200 int coroutine_fn bdrv_co_pwritev_part(BdrvChild *child,
2201 int64_t offset, int64_t bytes, QEMUIOVector *qiov, size_t qiov_offset,
2202 BdrvRequestFlags flags)
2203 {
2204 BlockDriverState *bs = child->bs;
2205 BdrvTrackedRequest req;
2206 uint64_t align = bs->bl.request_alignment;
2207 BdrvRequestPadding pad;
2208 int ret;
2209 bool padded = false;
2210 IO_CODE();
2211
2212 trace_bdrv_co_pwritev_part(child->bs, offset, bytes, flags);
2213
2214 if (!bdrv_is_inserted(bs)) {
2215 return -ENOMEDIUM;
2216 }
2217
2218 if (flags & BDRV_REQ_ZERO_WRITE) {
2219 ret = bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, NULL);
2220 } else {
2221 ret = bdrv_check_request32(offset, bytes, qiov, qiov_offset);
2222 }
2223 if (ret < 0) {
2224 return ret;
2225 }
2226
2227 /* If the request is misaligned then we can't make it efficient */
2228 if ((flags & BDRV_REQ_NO_FALLBACK) &&
2229 !QEMU_IS_ALIGNED(offset | bytes, align))
2230 {
2231 return -ENOTSUP;
2232 }
2233
2234 if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) {
2235 /*
2236 * Aligning zero request is nonsense. Even if driver has special meaning
2237 * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass
2238 * it to driver due to request_alignment.
2239 *
2240 * Still, no reason to return an error if someone do unaligned
2241 * zero-length write occasionally.
2242 */
2243 return 0;
2244 }
2245
2246 if (!(flags & BDRV_REQ_ZERO_WRITE)) {
2247 /*
2248 * Pad request for following read-modify-write cycle.
2249 * bdrv_co_do_zero_pwritev() does aligning by itself, so, we do
2250 * alignment only if there is no ZERO flag.
2251 */
2252 ret = bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad,
2253 &padded);
2254 if (ret < 0) {
2255 return ret;
2256 }
2257 }
2258
2259 bdrv_inc_in_flight(bs);
2260 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
2261
2262 if (flags & BDRV_REQ_ZERO_WRITE) {
2263 assert(!padded);
2264 ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req);
2265 goto out;
2266 }
2267
2268 if (padded) {
2269 /*
2270 * Request was unaligned to request_alignment and therefore
2271 * padded. We are going to do read-modify-write, and must
2272 * serialize the request to prevent interactions of the
2273 * widened region with other transactions.
2274 */
2275 assert(!(flags & BDRV_REQ_NO_WAIT));
2276 bdrv_make_request_serialising(&req, align);
2277 bdrv_padding_rmw_read(child, &req, &pad, false);
2278 }
2279
2280 ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align,
2281 qiov, qiov_offset, flags);
2282
2283 bdrv_padding_destroy(&pad);
2284
2285 out:
2286 tracked_request_end(&req);
2287 bdrv_dec_in_flight(bs);
2288
2289 return ret;
2290 }
2291
2292 int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset,
2293 int64_t bytes, BdrvRequestFlags flags)
2294 {
2295 IO_CODE();
2296 trace_bdrv_co_pwrite_zeroes(child->bs, offset, bytes, flags);
2297
2298 if (!(child->bs->open_flags & BDRV_O_UNMAP)) {
2299 flags &= ~BDRV_REQ_MAY_UNMAP;
2300 }
2301
2302 return bdrv_co_pwritev(child, offset, bytes, NULL,
2303 BDRV_REQ_ZERO_WRITE | flags);
2304 }
2305
2306 /*
2307 * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not.
2308 */
2309 int bdrv_flush_all(void)
2310 {
2311 BdrvNextIterator it;
2312 BlockDriverState *bs = NULL;
2313 int result = 0;
2314
2315 GLOBAL_STATE_CODE();
2316
2317 /*
2318 * bdrv queue is managed by record/replay,
2319 * creating new flush request for stopping
2320 * the VM may break the determinism
2321 */
2322 if (replay_events_enabled()) {
2323 return result;
2324 }
2325
2326 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
2327 AioContext *aio_context = bdrv_get_aio_context(bs);
2328 int ret;
2329
2330 aio_context_acquire(aio_context);
2331 ret = bdrv_flush(bs);
2332 if (ret < 0 && !result) {
2333 result = ret;
2334 }
2335 aio_context_release(aio_context);
2336 }
2337
2338 return result;
2339 }
2340
2341 /*
2342 * Returns the allocation status of the specified sectors.
2343 * Drivers not implementing the functionality are assumed to not support
2344 * backing files, hence all their sectors are reported as allocated.
2345 *
2346 * If 'want_zero' is true, the caller is querying for mapping
2347 * purposes, with a focus on valid BDRV_BLOCK_OFFSET_VALID, _DATA, and
2348 * _ZERO where possible; otherwise, the result favors larger 'pnum',
2349 * with a focus on accurate BDRV_BLOCK_ALLOCATED.
2350 *
2351 * If 'offset' is beyond the end of the disk image the return value is
2352 * BDRV_BLOCK_EOF and 'pnum' is set to 0.
2353 *
2354 * 'bytes' is the max value 'pnum' should be set to. If bytes goes
2355 * beyond the end of the disk image it will be clamped; if 'pnum' is set to
2356 * the end of the image, then the returned value will include BDRV_BLOCK_EOF.
2357 *
2358 * 'pnum' is set to the number of bytes (including and immediately
2359 * following the specified offset) that are easily known to be in the
2360 * same allocated/unallocated state. Note that a second call starting
2361 * at the original offset plus returned pnum may have the same status.
2362 * The returned value is non-zero on success except at end-of-file.
2363 *
2364 * Returns negative errno on failure. Otherwise, if the
2365 * BDRV_BLOCK_OFFSET_VALID bit is set, 'map' and 'file' (if non-NULL) are
2366 * set to the host mapping and BDS corresponding to the guest offset.
2367 */
2368 static int coroutine_fn bdrv_co_block_status(BlockDriverState *bs,
2369 bool want_zero,
2370 int64_t offset, int64_t bytes,
2371 int64_t *pnum, int64_t *map,
2372 BlockDriverState **file)
2373 {
2374 int64_t total_size;
2375 int64_t n; /* bytes */
2376 int ret;
2377 int64_t local_map = 0;
2378 BlockDriverState *local_file = NULL;
2379 int64_t aligned_offset, aligned_bytes;
2380 uint32_t align;
2381 bool has_filtered_child;
2382
2383 assert(pnum);
2384 *pnum = 0;
2385 total_size = bdrv_getlength(bs);
2386 if (total_size < 0) {
2387 ret = total_size;
2388 goto early_out;
2389 }
2390
2391 if (offset >= total_size) {
2392 ret = BDRV_BLOCK_EOF;
2393 goto early_out;
2394 }
2395 if (!bytes) {
2396 ret = 0;
2397 goto early_out;
2398 }
2399
2400 n = total_size - offset;
2401 if (n < bytes) {
2402 bytes = n;
2403 }
2404
2405 /* Must be non-NULL or bdrv_getlength() would have failed */
2406 assert(bs->drv);
2407 has_filtered_child = bdrv_filter_child(bs);
2408 if (!bs->drv->bdrv_co_block_status && !has_filtered_child) {
2409 *pnum = bytes;
2410 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
2411 if (offset + bytes == total_size) {
2412 ret |= BDRV_BLOCK_EOF;
2413 }
2414 if (bs->drv->protocol_name) {
2415 ret |= BDRV_BLOCK_OFFSET_VALID;
2416 local_map = offset;
2417 local_file = bs;
2418 }
2419 goto early_out;
2420 }
2421
2422 bdrv_inc_in_flight(bs);
2423
2424 /* Round out to request_alignment boundaries */
2425 align = bs->bl.request_alignment;
2426 aligned_offset = QEMU_ALIGN_DOWN(offset, align);
2427 aligned_bytes = ROUND_UP(offset + bytes, align) - aligned_offset;
2428
2429 if (bs->drv->bdrv_co_block_status) {
2430 /*
2431 * Use the block-status cache only for protocol nodes: Format
2432 * drivers are generally quick to inquire the status, but protocol
2433 * drivers often need to get information from outside of qemu, so
2434 * we do not have control over the actual implementation. There
2435 * have been cases where inquiring the status took an unreasonably
2436 * long time, and we can do nothing in qemu to fix it.
2437 * This is especially problematic for images with large data areas,
2438 * because finding the few holes in them and giving them special
2439 * treatment does not gain much performance. Therefore, we try to
2440 * cache the last-identified data region.
2441 *
2442 * Second, limiting ourselves to protocol nodes allows us to assume
2443 * the block status for data regions to be DATA | OFFSET_VALID, and
2444 * that the host offset is the same as the guest offset.
2445 *
2446 * Note that it is possible that external writers zero parts of
2447 * the cached regions without the cache being invalidated, and so
2448 * we may report zeroes as data. This is not catastrophic,
2449 * however, because reporting zeroes as data is fine.
2450 */
2451 if (QLIST_EMPTY(&bs->children) &&
2452 bdrv_bsc_is_data(bs, aligned_offset, pnum))
2453 {
2454 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID;
2455 local_file = bs;
2456 local_map = aligned_offset;
2457 } else {
2458 ret = bs->drv->bdrv_co_block_status(bs, want_zero, aligned_offset,
2459 aligned_bytes, pnum, &local_map,
2460 &local_file);
2461
2462 /*
2463 * Note that checking QLIST_EMPTY(&bs->children) is also done when
2464 * the cache is queried above. Technically, we do not need to check
2465 * it here; the worst that can happen is that we fill the cache for
2466 * non-protocol nodes, and then it is never used. However, filling
2467 * the cache requires an RCU update, so double check here to avoid
2468 * such an update if possible.
2469 *
2470 * Check want_zero, because we only want to update the cache when we
2471 * have accurate information about what is zero and what is data.
2472 */
2473 if (want_zero &&
2474 ret == (BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID) &&
2475 QLIST_EMPTY(&bs->children))
2476 {
2477 /*
2478 * When a protocol driver reports BLOCK_OFFSET_VALID, the
2479 * returned local_map value must be the same as the offset we
2480 * have passed (aligned_offset), and local_bs must be the node
2481 * itself.
2482 * Assert this, because we follow this rule when reading from
2483 * the cache (see the `local_file = bs` and
2484 * `local_map = aligned_offset` assignments above), and the
2485 * result the cache delivers must be the same as the driver
2486 * would deliver.
2487 */
2488 assert(local_file == bs);
2489 assert(local_map == aligned_offset);
2490 bdrv_bsc_fill(bs, aligned_offset, *pnum);
2491 }
2492 }
2493 } else {
2494 /* Default code for filters */
2495
2496 local_file = bdrv_filter_bs(bs);
2497 assert(local_file);
2498
2499 *pnum = aligned_bytes;
2500 local_map = aligned_offset;
2501 ret = BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID;
2502 }
2503 if (ret < 0) {
2504 *pnum = 0;
2505 goto out;
2506 }
2507
2508 /*
2509 * The driver's result must be a non-zero multiple of request_alignment.
2510 * Clamp pnum and adjust map to original request.
2511 */
2512 assert(*pnum && QEMU_IS_ALIGNED(*pnum, align) &&
2513 align > offset - aligned_offset);
2514 if (ret & BDRV_BLOCK_RECURSE) {
2515 assert(ret & BDRV_BLOCK_DATA);
2516 assert(ret & BDRV_BLOCK_OFFSET_VALID);
2517 assert(!(ret & BDRV_BLOCK_ZERO));
2518 }
2519
2520 *pnum -= offset - aligned_offset;
2521 if (*pnum > bytes) {
2522 *pnum = bytes;
2523 }
2524 if (ret & BDRV_BLOCK_OFFSET_VALID) {
2525 local_map += offset - aligned_offset;
2526 }
2527
2528 if (ret & BDRV_BLOCK_RAW) {
2529 assert(ret & BDRV_BLOCK_OFFSET_VALID && local_file);
2530 ret = bdrv_co_block_status(local_file, want_zero, local_map,
2531 *pnum, pnum, &local_map, &local_file);
2532 goto out;
2533 }
2534
2535 if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
2536 ret |= BDRV_BLOCK_ALLOCATED;
2537 } else if (bs->drv->supports_backing) {
2538 BlockDriverState *cow_bs = bdrv_cow_bs(bs);
2539
2540 if (!cow_bs) {
2541 ret |= BDRV_BLOCK_ZERO;
2542 } else if (want_zero) {
2543 int64_t size2 = bdrv_getlength(cow_bs);
2544
2545 if (size2 >= 0 && offset >= size2) {
2546 ret |= BDRV_BLOCK_ZERO;
2547 }
2548 }
2549 }
2550
2551 if (want_zero && ret & BDRV_BLOCK_RECURSE &&
2552 local_file && local_file != bs &&
2553 (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
2554 (ret & BDRV_BLOCK_OFFSET_VALID)) {
2555 int64_t file_pnum;
2556 int ret2;
2557
2558 ret2 = bdrv_co_block_status(local_file, want_zero, local_map,
2559 *pnum, &file_pnum, NULL, NULL);
2560 if (ret2 >= 0) {
2561 /* Ignore errors. This is just providing extra information, it
2562 * is useful but not necessary.
2563 */
2564 if (ret2 & BDRV_BLOCK_EOF &&
2565 (!file_pnum || ret2 & BDRV_BLOCK_ZERO)) {
2566 /*
2567 * It is valid for the format block driver to read
2568 * beyond the end of the underlying file's current
2569 * size; such areas read as zero.
2570 */
2571 ret |= BDRV_BLOCK_ZERO;
2572 } else {
2573 /* Limit request to the range reported by the protocol driver */
2574 *pnum = file_pnum;
2575 ret |= (ret2 & BDRV_BLOCK_ZERO);
2576 }
2577 }
2578 }
2579
2580 out:
2581 bdrv_dec_in_flight(bs);
2582 if (ret >= 0 && offset + *pnum == total_size) {
2583 ret |= BDRV_BLOCK_EOF;
2584 }
2585 early_out:
2586 if (file) {
2587 *file = local_file;
2588 }
2589 if (map) {
2590 *map = local_map;
2591 }
2592 return ret;
2593 }
2594
2595 int coroutine_fn
2596 bdrv_co_common_block_status_above(BlockDriverState *bs,
2597 BlockDriverState *base,
2598 bool include_base,
2599 bool want_zero,
2600 int64_t offset,
2601 int64_t bytes,
2602 int64_t *pnum,
2603 int64_t *map,
2604 BlockDriverState **file,
2605 int *depth)
2606 {
2607 int ret;
2608 BlockDriverState *p;
2609 int64_t eof = 0;
2610 int dummy;
2611 IO_CODE();
2612
2613 assert(!include_base || base); /* Can't include NULL base */
2614
2615 if (!depth) {
2616 depth = &dummy;
2617 }
2618 *depth = 0;
2619
2620 if (!include_base && bs == base) {
2621 *pnum = bytes;
2622 return 0;
2623 }
2624
2625 ret = bdrv_co_block_status(bs, want_zero, offset, bytes, pnum, map, file);
2626 ++*depth;
2627 if (ret < 0 || *pnum == 0 || ret & BDRV_BLOCK_ALLOCATED || bs == base) {
2628 return ret;
2629 }
2630
2631 if (ret & BDRV_BLOCK_EOF) {
2632 eof = offset + *pnum;
2633 }
2634
2635 assert(*pnum <= bytes);
2636 bytes = *pnum;
2637
2638 for (p = bdrv_filter_or_cow_bs(bs); include_base || p != base;
2639 p = bdrv_filter_or_cow_bs(p))
2640 {
2641 ret = bdrv_co_block_status(p, want_zero, offset, bytes, pnum, map,
2642 file);
2643 ++*depth;
2644 if (ret < 0) {
2645 return ret;
2646 }
2647 if (*pnum == 0) {
2648 /*
2649 * The top layer deferred to this layer, and because this layer is
2650 * short, any zeroes that we synthesize beyond EOF behave as if they
2651 * were allocated at this layer.
2652 *
2653 * We don't include BDRV_BLOCK_EOF into ret, as upper layer may be
2654 * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see
2655 * below.
2656 */
2657 assert(ret & BDRV_BLOCK_EOF);
2658 *pnum = bytes;
2659 if (file) {
2660 *file = p;
2661 }
2662 ret = BDRV_BLOCK_ZERO | BDRV_BLOCK_ALLOCATED;
2663 break;
2664 }
2665 if (ret & BDRV_BLOCK_ALLOCATED) {
2666 /*
2667 * We've found the node and the status, we must break.
2668 *
2669 * Drop BDRV_BLOCK_EOF, as it's not for upper layer, which may be
2670 * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see
2671 * below.
2672 */
2673 ret &= ~BDRV_BLOCK_EOF;
2674 break;
2675 }
2676
2677 if (p == base) {
2678 assert(include_base);
2679 break;
2680 }
2681
2682 /*
2683 * OK, [offset, offset + *pnum) region is unallocated on this layer,
2684 * let's continue the diving.
2685 */
2686 assert(*pnum <= bytes);
2687 bytes = *pnum;
2688 }
2689
2690 if (offset + *pnum == eof) {
2691 ret |= BDRV_BLOCK_EOF;
2692 }
2693
2694 return ret;
2695 }
2696
2697 int bdrv_block_status_above(BlockDriverState *bs, BlockDriverState *base,
2698 int64_t offset, int64_t bytes, int64_t *pnum,
2699 int64_t *map, BlockDriverState **file)
2700 {
2701 IO_CODE();
2702 return bdrv_common_block_status_above(bs, base, false, true, offset, bytes,
2703 pnum, map, file, NULL);
2704 }
2705
2706 int bdrv_block_status(BlockDriverState *bs, int64_t offset, int64_t bytes,
2707 int64_t *pnum, int64_t *map, BlockDriverState **file)
2708 {
2709 IO_CODE();
2710 return bdrv_block_status_above(bs, bdrv_filter_or_cow_bs(bs),
2711 offset, bytes, pnum, map, file);
2712 }
2713
2714 /*
2715 * Check @bs (and its backing chain) to see if the range defined
2716 * by @offset and @bytes is known to read as zeroes.
2717 * Return 1 if that is the case, 0 otherwise and -errno on error.
2718 * This test is meant to be fast rather than accurate so returning 0
2719 * does not guarantee non-zero data.
2720 */
2721 int coroutine_fn bdrv_co_is_zero_fast(BlockDriverState *bs, int64_t offset,
2722 int64_t bytes)
2723 {
2724 int ret;
2725 int64_t pnum = bytes;
2726 IO_CODE();
2727
2728 if (!bytes) {
2729 return 1;
2730 }
2731
2732 ret = bdrv_common_block_status_above(bs, NULL, false, false, offset,
2733 bytes, &pnum, NULL, NULL, NULL);
2734
2735 if (ret < 0) {
2736 return ret;
2737 }
2738
2739 return (pnum == bytes) && (ret & BDRV_BLOCK_ZERO);
2740 }
2741
2742 int coroutine_fn bdrv_is_allocated(BlockDriverState *bs, int64_t offset,
2743 int64_t bytes, int64_t *pnum)
2744 {
2745 int ret;
2746 int64_t dummy;
2747 IO_CODE();
2748
2749 ret = bdrv_common_block_status_above(bs, bs, true, false, offset,
2750 bytes, pnum ? pnum : &dummy, NULL,
2751 NULL, NULL);
2752 if (ret < 0) {
2753 return ret;
2754 }
2755 return !!(ret & BDRV_BLOCK_ALLOCATED);
2756 }
2757
2758 /*
2759 * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
2760 *
2761 * Return a positive depth if (a prefix of) the given range is allocated
2762 * in any image between BASE and TOP (BASE is only included if include_base
2763 * is set). Depth 1 is TOP, 2 is the first backing layer, and so forth.
2764 * BASE can be NULL to check if the given offset is allocated in any
2765 * image of the chain. Return 0 otherwise, or negative errno on
2766 * failure.
2767 *
2768 * 'pnum' is set to the number of bytes (including and immediately
2769 * following the specified offset) that are known to be in the same
2770 * allocated/unallocated state. Note that a subsequent call starting
2771 * at 'offset + *pnum' may return the same allocation status (in other
2772 * words, the result is not necessarily the maximum possible range);
2773 * but 'pnum' will only be 0 when end of file is reached.
2774 */
2775 int bdrv_is_allocated_above(BlockDriverState *top,
2776 BlockDriverState *base,
2777 bool include_base, int64_t offset,
2778 int64_t bytes, int64_t *pnum)
2779 {
2780 int depth;
2781 int ret = bdrv_common_block_status_above(top, base, include_base, false,
2782 offset, bytes, pnum, NULL, NULL,
2783 &depth);
2784 IO_CODE();
2785 if (ret < 0) {
2786 return ret;
2787 }
2788
2789 if (ret & BDRV_BLOCK_ALLOCATED) {
2790 return depth;
2791 }
2792 return 0;
2793 }
2794
2795 int coroutine_fn
2796 bdrv_co_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
2797 {
2798 BlockDriver *drv = bs->drv;
2799 BlockDriverState *child_bs = bdrv_primary_bs(bs);
2800 int ret;
2801 IO_CODE();
2802
2803 ret = bdrv_check_qiov_request(pos, qiov->size, qiov, 0, NULL);
2804 if (ret < 0) {
2805 return ret;
2806 }
2807
2808 if (!drv) {
2809 return -ENOMEDIUM;
2810 }
2811
2812 bdrv_inc_in_flight(bs);
2813
2814 if (drv->bdrv_load_vmstate) {
2815 ret = drv->bdrv_load_vmstate(bs, qiov, pos);
2816 } else if (child_bs) {
2817 ret = bdrv_co_readv_vmstate(child_bs, qiov, pos);
2818 } else {
2819 ret = -ENOTSUP;
2820 }
2821
2822 bdrv_dec_in_flight(bs);
2823
2824 return ret;
2825 }
2826
2827 int coroutine_fn
2828 bdrv_co_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
2829 {
2830 BlockDriver *drv = bs->drv;
2831 BlockDriverState *child_bs = bdrv_primary_bs(bs);
2832 int ret;
2833 IO_CODE();
2834
2835 ret = bdrv_check_qiov_request(pos, qiov->size, qiov, 0, NULL);
2836 if (ret < 0) {
2837 return ret;
2838 }
2839
2840 if (!drv) {
2841 return -ENOMEDIUM;
2842 }
2843
2844 bdrv_inc_in_flight(bs);
2845
2846 if (drv->bdrv_save_vmstate) {
2847 ret = drv->bdrv_save_vmstate(bs, qiov, pos);
2848 } else if (child_bs) {
2849 ret = bdrv_co_writev_vmstate(child_bs, qiov, pos);
2850 } else {
2851 ret = -ENOTSUP;
2852 }
2853
2854 bdrv_dec_in_flight(bs);
2855
2856 return ret;
2857 }
2858
2859 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
2860 int64_t pos, int size)
2861 {
2862 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size);
2863 int ret = bdrv_writev_vmstate(bs, &qiov, pos);
2864 IO_CODE();
2865
2866 return ret < 0 ? ret : size;
2867 }
2868
2869 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
2870 int64_t pos, int size)
2871 {
2872 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size);
2873 int ret = bdrv_readv_vmstate(bs, &qiov, pos);
2874 IO_CODE();
2875
2876 return ret < 0 ? ret : size;
2877 }
2878
2879 /**************************************************************/
2880 /* async I/Os */
2881
2882 void bdrv_aio_cancel(BlockAIOCB *acb)
2883 {
2884 IO_CODE();
2885 qemu_aio_ref(acb);
2886 bdrv_aio_cancel_async(acb);
2887 while (acb->refcnt > 1) {
2888 if (acb->aiocb_info->get_aio_context) {
2889 aio_poll(acb->aiocb_info->get_aio_context(acb), true);
2890 } else if (acb->bs) {
2891 /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so
2892 * assert that we're not using an I/O thread. Thread-safe
2893 * code should use bdrv_aio_cancel_async exclusively.
2894 */
2895 assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context());
2896 aio_poll(bdrv_get_aio_context(acb->bs), true);
2897 } else {
2898 abort();
2899 }
2900 }
2901 qemu_aio_unref(acb);
2902 }
2903
2904 /* Async version of aio cancel. The caller is not blocked if the acb implements
2905 * cancel_async, otherwise we do nothing and let the request normally complete.
2906 * In either case the completion callback must be called. */
2907 void bdrv_aio_cancel_async(BlockAIOCB *acb)
2908 {
2909 IO_CODE();
2910 if (acb->aiocb_info->cancel_async) {
2911 acb->aiocb_info->cancel_async(acb);
2912 }
2913 }
2914
2915 /**************************************************************/
2916 /* Coroutine block device emulation */
2917
2918 int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
2919 {
2920 BdrvChild *primary_child = bdrv_primary_child(bs);
2921 BdrvChild *child;
2922 int current_gen;
2923 int ret = 0;
2924 IO_CODE();
2925
2926 bdrv_inc_in_flight(bs);
2927
2928 if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs) ||
2929 bdrv_is_sg(bs)) {
2930 goto early_exit;
2931 }
2932
2933 qemu_co_mutex_lock(&bs->reqs_lock);
2934 current_gen = qatomic_read(&bs->write_gen);
2935
2936 /* Wait until any previous flushes are completed */
2937 while (bs->active_flush_req) {
2938 qemu_co_queue_wait(&bs->flush_queue, &bs->reqs_lock);
2939 }
2940
2941 /* Flushes reach this point in nondecreasing current_gen order. */
2942 bs->active_flush_req = true;
2943 qemu_co_mutex_unlock(&bs->reqs_lock);
2944
2945 /* Write back all layers by calling one driver function */
2946 if (bs->drv->bdrv_co_flush) {
2947 ret = bs->drv->bdrv_co_flush(bs);
2948 goto out;
2949 }
2950
2951 /* Write back cached data to the OS even with cache=unsafe */
2952 BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_OS);
2953 if (bs->drv->bdrv_co_flush_to_os) {
2954 ret = bs->drv->bdrv_co_flush_to_os(bs);
2955 if (ret < 0) {
2956 goto out;
2957 }
2958 }
2959
2960 /* But don't actually force it to the disk with cache=unsafe */
2961 if (bs->open_flags & BDRV_O_NO_FLUSH) {
2962 goto flush_children;
2963 }
2964
2965 /* Check if we really need to flush anything */
2966 if (bs->flushed_gen == current_gen) {
2967 goto flush_children;
2968 }
2969
2970 BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_DISK);
2971 if (!bs->drv) {
2972 /* bs->drv->bdrv_co_flush() might have ejected the BDS
2973 * (even in case of apparent success) */
2974 ret = -ENOMEDIUM;
2975 goto out;
2976 }
2977 if (bs->drv->bdrv_co_flush_to_disk) {
2978 ret = bs->drv->bdrv_co_flush_to_disk(bs);
2979 } else if (bs->drv->bdrv_aio_flush) {
2980 BlockAIOCB *acb;
2981 CoroutineIOCompletion co = {
2982 .coroutine = qemu_coroutine_self(),
2983 };
2984
2985 acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
2986 if (acb == NULL) {
2987 ret = -EIO;
2988 } else {
2989 qemu_coroutine_yield();
2990 ret = co.ret;
2991 }
2992 } else {
2993 /*
2994 * Some block drivers always operate in either writethrough or unsafe
2995 * mode and don't support bdrv_flush therefore. Usually qemu doesn't
2996 * know how the server works (because the behaviour is hardcoded or
2997 * depends on server-side configuration), so we can't ensure that
2998 * everything is safe on disk. Returning an error doesn't work because
2999 * that would break guests even if the server operates in writethrough
3000 * mode.
3001 *
3002 * Let's hope the user knows what he's doing.
3003 */
3004 ret = 0;
3005 }
3006
3007 if (ret < 0) {
3008 goto out;
3009 }
3010
3011 /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH
3012 * in the case of cache=unsafe, so there are no useless flushes.
3013 */
3014 flush_children:
3015 ret = 0;
3016 QLIST_FOREACH(child, &bs->children, next) {
3017 if (child->perm & (BLK_PERM_WRITE | BLK_PERM_WRITE_UNCHANGED)) {
3018 int this_child_ret = bdrv_co_flush(child->bs);
3019 if (!ret) {
3020 ret = this_child_ret;
3021 }
3022 }
3023 }
3024
3025 out:
3026 /* Notify any pending flushes that we have completed */
3027 if (ret == 0) {
3028 bs->flushed_gen = current_gen;
3029 }
3030
3031 qemu_co_mutex_lock(&bs->reqs_lock);
3032 bs->active_flush_req = false;
3033 /* Return value is ignored - it's ok if wait queue is empty */
3034 qemu_co_queue_next(&bs->flush_queue);
3035 qemu_co_mutex_unlock(&bs->reqs_lock);
3036
3037 early_exit:
3038 bdrv_dec_in_flight(bs);
3039 return ret;
3040 }
3041
3042 int coroutine_fn bdrv_co_pdiscard(BdrvChild *child, int64_t offset,
3043 int64_t bytes)
3044 {
3045 BdrvTrackedRequest req;
3046 int ret;
3047 int64_t max_pdiscard;
3048 int head, tail, align;
3049 BlockDriverState *bs = child->bs;
3050 IO_CODE();
3051
3052 if (!bs || !bs->drv || !bdrv_is_inserted(bs)) {
3053 return -ENOMEDIUM;
3054 }
3055
3056 if (bdrv_has_readonly_bitmaps(bs)) {
3057 return -EPERM;
3058 }
3059
3060 ret = bdrv_check_request(offset, bytes, NULL);
3061 if (ret < 0) {
3062 return ret;
3063 }
3064
3065 /* Do nothing if disabled. */
3066 if (!(bs->open_flags & BDRV_O_UNMAP)) {
3067 return 0;
3068 }
3069
3070 if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) {
3071 return 0;
3072 }
3073
3074 /* Invalidate the cached block-status data range if this discard overlaps */
3075 bdrv_bsc_invalidate_range(bs, offset, bytes);
3076
3077 /* Discard is advisory, but some devices track and coalesce
3078 * unaligned requests, so we must pass everything down rather than
3079 * round here. Still, most devices will just silently ignore
3080 * unaligned requests (by returning -ENOTSUP), so we must fragment
3081 * the request accordingly. */
3082 align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment);
3083 assert(align % bs->bl.request_alignment == 0);
3084 head = offset % align;
3085 tail = (offset + bytes) % align;
3086
3087 bdrv_inc_in_flight(bs);
3088 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_DISCARD);
3089
3090 ret = bdrv_co_write_req_prepare(child, offset, bytes, &req, 0);
3091 if (ret < 0) {
3092 goto out;
3093 }
3094
3095 max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT64_MAX),
3096 align);
3097 assert(max_pdiscard >= bs->bl.request_alignment);
3098
3099 while (bytes > 0) {
3100 int64_t num = bytes;
3101
3102 if (head) {
3103 /* Make small requests to get to alignment boundaries. */
3104 num = MIN(bytes, align - head);
3105 if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) {
3106 num %= bs->bl.request_alignment;
3107 }
3108 head = (head + num) % align;
3109 assert(num < max_pdiscard);
3110 } else if (tail) {
3111 if (num > align) {
3112 /* Shorten the request to the last aligned cluster. */
3113 num -= tail;
3114 } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) &&
3115 tail > bs->bl.request_alignment) {
3116 tail %= bs->bl.request_alignment;
3117 num -= tail;
3118 }
3119 }
3120 /* limit request size */
3121 if (num > max_pdiscard) {
3122 num = max_pdiscard;
3123 }
3124
3125 if (!bs->drv) {
3126 ret = -ENOMEDIUM;
3127 goto out;
3128 }
3129 if (bs->drv->bdrv_co_pdiscard) {
3130 ret = bs->drv->bdrv_co_pdiscard(bs, offset, num);
3131 } else {
3132 BlockAIOCB *acb;
3133 CoroutineIOCompletion co = {
3134 .coroutine = qemu_coroutine_self(),
3135 };
3136
3137 acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num,
3138 bdrv_co_io_em_complete, &co);
3139 if (acb == NULL) {
3140 ret = -EIO;
3141 goto out;
3142 } else {
3143 qemu_coroutine_yield();
3144 ret = co.ret;
3145 }
3146 }
3147 if (ret && ret != -ENOTSUP) {
3148 goto out;
3149 }
3150
3151 offset += num;
3152 bytes -= num;
3153 }
3154 ret = 0;
3155 out:
3156 bdrv_co_write_req_finish(child, req.offset, req.bytes, &req, ret);
3157 tracked_request_end(&req);
3158 bdrv_dec_in_flight(bs);
3159 return ret;
3160 }
3161
3162 int coroutine_fn bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf)
3163 {
3164 BlockDriver *drv = bs->drv;
3165 CoroutineIOCompletion co = {
3166 .coroutine = qemu_coroutine_self(),
3167 };
3168 BlockAIOCB *acb;
3169 IO_CODE();
3170
3171 bdrv_inc_in_flight(bs);
3172 if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) {
3173 co.ret = -ENOTSUP;
3174 goto out;
3175 }
3176
3177 if (drv->bdrv_co_ioctl) {
3178 co.ret = drv->bdrv_co_ioctl(bs, req, buf);
3179 } else {
3180 acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
3181 if (!acb) {
3182 co.ret = -ENOTSUP;
3183 goto out;
3184 }
3185 qemu_coroutine_yield();
3186 }
3187 out:
3188 bdrv_dec_in_flight(bs);
3189 return co.ret;
3190 }
3191
3192 void *qemu_blockalign(BlockDriverState *bs, size_t size)
3193 {
3194 IO_CODE();
3195 return qemu_memalign(bdrv_opt_mem_align(bs), size);
3196 }
3197
3198 void *qemu_blockalign0(BlockDriverState *bs, size_t size)
3199 {
3200 IO_CODE();
3201 return memset(qemu_blockalign(bs, size), 0, size);
3202 }
3203
3204 void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
3205 {
3206 size_t align = bdrv_opt_mem_align(bs);
3207 IO_CODE();
3208
3209 /* Ensure that NULL is never returned on success */
3210 assert(align > 0);
3211 if (size == 0) {
3212 size = align;
3213 }
3214
3215 return qemu_try_memalign(align, size);
3216 }
3217
3218 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
3219 {
3220 void *mem = qemu_try_blockalign(bs, size);
3221 IO_CODE();
3222
3223 if (mem) {
3224 memset(mem, 0, size);
3225 }
3226
3227 return mem;
3228 }
3229
3230 void bdrv_io_plug(BlockDriverState *bs)
3231 {
3232 BdrvChild *child;
3233 IO_CODE();
3234
3235 QLIST_FOREACH(child, &bs->children, next) {
3236 bdrv_io_plug(child->bs);
3237 }
3238
3239 if (qatomic_fetch_inc(&bs->io_plugged) == 0) {
3240 BlockDriver *drv = bs->drv;
3241 if (drv && drv->bdrv_io_plug) {
3242 drv->bdrv_io_plug(bs);
3243 }
3244 }
3245 }
3246
3247 void bdrv_io_unplug(BlockDriverState *bs)
3248 {
3249 BdrvChild *child;
3250 IO_CODE();
3251
3252 assert(bs->io_plugged);
3253 if (qatomic_fetch_dec(&bs->io_plugged) == 1) {
3254 BlockDriver *drv = bs->drv;
3255 if (drv && drv->bdrv_io_unplug) {
3256 drv->bdrv_io_unplug(bs);
3257 }
3258 }
3259
3260 QLIST_FOREACH(child, &bs->children, next) {
3261 bdrv_io_unplug(child->bs);
3262 }
3263 }
3264
3265 void bdrv_register_buf(BlockDriverState *bs, void *host, size_t size)
3266 {
3267 BdrvChild *child;
3268
3269 GLOBAL_STATE_CODE();
3270 if (bs->drv && bs->drv->bdrv_register_buf) {
3271 bs->drv->bdrv_register_buf(bs, host, size);
3272 }
3273 QLIST_FOREACH(child, &bs->children, next) {
3274 bdrv_register_buf(child->bs, host, size);
3275 }
3276 }
3277
3278 void bdrv_unregister_buf(BlockDriverState *bs, void *host)
3279 {
3280 BdrvChild *child;
3281
3282 GLOBAL_STATE_CODE();
3283 if (bs->drv && bs->drv->bdrv_unregister_buf) {
3284 bs->drv->bdrv_unregister_buf(bs, host);
3285 }
3286 QLIST_FOREACH(child, &bs->children, next) {
3287 bdrv_unregister_buf(child->bs, host);
3288 }
3289 }
3290
3291 static int coroutine_fn bdrv_co_copy_range_internal(
3292 BdrvChild *src, int64_t src_offset, BdrvChild *dst,
3293 int64_t dst_offset, int64_t bytes,
3294 BdrvRequestFlags read_flags, BdrvRequestFlags write_flags,
3295 bool recurse_src)
3296 {
3297 BdrvTrackedRequest req;
3298 int ret;
3299
3300 /* TODO We can support BDRV_REQ_NO_FALLBACK here */
3301 assert(!(read_flags & BDRV_REQ_NO_FALLBACK));
3302 assert(!(write_flags & BDRV_REQ_NO_FALLBACK));
3303 assert(!(read_flags & BDRV_REQ_NO_WAIT));
3304 assert(!(write_flags & BDRV_REQ_NO_WAIT));
3305
3306 if (!dst || !dst->bs || !bdrv_is_inserted(dst->bs)) {
3307 return -ENOMEDIUM;
3308 }
3309 ret = bdrv_check_request32(dst_offset, bytes, NULL, 0);
3310 if (ret) {
3311 return ret;
3312 }
3313 if (write_flags & BDRV_REQ_ZERO_WRITE) {
3314 return bdrv_co_pwrite_zeroes(dst, dst_offset, bytes, write_flags);
3315 }
3316
3317 if (!src || !src->bs || !bdrv_is_inserted(src->bs)) {
3318 return -ENOMEDIUM;
3319 }
3320 ret = bdrv_check_request32(src_offset, bytes, NULL, 0);
3321 if (ret) {
3322 return ret;
3323 }
3324
3325 if (!src->bs->drv->bdrv_co_copy_range_from
3326 || !dst->bs->drv->bdrv_co_copy_range_to
3327 || src->bs->encrypted || dst->bs->encrypted) {
3328 return -ENOTSUP;
3329 }
3330
3331 if (recurse_src) {
3332 bdrv_inc_in_flight(src->bs);
3333 tracked_request_begin(&req, src->bs, src_offset, bytes,
3334 BDRV_TRACKED_READ);
3335
3336 /* BDRV_REQ_SERIALISING is only for write operation */
3337 assert(!(read_flags & BDRV_REQ_SERIALISING));
3338 bdrv_wait_serialising_requests(&req);
3339
3340 ret = src->bs->drv->bdrv_co_copy_range_from(src->bs,
3341 src, src_offset,
3342 dst, dst_offset,
3343 bytes,
3344 read_flags, write_flags);
3345
3346 tracked_request_end(&req);
3347 bdrv_dec_in_flight(src->bs);
3348 } else {
3349 bdrv_inc_in_flight(dst->bs);
3350 tracked_request_begin(&req, dst->bs, dst_offset, bytes,
3351 BDRV_TRACKED_WRITE);
3352 ret = bdrv_co_write_req_prepare(dst, dst_offset, bytes, &req,
3353 write_flags);
3354 if (!ret) {
3355 ret = dst->bs->drv->bdrv_co_copy_range_to(dst->bs,
3356 src, src_offset,
3357 dst, dst_offset,
3358 bytes,
3359 read_flags, write_flags);
3360 }
3361 bdrv_co_write_req_finish(dst, dst_offset, bytes, &req, ret);
3362 tracked_request_end(&req);
3363 bdrv_dec_in_flight(dst->bs);
3364 }
3365
3366 return ret;
3367 }
3368
3369 /* Copy range from @src to @dst.
3370 *
3371 * See the comment of bdrv_co_copy_range for the parameter and return value
3372 * semantics. */
3373 int coroutine_fn bdrv_co_copy_range_from(BdrvChild *src, int64_t src_offset,
3374 BdrvChild *dst, int64_t dst_offset,
3375 int64_t bytes,
3376 BdrvRequestFlags read_flags,
3377 BdrvRequestFlags write_flags)
3378 {
3379 IO_CODE();
3380 trace_bdrv_co_copy_range_from(src, src_offset, dst, dst_offset, bytes,
3381 read_flags, write_flags);
3382 return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset,
3383 bytes, read_flags, write_flags, true);
3384 }
3385
3386 /* Copy range from @src to @dst.
3387 *
3388 * See the comment of bdrv_co_copy_range for the parameter and return value
3389 * semantics. */
3390 int coroutine_fn bdrv_co_copy_range_to(BdrvChild *src, int64_t src_offset,
3391 BdrvChild *dst, int64_t dst_offset,
3392 int64_t bytes,
3393 BdrvRequestFlags read_flags,
3394 BdrvRequestFlags write_flags)
3395 {
3396 IO_CODE();
3397 trace_bdrv_co_copy_range_to(src, src_offset, dst, dst_offset, bytes,
3398 read_flags, write_flags);
3399 return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset,
3400 bytes, read_flags, write_flags, false);
3401 }
3402
3403 int coroutine_fn bdrv_co_copy_range(BdrvChild *src, int64_t src_offset,
3404 BdrvChild *dst, int64_t dst_offset,
3405 int64_t bytes, BdrvRequestFlags read_flags,
3406 BdrvRequestFlags write_flags)
3407 {
3408 IO_CODE();
3409 return bdrv_co_copy_range_from(src, src_offset,
3410 dst, dst_offset,
3411 bytes, read_flags, write_flags);
3412 }
3413
3414 static void bdrv_parent_cb_resize(BlockDriverState *bs)
3415 {
3416 BdrvChild *c;
3417 QLIST_FOREACH(c, &bs->parents, next_parent) {
3418 if (c->klass->resize) {
3419 c->klass->resize(c);
3420 }
3421 }
3422 }
3423
3424 /**
3425 * Truncate file to 'offset' bytes (needed only for file protocols)
3426 *
3427 * If 'exact' is true, the file must be resized to exactly the given
3428 * 'offset'. Otherwise, it is sufficient for the node to be at least
3429 * 'offset' bytes in length.
3430 */
3431 int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset, bool exact,
3432 PreallocMode prealloc, BdrvRequestFlags flags,
3433 Error **errp)
3434 {
3435 BlockDriverState *bs = child->bs;
3436 BdrvChild *filtered, *backing;
3437 BlockDriver *drv = bs->drv;
3438 BdrvTrackedRequest req;
3439 int64_t old_size, new_bytes;
3440 int ret;
3441 IO_CODE();
3442
3443 /* if bs->drv == NULL, bs is closed, so there's nothing to do here */
3444 if (!drv) {
3445 error_setg(errp, "No medium inserted");
3446 return -ENOMEDIUM;
3447 }
3448 if (offset < 0) {
3449 error_setg(errp, "Image size cannot be negative");
3450 return -EINVAL;
3451 }
3452
3453 ret = bdrv_check_request(offset, 0, errp);
3454 if (ret < 0) {
3455 return ret;
3456 }
3457
3458 old_size = bdrv_getlength(bs);
3459 if (old_size < 0) {
3460 error_setg_errno(errp, -old_size, "Failed to get old image size");
3461 return old_size;
3462 }
3463
3464 if (bdrv_is_read_only(bs)) {
3465 error_setg(errp, "Image is read-only");
3466 return -EACCES;
3467 }
3468
3469 if (offset > old_size) {
3470 new_bytes = offset - old_size;
3471 } else {
3472 new_bytes = 0;
3473 }
3474
3475 bdrv_inc_in_flight(bs);
3476 tracked_request_begin(&req, bs, offset - new_bytes, new_bytes,
3477 BDRV_TRACKED_TRUNCATE);
3478
3479 /* If we are growing the image and potentially using preallocation for the
3480 * new area, we need to make sure that no write requests are made to it
3481 * concurrently or they might be overwritten by preallocation. */
3482 if (new_bytes) {
3483 bdrv_make_request_serialising(&req, 1);
3484 }
3485 ret = bdrv_co_write_req_prepare(child, offset - new_bytes, new_bytes, &req,
3486 0);
3487 if (ret < 0) {
3488 error_setg_errno(errp, -ret,
3489 "Failed to prepare request for truncation");
3490 goto out;
3491 }
3492
3493 filtered = bdrv_filter_child(bs);
3494 backing = bdrv_cow_child(bs);
3495
3496 /*
3497 * If the image has a backing file that is large enough that it would
3498 * provide data for the new area, we cannot leave it unallocated because
3499 * then the backing file content would become visible. Instead, zero-fill
3500 * the new area.
3501 *
3502 * Note that if the image has a backing file, but was opened without the
3503 * backing file, taking care of keeping things consistent with that backing
3504 * file is the user's responsibility.
3505 */
3506 if (new_bytes && backing) {
3507 int64_t backing_len;
3508
3509 backing_len = bdrv_getlength(backing->bs);
3510 if (backing_len < 0) {
3511 ret = backing_len;
3512 error_setg_errno(errp, -ret, "Could not get backing file size");
3513 goto out;
3514 }
3515
3516 if (backing_len > old_size) {
3517 flags |= BDRV_REQ_ZERO_WRITE;
3518 }
3519 }
3520
3521 if (drv->bdrv_co_truncate) {
3522 if (flags & ~bs->supported_truncate_flags) {
3523 error_setg(errp, "Block driver does not support requested flags");
3524 ret = -ENOTSUP;
3525 goto out;
3526 }
3527 ret = drv->bdrv_co_truncate(bs, offset, exact, prealloc, flags, errp);
3528 } else if (filtered) {
3529 ret = bdrv_co_truncate(filtered, offset, exact, prealloc, flags, errp);
3530 } else {
3531 error_setg(errp, "Image format driver does not support resize");
3532 ret = -ENOTSUP;
3533 goto out;
3534 }
3535 if (ret < 0) {
3536 goto out;
3537 }
3538
3539 ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS);
3540 if (ret < 0) {
3541 error_setg_errno(errp, -ret, "Could not refresh total sector count");
3542 } else {
3543 offset = bs->total_sectors * BDRV_SECTOR_SIZE;
3544 }
3545 /* It's possible that truncation succeeded but refresh_total_sectors
3546 * failed, but the latter doesn't affect how we should finish the request.
3547 * Pass 0 as the last parameter so that dirty bitmaps etc. are handled. */
3548 bdrv_co_write_req_finish(child, offset - new_bytes, new_bytes, &req, 0);
3549
3550 out:
3551 tracked_request_end(&req);
3552 bdrv_dec_in_flight(bs);
3553
3554 return ret;
3555 }
3556
3557 void bdrv_cancel_in_flight(BlockDriverState *bs)
3558 {
3559 GLOBAL_STATE_CODE();
3560 if (!bs || !bs->drv) {
3561 return;
3562 }
3563
3564 if (bs->drv->bdrv_cancel_in_flight) {
3565 bs->drv->bdrv_cancel_in_flight(bs);
3566 }
3567 }
3568
3569 int coroutine_fn
3570 bdrv_co_preadv_snapshot(BdrvChild *child, int64_t offset, int64_t bytes,
3571 QEMUIOVector *qiov, size_t qiov_offset)
3572 {
3573 BlockDriverState *bs = child->bs;
3574 BlockDriver *drv = bs->drv;
3575 int ret;
3576 IO_CODE();
3577
3578 if (!drv) {
3579 return -ENOMEDIUM;
3580 }
3581
3582 if (!drv->bdrv_co_preadv_snapshot) {
3583 return -ENOTSUP;
3584 }
3585
3586 bdrv_inc_in_flight(bs);
3587 ret = drv->bdrv_co_preadv_snapshot(bs, offset, bytes, qiov, qiov_offset);
3588 bdrv_dec_in_flight(bs);
3589
3590 return ret;
3591 }
3592
3593 int coroutine_fn
3594 bdrv_co_snapshot_block_status(BlockDriverState *bs,
3595 bool want_zero, int64_t offset, int64_t bytes,
3596 int64_t *pnum, int64_t *map,
3597 BlockDriverState **file)
3598 {
3599 BlockDriver *drv = bs->drv;
3600 int ret;
3601 IO_CODE();
3602
3603 if (!drv) {
3604 return -ENOMEDIUM;
3605 }
3606
3607 if (!drv->bdrv_co_snapshot_block_status) {
3608 return -ENOTSUP;
3609 }
3610
3611 bdrv_inc_in_flight(bs);
3612 ret = drv->bdrv_co_snapshot_block_status(bs, want_zero, offset, bytes,
3613 pnum, map, file);
3614 bdrv_dec_in_flight(bs);
3615
3616 return ret;
3617 }
3618
3619 int coroutine_fn
3620 bdrv_co_pdiscard_snapshot(BlockDriverState *bs, int64_t offset, int64_t bytes)
3621 {
3622 BlockDriver *drv = bs->drv;
3623 int ret;
3624 IO_CODE();
3625
3626 if (!drv) {
3627 return -ENOMEDIUM;
3628 }
3629
3630 if (!drv->bdrv_co_pdiscard_snapshot) {
3631 return -ENOTSUP;
3632 }
3633
3634 bdrv_inc_in_flight(bs);
3635 ret = drv->bdrv_co_pdiscard_snapshot(bs, offset, bytes);
3636 bdrv_dec_in_flight(bs);
3637
3638 return ret;
3639 }