]> git.proxmox.com Git - mirror_qemu.git/blob - block/io.c
block: Call drain callbacks only once
[mirror_qemu.git] / block / io.c
1 /*
2 * Block layer I/O functions
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 #include "qemu/osdep.h"
26 #include "trace.h"
27 #include "sysemu/block-backend.h"
28 #include "block/aio-wait.h"
29 #include "block/blockjob.h"
30 #include "block/blockjob_int.h"
31 #include "block/block_int.h"
32 #include "block/coroutines.h"
33 #include "block/write-threshold.h"
34 #include "qemu/cutils.h"
35 #include "qemu/memalign.h"
36 #include "qapi/error.h"
37 #include "qemu/error-report.h"
38 #include "qemu/main-loop.h"
39 #include "sysemu/replay.h"
40
41 /* Maximum bounce buffer for copy-on-read and write zeroes, in bytes */
42 #define MAX_BOUNCE_BUFFER (32768 << BDRV_SECTOR_BITS)
43
44 static void bdrv_parent_cb_resize(BlockDriverState *bs);
45 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
46 int64_t offset, int64_t bytes, BdrvRequestFlags flags);
47
48 static void bdrv_parent_drained_begin(BlockDriverState *bs, BdrvChild *ignore,
49 bool ignore_bds_parents)
50 {
51 BdrvChild *c, *next;
52
53 QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
54 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) {
55 continue;
56 }
57 bdrv_parent_drained_begin_single(c, false);
58 }
59 }
60
61 void bdrv_parent_drained_end_single(BdrvChild *c)
62 {
63 IO_OR_GS_CODE();
64
65 assert(c->quiesced_parent);
66 c->quiesced_parent = false;
67
68 if (c->klass->drained_end) {
69 c->klass->drained_end(c);
70 }
71 }
72
73 static void bdrv_parent_drained_end(BlockDriverState *bs, BdrvChild *ignore,
74 bool ignore_bds_parents)
75 {
76 BdrvChild *c;
77
78 QLIST_FOREACH(c, &bs->parents, next_parent) {
79 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) {
80 continue;
81 }
82 bdrv_parent_drained_end_single(c);
83 }
84 }
85
86 static bool bdrv_parent_drained_poll_single(BdrvChild *c)
87 {
88 if (c->klass->drained_poll) {
89 return c->klass->drained_poll(c);
90 }
91 return false;
92 }
93
94 static bool bdrv_parent_drained_poll(BlockDriverState *bs, BdrvChild *ignore,
95 bool ignore_bds_parents)
96 {
97 BdrvChild *c, *next;
98 bool busy = false;
99
100 QLIST_FOREACH_SAFE(c, &bs->parents, next_parent, next) {
101 if (c == ignore || (ignore_bds_parents && c->klass->parent_is_bds)) {
102 continue;
103 }
104 busy |= bdrv_parent_drained_poll_single(c);
105 }
106
107 return busy;
108 }
109
110 void bdrv_parent_drained_begin_single(BdrvChild *c, bool poll)
111 {
112 AioContext *ctx = bdrv_child_get_parent_aio_context(c);
113 IO_OR_GS_CODE();
114
115 assert(!c->quiesced_parent);
116 c->quiesced_parent = true;
117
118 if (c->klass->drained_begin) {
119 c->klass->drained_begin(c);
120 }
121 if (poll) {
122 AIO_WAIT_WHILE(ctx, bdrv_parent_drained_poll_single(c));
123 }
124 }
125
126 static void bdrv_merge_limits(BlockLimits *dst, const BlockLimits *src)
127 {
128 dst->pdiscard_alignment = MAX(dst->pdiscard_alignment,
129 src->pdiscard_alignment);
130 dst->opt_transfer = MAX(dst->opt_transfer, src->opt_transfer);
131 dst->max_transfer = MIN_NON_ZERO(dst->max_transfer, src->max_transfer);
132 dst->max_hw_transfer = MIN_NON_ZERO(dst->max_hw_transfer,
133 src->max_hw_transfer);
134 dst->opt_mem_alignment = MAX(dst->opt_mem_alignment,
135 src->opt_mem_alignment);
136 dst->min_mem_alignment = MAX(dst->min_mem_alignment,
137 src->min_mem_alignment);
138 dst->max_iov = MIN_NON_ZERO(dst->max_iov, src->max_iov);
139 dst->max_hw_iov = MIN_NON_ZERO(dst->max_hw_iov, src->max_hw_iov);
140 }
141
142 typedef struct BdrvRefreshLimitsState {
143 BlockDriverState *bs;
144 BlockLimits old_bl;
145 } BdrvRefreshLimitsState;
146
147 static void bdrv_refresh_limits_abort(void *opaque)
148 {
149 BdrvRefreshLimitsState *s = opaque;
150
151 s->bs->bl = s->old_bl;
152 }
153
154 static TransactionActionDrv bdrv_refresh_limits_drv = {
155 .abort = bdrv_refresh_limits_abort,
156 .clean = g_free,
157 };
158
159 /* @tran is allowed to be NULL, in this case no rollback is possible. */
160 void bdrv_refresh_limits(BlockDriverState *bs, Transaction *tran, Error **errp)
161 {
162 ERRP_GUARD();
163 BlockDriver *drv = bs->drv;
164 BdrvChild *c;
165 bool have_limits;
166
167 GLOBAL_STATE_CODE();
168
169 if (tran) {
170 BdrvRefreshLimitsState *s = g_new(BdrvRefreshLimitsState, 1);
171 *s = (BdrvRefreshLimitsState) {
172 .bs = bs,
173 .old_bl = bs->bl,
174 };
175 tran_add(tran, &bdrv_refresh_limits_drv, s);
176 }
177
178 memset(&bs->bl, 0, sizeof(bs->bl));
179
180 if (!drv) {
181 return;
182 }
183
184 /* Default alignment based on whether driver has byte interface */
185 bs->bl.request_alignment = (drv->bdrv_co_preadv ||
186 drv->bdrv_aio_preadv ||
187 drv->bdrv_co_preadv_part) ? 1 : 512;
188
189 /* Take some limits from the children as a default */
190 have_limits = false;
191 QLIST_FOREACH(c, &bs->children, next) {
192 if (c->role & (BDRV_CHILD_DATA | BDRV_CHILD_FILTERED | BDRV_CHILD_COW))
193 {
194 bdrv_merge_limits(&bs->bl, &c->bs->bl);
195 have_limits = true;
196 }
197 }
198
199 if (!have_limits) {
200 bs->bl.min_mem_alignment = 512;
201 bs->bl.opt_mem_alignment = qemu_real_host_page_size();
202
203 /* Safe default since most protocols use readv()/writev()/etc */
204 bs->bl.max_iov = IOV_MAX;
205 }
206
207 /* Then let the driver override it */
208 if (drv->bdrv_refresh_limits) {
209 drv->bdrv_refresh_limits(bs, errp);
210 if (*errp) {
211 return;
212 }
213 }
214
215 if (bs->bl.request_alignment > BDRV_MAX_ALIGNMENT) {
216 error_setg(errp, "Driver requires too large request alignment");
217 }
218 }
219
220 /**
221 * The copy-on-read flag is actually a reference count so multiple users may
222 * use the feature without worrying about clobbering its previous state.
223 * Copy-on-read stays enabled until all users have called to disable it.
224 */
225 void bdrv_enable_copy_on_read(BlockDriverState *bs)
226 {
227 IO_CODE();
228 qatomic_inc(&bs->copy_on_read);
229 }
230
231 void bdrv_disable_copy_on_read(BlockDriverState *bs)
232 {
233 int old = qatomic_fetch_dec(&bs->copy_on_read);
234 IO_CODE();
235 assert(old >= 1);
236 }
237
238 typedef struct {
239 Coroutine *co;
240 BlockDriverState *bs;
241 bool done;
242 bool begin;
243 bool poll;
244 BdrvChild *parent;
245 bool ignore_bds_parents;
246 } BdrvCoDrainData;
247
248 /* Returns true if BDRV_POLL_WHILE() should go into a blocking aio_poll() */
249 bool bdrv_drain_poll(BlockDriverState *bs, BdrvChild *ignore_parent,
250 bool ignore_bds_parents)
251 {
252 IO_OR_GS_CODE();
253
254 if (bdrv_parent_drained_poll(bs, ignore_parent, ignore_bds_parents)) {
255 return true;
256 }
257
258 if (qatomic_read(&bs->in_flight)) {
259 return true;
260 }
261
262 return false;
263 }
264
265 static bool bdrv_drain_poll_top_level(BlockDriverState *bs,
266 BdrvChild *ignore_parent)
267 {
268 return bdrv_drain_poll(bs, ignore_parent, false);
269 }
270
271 static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent,
272 bool ignore_bds_parents, bool poll);
273 static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent,
274 bool ignore_bds_parents);
275
276 static void bdrv_co_drain_bh_cb(void *opaque)
277 {
278 BdrvCoDrainData *data = opaque;
279 Coroutine *co = data->co;
280 BlockDriverState *bs = data->bs;
281
282 if (bs) {
283 AioContext *ctx = bdrv_get_aio_context(bs);
284 aio_context_acquire(ctx);
285 bdrv_dec_in_flight(bs);
286 if (data->begin) {
287 bdrv_do_drained_begin(bs, data->parent, data->ignore_bds_parents,
288 data->poll);
289 } else {
290 assert(!data->poll);
291 bdrv_do_drained_end(bs, data->parent, data->ignore_bds_parents);
292 }
293 aio_context_release(ctx);
294 } else {
295 assert(data->begin);
296 bdrv_drain_all_begin();
297 }
298
299 data->done = true;
300 aio_co_wake(co);
301 }
302
303 static void coroutine_fn bdrv_co_yield_to_drain(BlockDriverState *bs,
304 bool begin,
305 BdrvChild *parent,
306 bool ignore_bds_parents,
307 bool poll)
308 {
309 BdrvCoDrainData data;
310 Coroutine *self = qemu_coroutine_self();
311 AioContext *ctx = bdrv_get_aio_context(bs);
312 AioContext *co_ctx = qemu_coroutine_get_aio_context(self);
313
314 /* Calling bdrv_drain() from a BH ensures the current coroutine yields and
315 * other coroutines run if they were queued by aio_co_enter(). */
316
317 assert(qemu_in_coroutine());
318 data = (BdrvCoDrainData) {
319 .co = self,
320 .bs = bs,
321 .done = false,
322 .begin = begin,
323 .parent = parent,
324 .ignore_bds_parents = ignore_bds_parents,
325 .poll = poll,
326 };
327
328 if (bs) {
329 bdrv_inc_in_flight(bs);
330 }
331
332 /*
333 * Temporarily drop the lock across yield or we would get deadlocks.
334 * bdrv_co_drain_bh_cb() reaquires the lock as needed.
335 *
336 * When we yield below, the lock for the current context will be
337 * released, so if this is actually the lock that protects bs, don't drop
338 * it a second time.
339 */
340 if (ctx != co_ctx) {
341 aio_context_release(ctx);
342 }
343 replay_bh_schedule_oneshot_event(ctx, bdrv_co_drain_bh_cb, &data);
344
345 qemu_coroutine_yield();
346 /* If we are resumed from some other event (such as an aio completion or a
347 * timer callback), it is a bug in the caller that should be fixed. */
348 assert(data.done);
349
350 /* Reaquire the AioContext of bs if we dropped it */
351 if (ctx != co_ctx) {
352 aio_context_acquire(ctx);
353 }
354 }
355
356 void bdrv_do_drained_begin_quiesce(BlockDriverState *bs,
357 BdrvChild *parent, bool ignore_bds_parents)
358 {
359 IO_OR_GS_CODE();
360 assert(!qemu_in_coroutine());
361
362 /* Stop things in parent-to-child order */
363 if (qatomic_fetch_inc(&bs->quiesce_counter) == 0) {
364 aio_disable_external(bdrv_get_aio_context(bs));
365
366 /* TODO Remove ignore_bds_parents, we don't consider it any more */
367 bdrv_parent_drained_begin(bs, parent, false);
368 if (bs->drv && bs->drv->bdrv_drain_begin) {
369 bs->drv->bdrv_drain_begin(bs);
370 }
371 }
372 }
373
374 static void bdrv_do_drained_begin(BlockDriverState *bs, BdrvChild *parent,
375 bool ignore_bds_parents, bool poll)
376 {
377 if (qemu_in_coroutine()) {
378 bdrv_co_yield_to_drain(bs, true, parent, ignore_bds_parents, poll);
379 return;
380 }
381
382 bdrv_do_drained_begin_quiesce(bs, parent, ignore_bds_parents);
383
384 /*
385 * Wait for drained requests to finish.
386 *
387 * Calling BDRV_POLL_WHILE() only once for the top-level node is okay: The
388 * call is needed so things in this AioContext can make progress even
389 * though we don't return to the main AioContext loop - this automatically
390 * includes other nodes in the same AioContext and therefore all child
391 * nodes.
392 */
393 if (poll) {
394 assert(!ignore_bds_parents);
395 BDRV_POLL_WHILE(bs, bdrv_drain_poll_top_level(bs, parent));
396 }
397 }
398
399 void bdrv_drained_begin(BlockDriverState *bs)
400 {
401 IO_OR_GS_CODE();
402 bdrv_do_drained_begin(bs, NULL, false, true);
403 }
404
405 /**
406 * This function does not poll, nor must any of its recursively called
407 * functions.
408 */
409 static void bdrv_do_drained_end(BlockDriverState *bs, BdrvChild *parent,
410 bool ignore_bds_parents)
411 {
412 int old_quiesce_counter;
413
414 if (qemu_in_coroutine()) {
415 bdrv_co_yield_to_drain(bs, false, parent, ignore_bds_parents, false);
416 return;
417 }
418 assert(bs->quiesce_counter > 0);
419
420 /* Re-enable things in child-to-parent order */
421 old_quiesce_counter = qatomic_fetch_dec(&bs->quiesce_counter);
422 if (old_quiesce_counter == 1) {
423 if (bs->drv && bs->drv->bdrv_drain_end) {
424 bs->drv->bdrv_drain_end(bs);
425 }
426 /* TODO Remove ignore_bds_parents, we don't consider it any more */
427 bdrv_parent_drained_end(bs, parent, false);
428
429 aio_enable_external(bdrv_get_aio_context(bs));
430 }
431 }
432
433 void bdrv_drained_end(BlockDriverState *bs)
434 {
435 IO_OR_GS_CODE();
436 bdrv_do_drained_end(bs, NULL, false);
437 }
438
439 void bdrv_drain(BlockDriverState *bs)
440 {
441 IO_OR_GS_CODE();
442 bdrv_drained_begin(bs);
443 bdrv_drained_end(bs);
444 }
445
446 static void bdrv_drain_assert_idle(BlockDriverState *bs)
447 {
448 BdrvChild *child, *next;
449
450 assert(qatomic_read(&bs->in_flight) == 0);
451 QLIST_FOREACH_SAFE(child, &bs->children, next, next) {
452 bdrv_drain_assert_idle(child->bs);
453 }
454 }
455
456 unsigned int bdrv_drain_all_count = 0;
457
458 static bool bdrv_drain_all_poll(void)
459 {
460 BlockDriverState *bs = NULL;
461 bool result = false;
462 GLOBAL_STATE_CODE();
463
464 /* bdrv_drain_poll() can't make changes to the graph and we are holding the
465 * main AioContext lock, so iterating bdrv_next_all_states() is safe. */
466 while ((bs = bdrv_next_all_states(bs))) {
467 AioContext *aio_context = bdrv_get_aio_context(bs);
468 aio_context_acquire(aio_context);
469 result |= bdrv_drain_poll(bs, NULL, true);
470 aio_context_release(aio_context);
471 }
472
473 return result;
474 }
475
476 /*
477 * Wait for pending requests to complete across all BlockDriverStates
478 *
479 * This function does not flush data to disk, use bdrv_flush_all() for that
480 * after calling this function.
481 *
482 * This pauses all block jobs and disables external clients. It must
483 * be paired with bdrv_drain_all_end().
484 *
485 * NOTE: no new block jobs or BlockDriverStates can be created between
486 * the bdrv_drain_all_begin() and bdrv_drain_all_end() calls.
487 */
488 void bdrv_drain_all_begin(void)
489 {
490 BlockDriverState *bs = NULL;
491 GLOBAL_STATE_CODE();
492
493 if (qemu_in_coroutine()) {
494 bdrv_co_yield_to_drain(NULL, true, NULL, true, true);
495 return;
496 }
497
498 /*
499 * bdrv queue is managed by record/replay,
500 * waiting for finishing the I/O requests may
501 * be infinite
502 */
503 if (replay_events_enabled()) {
504 return;
505 }
506
507 /* AIO_WAIT_WHILE() with a NULL context can only be called from the main
508 * loop AioContext, so make sure we're in the main context. */
509 assert(qemu_get_current_aio_context() == qemu_get_aio_context());
510 assert(bdrv_drain_all_count < INT_MAX);
511 bdrv_drain_all_count++;
512
513 /* Quiesce all nodes, without polling in-flight requests yet. The graph
514 * cannot change during this loop. */
515 while ((bs = bdrv_next_all_states(bs))) {
516 AioContext *aio_context = bdrv_get_aio_context(bs);
517
518 aio_context_acquire(aio_context);
519 bdrv_do_drained_begin(bs, NULL, true, false);
520 aio_context_release(aio_context);
521 }
522
523 /* Now poll the in-flight requests */
524 AIO_WAIT_WHILE(NULL, bdrv_drain_all_poll());
525
526 while ((bs = bdrv_next_all_states(bs))) {
527 bdrv_drain_assert_idle(bs);
528 }
529 }
530
531 void bdrv_drain_all_end_quiesce(BlockDriverState *bs)
532 {
533 GLOBAL_STATE_CODE();
534
535 g_assert(bs->quiesce_counter > 0);
536 g_assert(!bs->refcnt);
537
538 while (bs->quiesce_counter) {
539 bdrv_do_drained_end(bs, NULL, true);
540 }
541 }
542
543 void bdrv_drain_all_end(void)
544 {
545 BlockDriverState *bs = NULL;
546 GLOBAL_STATE_CODE();
547
548 /*
549 * bdrv queue is managed by record/replay,
550 * waiting for finishing the I/O requests may
551 * be endless
552 */
553 if (replay_events_enabled()) {
554 return;
555 }
556
557 while ((bs = bdrv_next_all_states(bs))) {
558 AioContext *aio_context = bdrv_get_aio_context(bs);
559
560 aio_context_acquire(aio_context);
561 bdrv_do_drained_end(bs, NULL, true);
562 aio_context_release(aio_context);
563 }
564
565 assert(qemu_get_current_aio_context() == qemu_get_aio_context());
566 assert(bdrv_drain_all_count > 0);
567 bdrv_drain_all_count--;
568 }
569
570 void bdrv_drain_all(void)
571 {
572 GLOBAL_STATE_CODE();
573 bdrv_drain_all_begin();
574 bdrv_drain_all_end();
575 }
576
577 /**
578 * Remove an active request from the tracked requests list
579 *
580 * This function should be called when a tracked request is completing.
581 */
582 static void coroutine_fn tracked_request_end(BdrvTrackedRequest *req)
583 {
584 if (req->serialising) {
585 qatomic_dec(&req->bs->serialising_in_flight);
586 }
587
588 qemu_co_mutex_lock(&req->bs->reqs_lock);
589 QLIST_REMOVE(req, list);
590 qemu_co_queue_restart_all(&req->wait_queue);
591 qemu_co_mutex_unlock(&req->bs->reqs_lock);
592 }
593
594 /**
595 * Add an active request to the tracked requests list
596 */
597 static void coroutine_fn tracked_request_begin(BdrvTrackedRequest *req,
598 BlockDriverState *bs,
599 int64_t offset,
600 int64_t bytes,
601 enum BdrvTrackedRequestType type)
602 {
603 bdrv_check_request(offset, bytes, &error_abort);
604
605 *req = (BdrvTrackedRequest){
606 .bs = bs,
607 .offset = offset,
608 .bytes = bytes,
609 .type = type,
610 .co = qemu_coroutine_self(),
611 .serialising = false,
612 .overlap_offset = offset,
613 .overlap_bytes = bytes,
614 };
615
616 qemu_co_queue_init(&req->wait_queue);
617
618 qemu_co_mutex_lock(&bs->reqs_lock);
619 QLIST_INSERT_HEAD(&bs->tracked_requests, req, list);
620 qemu_co_mutex_unlock(&bs->reqs_lock);
621 }
622
623 static bool tracked_request_overlaps(BdrvTrackedRequest *req,
624 int64_t offset, int64_t bytes)
625 {
626 bdrv_check_request(offset, bytes, &error_abort);
627
628 /* aaaa bbbb */
629 if (offset >= req->overlap_offset + req->overlap_bytes) {
630 return false;
631 }
632 /* bbbb aaaa */
633 if (req->overlap_offset >= offset + bytes) {
634 return false;
635 }
636 return true;
637 }
638
639 /* Called with self->bs->reqs_lock held */
640 static coroutine_fn BdrvTrackedRequest *
641 bdrv_find_conflicting_request(BdrvTrackedRequest *self)
642 {
643 BdrvTrackedRequest *req;
644
645 QLIST_FOREACH(req, &self->bs->tracked_requests, list) {
646 if (req == self || (!req->serialising && !self->serialising)) {
647 continue;
648 }
649 if (tracked_request_overlaps(req, self->overlap_offset,
650 self->overlap_bytes))
651 {
652 /*
653 * Hitting this means there was a reentrant request, for
654 * example, a block driver issuing nested requests. This must
655 * never happen since it means deadlock.
656 */
657 assert(qemu_coroutine_self() != req->co);
658
659 /*
660 * If the request is already (indirectly) waiting for us, or
661 * will wait for us as soon as it wakes up, then just go on
662 * (instead of producing a deadlock in the former case).
663 */
664 if (!req->waiting_for) {
665 return req;
666 }
667 }
668 }
669
670 return NULL;
671 }
672
673 /* Called with self->bs->reqs_lock held */
674 static void coroutine_fn
675 bdrv_wait_serialising_requests_locked(BdrvTrackedRequest *self)
676 {
677 BdrvTrackedRequest *req;
678
679 while ((req = bdrv_find_conflicting_request(self))) {
680 self->waiting_for = req;
681 qemu_co_queue_wait(&req->wait_queue, &self->bs->reqs_lock);
682 self->waiting_for = NULL;
683 }
684 }
685
686 /* Called with req->bs->reqs_lock held */
687 static void tracked_request_set_serialising(BdrvTrackedRequest *req,
688 uint64_t align)
689 {
690 int64_t overlap_offset = req->offset & ~(align - 1);
691 int64_t overlap_bytes =
692 ROUND_UP(req->offset + req->bytes, align) - overlap_offset;
693
694 bdrv_check_request(req->offset, req->bytes, &error_abort);
695
696 if (!req->serialising) {
697 qatomic_inc(&req->bs->serialising_in_flight);
698 req->serialising = true;
699 }
700
701 req->overlap_offset = MIN(req->overlap_offset, overlap_offset);
702 req->overlap_bytes = MAX(req->overlap_bytes, overlap_bytes);
703 }
704
705 /**
706 * Return the tracked request on @bs for the current coroutine, or
707 * NULL if there is none.
708 */
709 BdrvTrackedRequest *coroutine_fn bdrv_co_get_self_request(BlockDriverState *bs)
710 {
711 BdrvTrackedRequest *req;
712 Coroutine *self = qemu_coroutine_self();
713 IO_CODE();
714
715 QLIST_FOREACH(req, &bs->tracked_requests, list) {
716 if (req->co == self) {
717 return req;
718 }
719 }
720
721 return NULL;
722 }
723
724 /**
725 * Round a region to cluster boundaries
726 */
727 void bdrv_round_to_clusters(BlockDriverState *bs,
728 int64_t offset, int64_t bytes,
729 int64_t *cluster_offset,
730 int64_t *cluster_bytes)
731 {
732 BlockDriverInfo bdi;
733 IO_CODE();
734 if (bdrv_get_info(bs, &bdi) < 0 || bdi.cluster_size == 0) {
735 *cluster_offset = offset;
736 *cluster_bytes = bytes;
737 } else {
738 int64_t c = bdi.cluster_size;
739 *cluster_offset = QEMU_ALIGN_DOWN(offset, c);
740 *cluster_bytes = QEMU_ALIGN_UP(offset - *cluster_offset + bytes, c);
741 }
742 }
743
744 static int bdrv_get_cluster_size(BlockDriverState *bs)
745 {
746 BlockDriverInfo bdi;
747 int ret;
748
749 ret = bdrv_get_info(bs, &bdi);
750 if (ret < 0 || bdi.cluster_size == 0) {
751 return bs->bl.request_alignment;
752 } else {
753 return bdi.cluster_size;
754 }
755 }
756
757 void bdrv_inc_in_flight(BlockDriverState *bs)
758 {
759 IO_CODE();
760 qatomic_inc(&bs->in_flight);
761 }
762
763 void bdrv_wakeup(BlockDriverState *bs)
764 {
765 IO_CODE();
766 aio_wait_kick();
767 }
768
769 void bdrv_dec_in_flight(BlockDriverState *bs)
770 {
771 IO_CODE();
772 qatomic_dec(&bs->in_flight);
773 bdrv_wakeup(bs);
774 }
775
776 static void coroutine_fn
777 bdrv_wait_serialising_requests(BdrvTrackedRequest *self)
778 {
779 BlockDriverState *bs = self->bs;
780
781 if (!qatomic_read(&bs->serialising_in_flight)) {
782 return;
783 }
784
785 qemu_co_mutex_lock(&bs->reqs_lock);
786 bdrv_wait_serialising_requests_locked(self);
787 qemu_co_mutex_unlock(&bs->reqs_lock);
788 }
789
790 void coroutine_fn bdrv_make_request_serialising(BdrvTrackedRequest *req,
791 uint64_t align)
792 {
793 IO_CODE();
794
795 qemu_co_mutex_lock(&req->bs->reqs_lock);
796
797 tracked_request_set_serialising(req, align);
798 bdrv_wait_serialising_requests_locked(req);
799
800 qemu_co_mutex_unlock(&req->bs->reqs_lock);
801 }
802
803 int bdrv_check_qiov_request(int64_t offset, int64_t bytes,
804 QEMUIOVector *qiov, size_t qiov_offset,
805 Error **errp)
806 {
807 /*
808 * Check generic offset/bytes correctness
809 */
810
811 if (offset < 0) {
812 error_setg(errp, "offset is negative: %" PRIi64, offset);
813 return -EIO;
814 }
815
816 if (bytes < 0) {
817 error_setg(errp, "bytes is negative: %" PRIi64, bytes);
818 return -EIO;
819 }
820
821 if (bytes > BDRV_MAX_LENGTH) {
822 error_setg(errp, "bytes(%" PRIi64 ") exceeds maximum(%" PRIi64 ")",
823 bytes, BDRV_MAX_LENGTH);
824 return -EIO;
825 }
826
827 if (offset > BDRV_MAX_LENGTH) {
828 error_setg(errp, "offset(%" PRIi64 ") exceeds maximum(%" PRIi64 ")",
829 offset, BDRV_MAX_LENGTH);
830 return -EIO;
831 }
832
833 if (offset > BDRV_MAX_LENGTH - bytes) {
834 error_setg(errp, "sum of offset(%" PRIi64 ") and bytes(%" PRIi64 ") "
835 "exceeds maximum(%" PRIi64 ")", offset, bytes,
836 BDRV_MAX_LENGTH);
837 return -EIO;
838 }
839
840 if (!qiov) {
841 return 0;
842 }
843
844 /*
845 * Check qiov and qiov_offset
846 */
847
848 if (qiov_offset > qiov->size) {
849 error_setg(errp, "qiov_offset(%zu) overflow io vector size(%zu)",
850 qiov_offset, qiov->size);
851 return -EIO;
852 }
853
854 if (bytes > qiov->size - qiov_offset) {
855 error_setg(errp, "bytes(%" PRIi64 ") + qiov_offset(%zu) overflow io "
856 "vector size(%zu)", bytes, qiov_offset, qiov->size);
857 return -EIO;
858 }
859
860 return 0;
861 }
862
863 int bdrv_check_request(int64_t offset, int64_t bytes, Error **errp)
864 {
865 return bdrv_check_qiov_request(offset, bytes, NULL, 0, errp);
866 }
867
868 static int bdrv_check_request32(int64_t offset, int64_t bytes,
869 QEMUIOVector *qiov, size_t qiov_offset)
870 {
871 int ret = bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, NULL);
872 if (ret < 0) {
873 return ret;
874 }
875
876 if (bytes > BDRV_REQUEST_MAX_BYTES) {
877 return -EIO;
878 }
879
880 return 0;
881 }
882
883 /*
884 * Completely zero out a block device with the help of bdrv_pwrite_zeroes.
885 * The operation is sped up by checking the block status and only writing
886 * zeroes to the device if they currently do not return zeroes. Optional
887 * flags are passed through to bdrv_pwrite_zeroes (e.g. BDRV_REQ_MAY_UNMAP,
888 * BDRV_REQ_FUA).
889 *
890 * Returns < 0 on error, 0 on success. For error codes see bdrv_pwrite().
891 */
892 int bdrv_make_zero(BdrvChild *child, BdrvRequestFlags flags)
893 {
894 int ret;
895 int64_t target_size, bytes, offset = 0;
896 BlockDriverState *bs = child->bs;
897 IO_CODE();
898
899 target_size = bdrv_getlength(bs);
900 if (target_size < 0) {
901 return target_size;
902 }
903
904 for (;;) {
905 bytes = MIN(target_size - offset, BDRV_REQUEST_MAX_BYTES);
906 if (bytes <= 0) {
907 return 0;
908 }
909 ret = bdrv_block_status(bs, offset, bytes, &bytes, NULL, NULL);
910 if (ret < 0) {
911 return ret;
912 }
913 if (ret & BDRV_BLOCK_ZERO) {
914 offset += bytes;
915 continue;
916 }
917 ret = bdrv_pwrite_zeroes(child, offset, bytes, flags);
918 if (ret < 0) {
919 return ret;
920 }
921 offset += bytes;
922 }
923 }
924
925 /*
926 * Writes to the file and ensures that no writes are reordered across this
927 * request (acts as a barrier)
928 *
929 * Returns 0 on success, -errno in error cases.
930 */
931 int coroutine_fn bdrv_co_pwrite_sync(BdrvChild *child, int64_t offset,
932 int64_t bytes, const void *buf,
933 BdrvRequestFlags flags)
934 {
935 int ret;
936 IO_CODE();
937
938 ret = bdrv_co_pwrite(child, offset, bytes, buf, flags);
939 if (ret < 0) {
940 return ret;
941 }
942
943 ret = bdrv_co_flush(child->bs);
944 if (ret < 0) {
945 return ret;
946 }
947
948 return 0;
949 }
950
951 typedef struct CoroutineIOCompletion {
952 Coroutine *coroutine;
953 int ret;
954 } CoroutineIOCompletion;
955
956 static void bdrv_co_io_em_complete(void *opaque, int ret)
957 {
958 CoroutineIOCompletion *co = opaque;
959
960 co->ret = ret;
961 aio_co_wake(co->coroutine);
962 }
963
964 static int coroutine_fn bdrv_driver_preadv(BlockDriverState *bs,
965 int64_t offset, int64_t bytes,
966 QEMUIOVector *qiov,
967 size_t qiov_offset, int flags)
968 {
969 BlockDriver *drv = bs->drv;
970 int64_t sector_num;
971 unsigned int nb_sectors;
972 QEMUIOVector local_qiov;
973 int ret;
974
975 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
976 assert(!(flags & ~bs->supported_read_flags));
977
978 if (!drv) {
979 return -ENOMEDIUM;
980 }
981
982 if (drv->bdrv_co_preadv_part) {
983 return drv->bdrv_co_preadv_part(bs, offset, bytes, qiov, qiov_offset,
984 flags);
985 }
986
987 if (qiov_offset > 0 || bytes != qiov->size) {
988 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
989 qiov = &local_qiov;
990 }
991
992 if (drv->bdrv_co_preadv) {
993 ret = drv->bdrv_co_preadv(bs, offset, bytes, qiov, flags);
994 goto out;
995 }
996
997 if (drv->bdrv_aio_preadv) {
998 BlockAIOCB *acb;
999 CoroutineIOCompletion co = {
1000 .coroutine = qemu_coroutine_self(),
1001 };
1002
1003 acb = drv->bdrv_aio_preadv(bs, offset, bytes, qiov, flags,
1004 bdrv_co_io_em_complete, &co);
1005 if (acb == NULL) {
1006 ret = -EIO;
1007 goto out;
1008 } else {
1009 qemu_coroutine_yield();
1010 ret = co.ret;
1011 goto out;
1012 }
1013 }
1014
1015 sector_num = offset >> BDRV_SECTOR_BITS;
1016 nb_sectors = bytes >> BDRV_SECTOR_BITS;
1017
1018 assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
1019 assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
1020 assert(bytes <= BDRV_REQUEST_MAX_BYTES);
1021 assert(drv->bdrv_co_readv);
1022
1023 ret = drv->bdrv_co_readv(bs, sector_num, nb_sectors, qiov);
1024
1025 out:
1026 if (qiov == &local_qiov) {
1027 qemu_iovec_destroy(&local_qiov);
1028 }
1029
1030 return ret;
1031 }
1032
1033 static int coroutine_fn bdrv_driver_pwritev(BlockDriverState *bs,
1034 int64_t offset, int64_t bytes,
1035 QEMUIOVector *qiov,
1036 size_t qiov_offset,
1037 BdrvRequestFlags flags)
1038 {
1039 BlockDriver *drv = bs->drv;
1040 bool emulate_fua = false;
1041 int64_t sector_num;
1042 unsigned int nb_sectors;
1043 QEMUIOVector local_qiov;
1044 int ret;
1045
1046 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1047
1048 if (!drv) {
1049 return -ENOMEDIUM;
1050 }
1051
1052 if ((flags & BDRV_REQ_FUA) &&
1053 (~bs->supported_write_flags & BDRV_REQ_FUA)) {
1054 flags &= ~BDRV_REQ_FUA;
1055 emulate_fua = true;
1056 }
1057
1058 flags &= bs->supported_write_flags;
1059
1060 if (drv->bdrv_co_pwritev_part) {
1061 ret = drv->bdrv_co_pwritev_part(bs, offset, bytes, qiov, qiov_offset,
1062 flags);
1063 goto emulate_flags;
1064 }
1065
1066 if (qiov_offset > 0 || bytes != qiov->size) {
1067 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1068 qiov = &local_qiov;
1069 }
1070
1071 if (drv->bdrv_co_pwritev) {
1072 ret = drv->bdrv_co_pwritev(bs, offset, bytes, qiov, flags);
1073 goto emulate_flags;
1074 }
1075
1076 if (drv->bdrv_aio_pwritev) {
1077 BlockAIOCB *acb;
1078 CoroutineIOCompletion co = {
1079 .coroutine = qemu_coroutine_self(),
1080 };
1081
1082 acb = drv->bdrv_aio_pwritev(bs, offset, bytes, qiov, flags,
1083 bdrv_co_io_em_complete, &co);
1084 if (acb == NULL) {
1085 ret = -EIO;
1086 } else {
1087 qemu_coroutine_yield();
1088 ret = co.ret;
1089 }
1090 goto emulate_flags;
1091 }
1092
1093 sector_num = offset >> BDRV_SECTOR_BITS;
1094 nb_sectors = bytes >> BDRV_SECTOR_BITS;
1095
1096 assert(QEMU_IS_ALIGNED(offset, BDRV_SECTOR_SIZE));
1097 assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
1098 assert(bytes <= BDRV_REQUEST_MAX_BYTES);
1099
1100 assert(drv->bdrv_co_writev);
1101 ret = drv->bdrv_co_writev(bs, sector_num, nb_sectors, qiov, flags);
1102
1103 emulate_flags:
1104 if (ret == 0 && emulate_fua) {
1105 ret = bdrv_co_flush(bs);
1106 }
1107
1108 if (qiov == &local_qiov) {
1109 qemu_iovec_destroy(&local_qiov);
1110 }
1111
1112 return ret;
1113 }
1114
1115 static int coroutine_fn
1116 bdrv_driver_pwritev_compressed(BlockDriverState *bs, int64_t offset,
1117 int64_t bytes, QEMUIOVector *qiov,
1118 size_t qiov_offset)
1119 {
1120 BlockDriver *drv = bs->drv;
1121 QEMUIOVector local_qiov;
1122 int ret;
1123
1124 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1125
1126 if (!drv) {
1127 return -ENOMEDIUM;
1128 }
1129
1130 if (!block_driver_can_compress(drv)) {
1131 return -ENOTSUP;
1132 }
1133
1134 if (drv->bdrv_co_pwritev_compressed_part) {
1135 return drv->bdrv_co_pwritev_compressed_part(bs, offset, bytes,
1136 qiov, qiov_offset);
1137 }
1138
1139 if (qiov_offset == 0) {
1140 return drv->bdrv_co_pwritev_compressed(bs, offset, bytes, qiov);
1141 }
1142
1143 qemu_iovec_init_slice(&local_qiov, qiov, qiov_offset, bytes);
1144 ret = drv->bdrv_co_pwritev_compressed(bs, offset, bytes, &local_qiov);
1145 qemu_iovec_destroy(&local_qiov);
1146
1147 return ret;
1148 }
1149
1150 static int coroutine_fn bdrv_co_do_copy_on_readv(BdrvChild *child,
1151 int64_t offset, int64_t bytes, QEMUIOVector *qiov,
1152 size_t qiov_offset, int flags)
1153 {
1154 BlockDriverState *bs = child->bs;
1155
1156 /* Perform I/O through a temporary buffer so that users who scribble over
1157 * their read buffer while the operation is in progress do not end up
1158 * modifying the image file. This is critical for zero-copy guest I/O
1159 * where anything might happen inside guest memory.
1160 */
1161 void *bounce_buffer = NULL;
1162
1163 BlockDriver *drv = bs->drv;
1164 int64_t cluster_offset;
1165 int64_t cluster_bytes;
1166 int64_t skip_bytes;
1167 int ret;
1168 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer,
1169 BDRV_REQUEST_MAX_BYTES);
1170 int64_t progress = 0;
1171 bool skip_write;
1172
1173 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1174
1175 if (!drv) {
1176 return -ENOMEDIUM;
1177 }
1178
1179 /*
1180 * Do not write anything when the BDS is inactive. That is not
1181 * allowed, and it would not help.
1182 */
1183 skip_write = (bs->open_flags & BDRV_O_INACTIVE);
1184
1185 /* FIXME We cannot require callers to have write permissions when all they
1186 * are doing is a read request. If we did things right, write permissions
1187 * would be obtained anyway, but internally by the copy-on-read code. As
1188 * long as it is implemented here rather than in a separate filter driver,
1189 * the copy-on-read code doesn't have its own BdrvChild, however, for which
1190 * it could request permissions. Therefore we have to bypass the permission
1191 * system for the moment. */
1192 // assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE));
1193
1194 /* Cover entire cluster so no additional backing file I/O is required when
1195 * allocating cluster in the image file. Note that this value may exceed
1196 * BDRV_REQUEST_MAX_BYTES (even when the original read did not), which
1197 * is one reason we loop rather than doing it all at once.
1198 */
1199 bdrv_round_to_clusters(bs, offset, bytes, &cluster_offset, &cluster_bytes);
1200 skip_bytes = offset - cluster_offset;
1201
1202 trace_bdrv_co_do_copy_on_readv(bs, offset, bytes,
1203 cluster_offset, cluster_bytes);
1204
1205 while (cluster_bytes) {
1206 int64_t pnum;
1207
1208 if (skip_write) {
1209 ret = 1; /* "already allocated", so nothing will be copied */
1210 pnum = MIN(cluster_bytes, max_transfer);
1211 } else {
1212 ret = bdrv_is_allocated(bs, cluster_offset,
1213 MIN(cluster_bytes, max_transfer), &pnum);
1214 if (ret < 0) {
1215 /*
1216 * Safe to treat errors in querying allocation as if
1217 * unallocated; we'll probably fail again soon on the
1218 * read, but at least that will set a decent errno.
1219 */
1220 pnum = MIN(cluster_bytes, max_transfer);
1221 }
1222
1223 /* Stop at EOF if the image ends in the middle of the cluster */
1224 if (ret == 0 && pnum == 0) {
1225 assert(progress >= bytes);
1226 break;
1227 }
1228
1229 assert(skip_bytes < pnum);
1230 }
1231
1232 if (ret <= 0) {
1233 QEMUIOVector local_qiov;
1234
1235 /* Must copy-on-read; use the bounce buffer */
1236 pnum = MIN(pnum, MAX_BOUNCE_BUFFER);
1237 if (!bounce_buffer) {
1238 int64_t max_we_need = MAX(pnum, cluster_bytes - pnum);
1239 int64_t max_allowed = MIN(max_transfer, MAX_BOUNCE_BUFFER);
1240 int64_t bounce_buffer_len = MIN(max_we_need, max_allowed);
1241
1242 bounce_buffer = qemu_try_blockalign(bs, bounce_buffer_len);
1243 if (!bounce_buffer) {
1244 ret = -ENOMEM;
1245 goto err;
1246 }
1247 }
1248 qemu_iovec_init_buf(&local_qiov, bounce_buffer, pnum);
1249
1250 ret = bdrv_driver_preadv(bs, cluster_offset, pnum,
1251 &local_qiov, 0, 0);
1252 if (ret < 0) {
1253 goto err;
1254 }
1255
1256 bdrv_debug_event(bs, BLKDBG_COR_WRITE);
1257 if (drv->bdrv_co_pwrite_zeroes &&
1258 buffer_is_zero(bounce_buffer, pnum)) {
1259 /* FIXME: Should we (perhaps conditionally) be setting
1260 * BDRV_REQ_MAY_UNMAP, if it will allow for a sparser copy
1261 * that still correctly reads as zero? */
1262 ret = bdrv_co_do_pwrite_zeroes(bs, cluster_offset, pnum,
1263 BDRV_REQ_WRITE_UNCHANGED);
1264 } else {
1265 /* This does not change the data on the disk, it is not
1266 * necessary to flush even in cache=writethrough mode.
1267 */
1268 ret = bdrv_driver_pwritev(bs, cluster_offset, pnum,
1269 &local_qiov, 0,
1270 BDRV_REQ_WRITE_UNCHANGED);
1271 }
1272
1273 if (ret < 0) {
1274 /* It might be okay to ignore write errors for guest
1275 * requests. If this is a deliberate copy-on-read
1276 * then we don't want to ignore the error. Simply
1277 * report it in all cases.
1278 */
1279 goto err;
1280 }
1281
1282 if (!(flags & BDRV_REQ_PREFETCH)) {
1283 qemu_iovec_from_buf(qiov, qiov_offset + progress,
1284 bounce_buffer + skip_bytes,
1285 MIN(pnum - skip_bytes, bytes - progress));
1286 }
1287 } else if (!(flags & BDRV_REQ_PREFETCH)) {
1288 /* Read directly into the destination */
1289 ret = bdrv_driver_preadv(bs, offset + progress,
1290 MIN(pnum - skip_bytes, bytes - progress),
1291 qiov, qiov_offset + progress, 0);
1292 if (ret < 0) {
1293 goto err;
1294 }
1295 }
1296
1297 cluster_offset += pnum;
1298 cluster_bytes -= pnum;
1299 progress += pnum - skip_bytes;
1300 skip_bytes = 0;
1301 }
1302 ret = 0;
1303
1304 err:
1305 qemu_vfree(bounce_buffer);
1306 return ret;
1307 }
1308
1309 /*
1310 * Forwards an already correctly aligned request to the BlockDriver. This
1311 * handles copy on read, zeroing after EOF, and fragmentation of large
1312 * reads; any other features must be implemented by the caller.
1313 */
1314 static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child,
1315 BdrvTrackedRequest *req, int64_t offset, int64_t bytes,
1316 int64_t align, QEMUIOVector *qiov, size_t qiov_offset, int flags)
1317 {
1318 BlockDriverState *bs = child->bs;
1319 int64_t total_bytes, max_bytes;
1320 int ret = 0;
1321 int64_t bytes_remaining = bytes;
1322 int max_transfer;
1323
1324 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1325 assert(is_power_of_2(align));
1326 assert((offset & (align - 1)) == 0);
1327 assert((bytes & (align - 1)) == 0);
1328 assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1329 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
1330 align);
1331
1332 /*
1333 * TODO: We would need a per-BDS .supported_read_flags and
1334 * potential fallback support, if we ever implement any read flags
1335 * to pass through to drivers. For now, there aren't any
1336 * passthrough flags except the BDRV_REQ_REGISTERED_BUF optimization hint.
1337 */
1338 assert(!(flags & ~(BDRV_REQ_COPY_ON_READ | BDRV_REQ_PREFETCH |
1339 BDRV_REQ_REGISTERED_BUF)));
1340
1341 /* Handle Copy on Read and associated serialisation */
1342 if (flags & BDRV_REQ_COPY_ON_READ) {
1343 /* If we touch the same cluster it counts as an overlap. This
1344 * guarantees that allocating writes will be serialized and not race
1345 * with each other for the same cluster. For example, in copy-on-read
1346 * it ensures that the CoR read and write operations are atomic and
1347 * guest writes cannot interleave between them. */
1348 bdrv_make_request_serialising(req, bdrv_get_cluster_size(bs));
1349 } else {
1350 bdrv_wait_serialising_requests(req);
1351 }
1352
1353 if (flags & BDRV_REQ_COPY_ON_READ) {
1354 int64_t pnum;
1355
1356 /* The flag BDRV_REQ_COPY_ON_READ has reached its addressee */
1357 flags &= ~BDRV_REQ_COPY_ON_READ;
1358
1359 ret = bdrv_is_allocated(bs, offset, bytes, &pnum);
1360 if (ret < 0) {
1361 goto out;
1362 }
1363
1364 if (!ret || pnum != bytes) {
1365 ret = bdrv_co_do_copy_on_readv(child, offset, bytes,
1366 qiov, qiov_offset, flags);
1367 goto out;
1368 } else if (flags & BDRV_REQ_PREFETCH) {
1369 goto out;
1370 }
1371 }
1372
1373 /* Forward the request to the BlockDriver, possibly fragmenting it */
1374 total_bytes = bdrv_getlength(bs);
1375 if (total_bytes < 0) {
1376 ret = total_bytes;
1377 goto out;
1378 }
1379
1380 assert(!(flags & ~(bs->supported_read_flags | BDRV_REQ_REGISTERED_BUF)));
1381
1382 max_bytes = ROUND_UP(MAX(0, total_bytes - offset), align);
1383 if (bytes <= max_bytes && bytes <= max_transfer) {
1384 ret = bdrv_driver_preadv(bs, offset, bytes, qiov, qiov_offset, flags);
1385 goto out;
1386 }
1387
1388 while (bytes_remaining) {
1389 int64_t num;
1390
1391 if (max_bytes) {
1392 num = MIN(bytes_remaining, MIN(max_bytes, max_transfer));
1393 assert(num);
1394
1395 ret = bdrv_driver_preadv(bs, offset + bytes - bytes_remaining,
1396 num, qiov,
1397 qiov_offset + bytes - bytes_remaining,
1398 flags);
1399 max_bytes -= num;
1400 } else {
1401 num = bytes_remaining;
1402 ret = qemu_iovec_memset(qiov, qiov_offset + bytes - bytes_remaining,
1403 0, bytes_remaining);
1404 }
1405 if (ret < 0) {
1406 goto out;
1407 }
1408 bytes_remaining -= num;
1409 }
1410
1411 out:
1412 return ret < 0 ? ret : 0;
1413 }
1414
1415 /*
1416 * Request padding
1417 *
1418 * |<---- align ----->| |<----- align ---->|
1419 * |<- head ->|<------------- bytes ------------->|<-- tail -->|
1420 * | | | | | |
1421 * -*----------$-------*-------- ... --------*-----$------------*---
1422 * | | | | | |
1423 * | offset | | end |
1424 * ALIGN_DOWN(offset) ALIGN_UP(offset) ALIGN_DOWN(end) ALIGN_UP(end)
1425 * [buf ... ) [tail_buf )
1426 *
1427 * @buf is an aligned allocation needed to store @head and @tail paddings. @head
1428 * is placed at the beginning of @buf and @tail at the @end.
1429 *
1430 * @tail_buf is a pointer to sub-buffer, corresponding to align-sized chunk
1431 * around tail, if tail exists.
1432 *
1433 * @merge_reads is true for small requests,
1434 * if @buf_len == @head + bytes + @tail. In this case it is possible that both
1435 * head and tail exist but @buf_len == align and @tail_buf == @buf.
1436 */
1437 typedef struct BdrvRequestPadding {
1438 uint8_t *buf;
1439 size_t buf_len;
1440 uint8_t *tail_buf;
1441 size_t head;
1442 size_t tail;
1443 bool merge_reads;
1444 QEMUIOVector local_qiov;
1445 } BdrvRequestPadding;
1446
1447 static bool bdrv_init_padding(BlockDriverState *bs,
1448 int64_t offset, int64_t bytes,
1449 BdrvRequestPadding *pad)
1450 {
1451 int64_t align = bs->bl.request_alignment;
1452 int64_t sum;
1453
1454 bdrv_check_request(offset, bytes, &error_abort);
1455 assert(align <= INT_MAX); /* documented in block/block_int.h */
1456 assert(align <= SIZE_MAX / 2); /* so we can allocate the buffer */
1457
1458 memset(pad, 0, sizeof(*pad));
1459
1460 pad->head = offset & (align - 1);
1461 pad->tail = ((offset + bytes) & (align - 1));
1462 if (pad->tail) {
1463 pad->tail = align - pad->tail;
1464 }
1465
1466 if (!pad->head && !pad->tail) {
1467 return false;
1468 }
1469
1470 assert(bytes); /* Nothing good in aligning zero-length requests */
1471
1472 sum = pad->head + bytes + pad->tail;
1473 pad->buf_len = (sum > align && pad->head && pad->tail) ? 2 * align : align;
1474 pad->buf = qemu_blockalign(bs, pad->buf_len);
1475 pad->merge_reads = sum == pad->buf_len;
1476 if (pad->tail) {
1477 pad->tail_buf = pad->buf + pad->buf_len - align;
1478 }
1479
1480 return true;
1481 }
1482
1483 static coroutine_fn int bdrv_padding_rmw_read(BdrvChild *child,
1484 BdrvTrackedRequest *req,
1485 BdrvRequestPadding *pad,
1486 bool zero_middle)
1487 {
1488 QEMUIOVector local_qiov;
1489 BlockDriverState *bs = child->bs;
1490 uint64_t align = bs->bl.request_alignment;
1491 int ret;
1492
1493 assert(req->serialising && pad->buf);
1494
1495 if (pad->head || pad->merge_reads) {
1496 int64_t bytes = pad->merge_reads ? pad->buf_len : align;
1497
1498 qemu_iovec_init_buf(&local_qiov, pad->buf, bytes);
1499
1500 if (pad->head) {
1501 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
1502 }
1503 if (pad->merge_reads && pad->tail) {
1504 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1505 }
1506 ret = bdrv_aligned_preadv(child, req, req->overlap_offset, bytes,
1507 align, &local_qiov, 0, 0);
1508 if (ret < 0) {
1509 return ret;
1510 }
1511 if (pad->head) {
1512 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
1513 }
1514 if (pad->merge_reads && pad->tail) {
1515 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1516 }
1517
1518 if (pad->merge_reads) {
1519 goto zero_mem;
1520 }
1521 }
1522
1523 if (pad->tail) {
1524 qemu_iovec_init_buf(&local_qiov, pad->tail_buf, align);
1525
1526 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
1527 ret = bdrv_aligned_preadv(
1528 child, req,
1529 req->overlap_offset + req->overlap_bytes - align,
1530 align, align, &local_qiov, 0, 0);
1531 if (ret < 0) {
1532 return ret;
1533 }
1534 bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
1535 }
1536
1537 zero_mem:
1538 if (zero_middle) {
1539 memset(pad->buf + pad->head, 0, pad->buf_len - pad->head - pad->tail);
1540 }
1541
1542 return 0;
1543 }
1544
1545 static void bdrv_padding_destroy(BdrvRequestPadding *pad)
1546 {
1547 if (pad->buf) {
1548 qemu_vfree(pad->buf);
1549 qemu_iovec_destroy(&pad->local_qiov);
1550 }
1551 memset(pad, 0, sizeof(*pad));
1552 }
1553
1554 /*
1555 * bdrv_pad_request
1556 *
1557 * Exchange request parameters with padded request if needed. Don't include RMW
1558 * read of padding, bdrv_padding_rmw_read() should be called separately if
1559 * needed.
1560 *
1561 * Request parameters (@qiov, &qiov_offset, &offset, &bytes) are in-out:
1562 * - on function start they represent original request
1563 * - on failure or when padding is not needed they are unchanged
1564 * - on success when padding is needed they represent padded request
1565 */
1566 static int bdrv_pad_request(BlockDriverState *bs,
1567 QEMUIOVector **qiov, size_t *qiov_offset,
1568 int64_t *offset, int64_t *bytes,
1569 BdrvRequestPadding *pad, bool *padded,
1570 BdrvRequestFlags *flags)
1571 {
1572 int ret;
1573
1574 bdrv_check_qiov_request(*offset, *bytes, *qiov, *qiov_offset, &error_abort);
1575
1576 if (!bdrv_init_padding(bs, *offset, *bytes, pad)) {
1577 if (padded) {
1578 *padded = false;
1579 }
1580 return 0;
1581 }
1582
1583 ret = qemu_iovec_init_extended(&pad->local_qiov, pad->buf, pad->head,
1584 *qiov, *qiov_offset, *bytes,
1585 pad->buf + pad->buf_len - pad->tail,
1586 pad->tail);
1587 if (ret < 0) {
1588 bdrv_padding_destroy(pad);
1589 return ret;
1590 }
1591 *bytes += pad->head + pad->tail;
1592 *offset -= pad->head;
1593 *qiov = &pad->local_qiov;
1594 *qiov_offset = 0;
1595 if (padded) {
1596 *padded = true;
1597 }
1598 if (flags) {
1599 /* Can't use optimization hint with bounce buffer */
1600 *flags &= ~BDRV_REQ_REGISTERED_BUF;
1601 }
1602
1603 return 0;
1604 }
1605
1606 int coroutine_fn bdrv_co_preadv(BdrvChild *child,
1607 int64_t offset, int64_t bytes, QEMUIOVector *qiov,
1608 BdrvRequestFlags flags)
1609 {
1610 IO_CODE();
1611 return bdrv_co_preadv_part(child, offset, bytes, qiov, 0, flags);
1612 }
1613
1614 int coroutine_fn bdrv_co_preadv_part(BdrvChild *child,
1615 int64_t offset, int64_t bytes,
1616 QEMUIOVector *qiov, size_t qiov_offset,
1617 BdrvRequestFlags flags)
1618 {
1619 BlockDriverState *bs = child->bs;
1620 BdrvTrackedRequest req;
1621 BdrvRequestPadding pad;
1622 int ret;
1623 IO_CODE();
1624
1625 trace_bdrv_co_preadv_part(bs, offset, bytes, flags);
1626
1627 if (!bdrv_is_inserted(bs)) {
1628 return -ENOMEDIUM;
1629 }
1630
1631 ret = bdrv_check_request32(offset, bytes, qiov, qiov_offset);
1632 if (ret < 0) {
1633 return ret;
1634 }
1635
1636 if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) {
1637 /*
1638 * Aligning zero request is nonsense. Even if driver has special meaning
1639 * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass
1640 * it to driver due to request_alignment.
1641 *
1642 * Still, no reason to return an error if someone do unaligned
1643 * zero-length read occasionally.
1644 */
1645 return 0;
1646 }
1647
1648 bdrv_inc_in_flight(bs);
1649
1650 /* Don't do copy-on-read if we read data before write operation */
1651 if (qatomic_read(&bs->copy_on_read)) {
1652 flags |= BDRV_REQ_COPY_ON_READ;
1653 }
1654
1655 ret = bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad,
1656 NULL, &flags);
1657 if (ret < 0) {
1658 goto fail;
1659 }
1660
1661 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
1662 ret = bdrv_aligned_preadv(child, &req, offset, bytes,
1663 bs->bl.request_alignment,
1664 qiov, qiov_offset, flags);
1665 tracked_request_end(&req);
1666 bdrv_padding_destroy(&pad);
1667
1668 fail:
1669 bdrv_dec_in_flight(bs);
1670
1671 return ret;
1672 }
1673
1674 static int coroutine_fn bdrv_co_do_pwrite_zeroes(BlockDriverState *bs,
1675 int64_t offset, int64_t bytes, BdrvRequestFlags flags)
1676 {
1677 BlockDriver *drv = bs->drv;
1678 QEMUIOVector qiov;
1679 void *buf = NULL;
1680 int ret = 0;
1681 bool need_flush = false;
1682 int head = 0;
1683 int tail = 0;
1684
1685 int64_t max_write_zeroes = MIN_NON_ZERO(bs->bl.max_pwrite_zeroes,
1686 INT64_MAX);
1687 int alignment = MAX(bs->bl.pwrite_zeroes_alignment,
1688 bs->bl.request_alignment);
1689 int max_transfer = MIN_NON_ZERO(bs->bl.max_transfer, MAX_BOUNCE_BUFFER);
1690
1691 bdrv_check_request(offset, bytes, &error_abort);
1692
1693 if (!drv) {
1694 return -ENOMEDIUM;
1695 }
1696
1697 if ((flags & ~bs->supported_zero_flags) & BDRV_REQ_NO_FALLBACK) {
1698 return -ENOTSUP;
1699 }
1700
1701 /* By definition there is no user buffer so this flag doesn't make sense */
1702 if (flags & BDRV_REQ_REGISTERED_BUF) {
1703 return -EINVAL;
1704 }
1705
1706 /* Invalidate the cached block-status data range if this write overlaps */
1707 bdrv_bsc_invalidate_range(bs, offset, bytes);
1708
1709 assert(alignment % bs->bl.request_alignment == 0);
1710 head = offset % alignment;
1711 tail = (offset + bytes) % alignment;
1712 max_write_zeroes = QEMU_ALIGN_DOWN(max_write_zeroes, alignment);
1713 assert(max_write_zeroes >= bs->bl.request_alignment);
1714
1715 while (bytes > 0 && !ret) {
1716 int64_t num = bytes;
1717
1718 /* Align request. Block drivers can expect the "bulk" of the request
1719 * to be aligned, and that unaligned requests do not cross cluster
1720 * boundaries.
1721 */
1722 if (head) {
1723 /* Make a small request up to the first aligned sector. For
1724 * convenience, limit this request to max_transfer even if
1725 * we don't need to fall back to writes. */
1726 num = MIN(MIN(bytes, max_transfer), alignment - head);
1727 head = (head + num) % alignment;
1728 assert(num < max_write_zeroes);
1729 } else if (tail && num > alignment) {
1730 /* Shorten the request to the last aligned sector. */
1731 num -= tail;
1732 }
1733
1734 /* limit request size */
1735 if (num > max_write_zeroes) {
1736 num = max_write_zeroes;
1737 }
1738
1739 ret = -ENOTSUP;
1740 /* First try the efficient write zeroes operation */
1741 if (drv->bdrv_co_pwrite_zeroes) {
1742 ret = drv->bdrv_co_pwrite_zeroes(bs, offset, num,
1743 flags & bs->supported_zero_flags);
1744 if (ret != -ENOTSUP && (flags & BDRV_REQ_FUA) &&
1745 !(bs->supported_zero_flags & BDRV_REQ_FUA)) {
1746 need_flush = true;
1747 }
1748 } else {
1749 assert(!bs->supported_zero_flags);
1750 }
1751
1752 if (ret == -ENOTSUP && !(flags & BDRV_REQ_NO_FALLBACK)) {
1753 /* Fall back to bounce buffer if write zeroes is unsupported */
1754 BdrvRequestFlags write_flags = flags & ~BDRV_REQ_ZERO_WRITE;
1755
1756 if ((flags & BDRV_REQ_FUA) &&
1757 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1758 /* No need for bdrv_driver_pwrite() to do a fallback
1759 * flush on each chunk; use just one at the end */
1760 write_flags &= ~BDRV_REQ_FUA;
1761 need_flush = true;
1762 }
1763 num = MIN(num, max_transfer);
1764 if (buf == NULL) {
1765 buf = qemu_try_blockalign0(bs, num);
1766 if (buf == NULL) {
1767 ret = -ENOMEM;
1768 goto fail;
1769 }
1770 }
1771 qemu_iovec_init_buf(&qiov, buf, num);
1772
1773 ret = bdrv_driver_pwritev(bs, offset, num, &qiov, 0, write_flags);
1774
1775 /* Keep bounce buffer around if it is big enough for all
1776 * all future requests.
1777 */
1778 if (num < max_transfer) {
1779 qemu_vfree(buf);
1780 buf = NULL;
1781 }
1782 }
1783
1784 offset += num;
1785 bytes -= num;
1786 }
1787
1788 fail:
1789 if (ret == 0 && need_flush) {
1790 ret = bdrv_co_flush(bs);
1791 }
1792 qemu_vfree(buf);
1793 return ret;
1794 }
1795
1796 static inline int coroutine_fn
1797 bdrv_co_write_req_prepare(BdrvChild *child, int64_t offset, int64_t bytes,
1798 BdrvTrackedRequest *req, int flags)
1799 {
1800 BlockDriverState *bs = child->bs;
1801
1802 bdrv_check_request(offset, bytes, &error_abort);
1803
1804 if (bdrv_is_read_only(bs)) {
1805 return -EPERM;
1806 }
1807
1808 assert(!(bs->open_flags & BDRV_O_INACTIVE));
1809 assert((bs->open_flags & BDRV_O_NO_IO) == 0);
1810 assert(!(flags & ~BDRV_REQ_MASK));
1811 assert(!((flags & BDRV_REQ_NO_WAIT) && !(flags & BDRV_REQ_SERIALISING)));
1812
1813 if (flags & BDRV_REQ_SERIALISING) {
1814 QEMU_LOCK_GUARD(&bs->reqs_lock);
1815
1816 tracked_request_set_serialising(req, bdrv_get_cluster_size(bs));
1817
1818 if ((flags & BDRV_REQ_NO_WAIT) && bdrv_find_conflicting_request(req)) {
1819 return -EBUSY;
1820 }
1821
1822 bdrv_wait_serialising_requests_locked(req);
1823 } else {
1824 bdrv_wait_serialising_requests(req);
1825 }
1826
1827 assert(req->overlap_offset <= offset);
1828 assert(offset + bytes <= req->overlap_offset + req->overlap_bytes);
1829 assert(offset + bytes <= bs->total_sectors * BDRV_SECTOR_SIZE ||
1830 child->perm & BLK_PERM_RESIZE);
1831
1832 switch (req->type) {
1833 case BDRV_TRACKED_WRITE:
1834 case BDRV_TRACKED_DISCARD:
1835 if (flags & BDRV_REQ_WRITE_UNCHANGED) {
1836 assert(child->perm & (BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE));
1837 } else {
1838 assert(child->perm & BLK_PERM_WRITE);
1839 }
1840 bdrv_write_threshold_check_write(bs, offset, bytes);
1841 return 0;
1842 case BDRV_TRACKED_TRUNCATE:
1843 assert(child->perm & BLK_PERM_RESIZE);
1844 return 0;
1845 default:
1846 abort();
1847 }
1848 }
1849
1850 static inline void coroutine_fn
1851 bdrv_co_write_req_finish(BdrvChild *child, int64_t offset, int64_t bytes,
1852 BdrvTrackedRequest *req, int ret)
1853 {
1854 int64_t end_sector = DIV_ROUND_UP(offset + bytes, BDRV_SECTOR_SIZE);
1855 BlockDriverState *bs = child->bs;
1856
1857 bdrv_check_request(offset, bytes, &error_abort);
1858
1859 qatomic_inc(&bs->write_gen);
1860
1861 /*
1862 * Discard cannot extend the image, but in error handling cases, such as
1863 * when reverting a qcow2 cluster allocation, the discarded range can pass
1864 * the end of image file, so we cannot assert about BDRV_TRACKED_DISCARD
1865 * here. Instead, just skip it, since semantically a discard request
1866 * beyond EOF cannot expand the image anyway.
1867 */
1868 if (ret == 0 &&
1869 (req->type == BDRV_TRACKED_TRUNCATE ||
1870 end_sector > bs->total_sectors) &&
1871 req->type != BDRV_TRACKED_DISCARD) {
1872 bs->total_sectors = end_sector;
1873 bdrv_parent_cb_resize(bs);
1874 bdrv_dirty_bitmap_truncate(bs, end_sector << BDRV_SECTOR_BITS);
1875 }
1876 if (req->bytes) {
1877 switch (req->type) {
1878 case BDRV_TRACKED_WRITE:
1879 stat64_max(&bs->wr_highest_offset, offset + bytes);
1880 /* fall through, to set dirty bits */
1881 case BDRV_TRACKED_DISCARD:
1882 bdrv_set_dirty(bs, offset, bytes);
1883 break;
1884 default:
1885 break;
1886 }
1887 }
1888 }
1889
1890 /*
1891 * Forwards an already correctly aligned write request to the BlockDriver,
1892 * after possibly fragmenting it.
1893 */
1894 static int coroutine_fn bdrv_aligned_pwritev(BdrvChild *child,
1895 BdrvTrackedRequest *req, int64_t offset, int64_t bytes,
1896 int64_t align, QEMUIOVector *qiov, size_t qiov_offset,
1897 BdrvRequestFlags flags)
1898 {
1899 BlockDriverState *bs = child->bs;
1900 BlockDriver *drv = bs->drv;
1901 int ret;
1902
1903 int64_t bytes_remaining = bytes;
1904 int max_transfer;
1905
1906 bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, &error_abort);
1907
1908 if (!drv) {
1909 return -ENOMEDIUM;
1910 }
1911
1912 if (bdrv_has_readonly_bitmaps(bs)) {
1913 return -EPERM;
1914 }
1915
1916 assert(is_power_of_2(align));
1917 assert((offset & (align - 1)) == 0);
1918 assert((bytes & (align - 1)) == 0);
1919 max_transfer = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_transfer, INT_MAX),
1920 align);
1921
1922 ret = bdrv_co_write_req_prepare(child, offset, bytes, req, flags);
1923
1924 if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
1925 !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_pwrite_zeroes &&
1926 qemu_iovec_is_zero(qiov, qiov_offset, bytes)) {
1927 flags |= BDRV_REQ_ZERO_WRITE;
1928 if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
1929 flags |= BDRV_REQ_MAY_UNMAP;
1930 }
1931 }
1932
1933 if (ret < 0) {
1934 /* Do nothing, write notifier decided to fail this request */
1935 } else if (flags & BDRV_REQ_ZERO_WRITE) {
1936 bdrv_debug_event(bs, BLKDBG_PWRITEV_ZERO);
1937 ret = bdrv_co_do_pwrite_zeroes(bs, offset, bytes, flags);
1938 } else if (flags & BDRV_REQ_WRITE_COMPRESSED) {
1939 ret = bdrv_driver_pwritev_compressed(bs, offset, bytes,
1940 qiov, qiov_offset);
1941 } else if (bytes <= max_transfer) {
1942 bdrv_debug_event(bs, BLKDBG_PWRITEV);
1943 ret = bdrv_driver_pwritev(bs, offset, bytes, qiov, qiov_offset, flags);
1944 } else {
1945 bdrv_debug_event(bs, BLKDBG_PWRITEV);
1946 while (bytes_remaining) {
1947 int num = MIN(bytes_remaining, max_transfer);
1948 int local_flags = flags;
1949
1950 assert(num);
1951 if (num < bytes_remaining && (flags & BDRV_REQ_FUA) &&
1952 !(bs->supported_write_flags & BDRV_REQ_FUA)) {
1953 /* If FUA is going to be emulated by flush, we only
1954 * need to flush on the last iteration */
1955 local_flags &= ~BDRV_REQ_FUA;
1956 }
1957
1958 ret = bdrv_driver_pwritev(bs, offset + bytes - bytes_remaining,
1959 num, qiov,
1960 qiov_offset + bytes - bytes_remaining,
1961 local_flags);
1962 if (ret < 0) {
1963 break;
1964 }
1965 bytes_remaining -= num;
1966 }
1967 }
1968 bdrv_debug_event(bs, BLKDBG_PWRITEV_DONE);
1969
1970 if (ret >= 0) {
1971 ret = 0;
1972 }
1973 bdrv_co_write_req_finish(child, offset, bytes, req, ret);
1974
1975 return ret;
1976 }
1977
1978 static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child,
1979 int64_t offset,
1980 int64_t bytes,
1981 BdrvRequestFlags flags,
1982 BdrvTrackedRequest *req)
1983 {
1984 BlockDriverState *bs = child->bs;
1985 QEMUIOVector local_qiov;
1986 uint64_t align = bs->bl.request_alignment;
1987 int ret = 0;
1988 bool padding;
1989 BdrvRequestPadding pad;
1990
1991 /* This flag doesn't make sense for padding or zero writes */
1992 flags &= ~BDRV_REQ_REGISTERED_BUF;
1993
1994 padding = bdrv_init_padding(bs, offset, bytes, &pad);
1995 if (padding) {
1996 assert(!(flags & BDRV_REQ_NO_WAIT));
1997 bdrv_make_request_serialising(req, align);
1998
1999 bdrv_padding_rmw_read(child, req, &pad, true);
2000
2001 if (pad.head || pad.merge_reads) {
2002 int64_t aligned_offset = offset & ~(align - 1);
2003 int64_t write_bytes = pad.merge_reads ? pad.buf_len : align;
2004
2005 qemu_iovec_init_buf(&local_qiov, pad.buf, write_bytes);
2006 ret = bdrv_aligned_pwritev(child, req, aligned_offset, write_bytes,
2007 align, &local_qiov, 0,
2008 flags & ~BDRV_REQ_ZERO_WRITE);
2009 if (ret < 0 || pad.merge_reads) {
2010 /* Error or all work is done */
2011 goto out;
2012 }
2013 offset += write_bytes - pad.head;
2014 bytes -= write_bytes - pad.head;
2015 }
2016 }
2017
2018 assert(!bytes || (offset & (align - 1)) == 0);
2019 if (bytes >= align) {
2020 /* Write the aligned part in the middle. */
2021 int64_t aligned_bytes = bytes & ~(align - 1);
2022 ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align,
2023 NULL, 0, flags);
2024 if (ret < 0) {
2025 goto out;
2026 }
2027 bytes -= aligned_bytes;
2028 offset += aligned_bytes;
2029 }
2030
2031 assert(!bytes || (offset & (align - 1)) == 0);
2032 if (bytes) {
2033 assert(align == pad.tail + bytes);
2034
2035 qemu_iovec_init_buf(&local_qiov, pad.tail_buf, align);
2036 ret = bdrv_aligned_pwritev(child, req, offset, align, align,
2037 &local_qiov, 0,
2038 flags & ~BDRV_REQ_ZERO_WRITE);
2039 }
2040
2041 out:
2042 bdrv_padding_destroy(&pad);
2043
2044 return ret;
2045 }
2046
2047 /*
2048 * Handle a write request in coroutine context
2049 */
2050 int coroutine_fn bdrv_co_pwritev(BdrvChild *child,
2051 int64_t offset, int64_t bytes, QEMUIOVector *qiov,
2052 BdrvRequestFlags flags)
2053 {
2054 IO_CODE();
2055 return bdrv_co_pwritev_part(child, offset, bytes, qiov, 0, flags);
2056 }
2057
2058 int coroutine_fn bdrv_co_pwritev_part(BdrvChild *child,
2059 int64_t offset, int64_t bytes, QEMUIOVector *qiov, size_t qiov_offset,
2060 BdrvRequestFlags flags)
2061 {
2062 BlockDriverState *bs = child->bs;
2063 BdrvTrackedRequest req;
2064 uint64_t align = bs->bl.request_alignment;
2065 BdrvRequestPadding pad;
2066 int ret;
2067 bool padded = false;
2068 IO_CODE();
2069
2070 trace_bdrv_co_pwritev_part(child->bs, offset, bytes, flags);
2071
2072 if (!bdrv_is_inserted(bs)) {
2073 return -ENOMEDIUM;
2074 }
2075
2076 if (flags & BDRV_REQ_ZERO_WRITE) {
2077 ret = bdrv_check_qiov_request(offset, bytes, qiov, qiov_offset, NULL);
2078 } else {
2079 ret = bdrv_check_request32(offset, bytes, qiov, qiov_offset);
2080 }
2081 if (ret < 0) {
2082 return ret;
2083 }
2084
2085 /* If the request is misaligned then we can't make it efficient */
2086 if ((flags & BDRV_REQ_NO_FALLBACK) &&
2087 !QEMU_IS_ALIGNED(offset | bytes, align))
2088 {
2089 return -ENOTSUP;
2090 }
2091
2092 if (bytes == 0 && !QEMU_IS_ALIGNED(offset, bs->bl.request_alignment)) {
2093 /*
2094 * Aligning zero request is nonsense. Even if driver has special meaning
2095 * of zero-length (like qcow2_co_pwritev_compressed_part), we can't pass
2096 * it to driver due to request_alignment.
2097 *
2098 * Still, no reason to return an error if someone do unaligned
2099 * zero-length write occasionally.
2100 */
2101 return 0;
2102 }
2103
2104 if (!(flags & BDRV_REQ_ZERO_WRITE)) {
2105 /*
2106 * Pad request for following read-modify-write cycle.
2107 * bdrv_co_do_zero_pwritev() does aligning by itself, so, we do
2108 * alignment only if there is no ZERO flag.
2109 */
2110 ret = bdrv_pad_request(bs, &qiov, &qiov_offset, &offset, &bytes, &pad,
2111 &padded, &flags);
2112 if (ret < 0) {
2113 return ret;
2114 }
2115 }
2116
2117 bdrv_inc_in_flight(bs);
2118 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_WRITE);
2119
2120 if (flags & BDRV_REQ_ZERO_WRITE) {
2121 assert(!padded);
2122 ret = bdrv_co_do_zero_pwritev(child, offset, bytes, flags, &req);
2123 goto out;
2124 }
2125
2126 if (padded) {
2127 /*
2128 * Request was unaligned to request_alignment and therefore
2129 * padded. We are going to do read-modify-write, and must
2130 * serialize the request to prevent interactions of the
2131 * widened region with other transactions.
2132 */
2133 assert(!(flags & BDRV_REQ_NO_WAIT));
2134 bdrv_make_request_serialising(&req, align);
2135 bdrv_padding_rmw_read(child, &req, &pad, false);
2136 }
2137
2138 ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align,
2139 qiov, qiov_offset, flags);
2140
2141 bdrv_padding_destroy(&pad);
2142
2143 out:
2144 tracked_request_end(&req);
2145 bdrv_dec_in_flight(bs);
2146
2147 return ret;
2148 }
2149
2150 int coroutine_fn bdrv_co_pwrite_zeroes(BdrvChild *child, int64_t offset,
2151 int64_t bytes, BdrvRequestFlags flags)
2152 {
2153 IO_CODE();
2154 trace_bdrv_co_pwrite_zeroes(child->bs, offset, bytes, flags);
2155
2156 if (!(child->bs->open_flags & BDRV_O_UNMAP)) {
2157 flags &= ~BDRV_REQ_MAY_UNMAP;
2158 }
2159
2160 return bdrv_co_pwritev(child, offset, bytes, NULL,
2161 BDRV_REQ_ZERO_WRITE | flags);
2162 }
2163
2164 /*
2165 * Flush ALL BDSes regardless of if they are reachable via a BlkBackend or not.
2166 */
2167 int bdrv_flush_all(void)
2168 {
2169 BdrvNextIterator it;
2170 BlockDriverState *bs = NULL;
2171 int result = 0;
2172
2173 GLOBAL_STATE_CODE();
2174
2175 /*
2176 * bdrv queue is managed by record/replay,
2177 * creating new flush request for stopping
2178 * the VM may break the determinism
2179 */
2180 if (replay_events_enabled()) {
2181 return result;
2182 }
2183
2184 for (bs = bdrv_first(&it); bs; bs = bdrv_next(&it)) {
2185 AioContext *aio_context = bdrv_get_aio_context(bs);
2186 int ret;
2187
2188 aio_context_acquire(aio_context);
2189 ret = bdrv_flush(bs);
2190 if (ret < 0 && !result) {
2191 result = ret;
2192 }
2193 aio_context_release(aio_context);
2194 }
2195
2196 return result;
2197 }
2198
2199 /*
2200 * Returns the allocation status of the specified sectors.
2201 * Drivers not implementing the functionality are assumed to not support
2202 * backing files, hence all their sectors are reported as allocated.
2203 *
2204 * If 'want_zero' is true, the caller is querying for mapping
2205 * purposes, with a focus on valid BDRV_BLOCK_OFFSET_VALID, _DATA, and
2206 * _ZERO where possible; otherwise, the result favors larger 'pnum',
2207 * with a focus on accurate BDRV_BLOCK_ALLOCATED.
2208 *
2209 * If 'offset' is beyond the end of the disk image the return value is
2210 * BDRV_BLOCK_EOF and 'pnum' is set to 0.
2211 *
2212 * 'bytes' is the max value 'pnum' should be set to. If bytes goes
2213 * beyond the end of the disk image it will be clamped; if 'pnum' is set to
2214 * the end of the image, then the returned value will include BDRV_BLOCK_EOF.
2215 *
2216 * 'pnum' is set to the number of bytes (including and immediately
2217 * following the specified offset) that are easily known to be in the
2218 * same allocated/unallocated state. Note that a second call starting
2219 * at the original offset plus returned pnum may have the same status.
2220 * The returned value is non-zero on success except at end-of-file.
2221 *
2222 * Returns negative errno on failure. Otherwise, if the
2223 * BDRV_BLOCK_OFFSET_VALID bit is set, 'map' and 'file' (if non-NULL) are
2224 * set to the host mapping and BDS corresponding to the guest offset.
2225 */
2226 static int coroutine_fn bdrv_co_block_status(BlockDriverState *bs,
2227 bool want_zero,
2228 int64_t offset, int64_t bytes,
2229 int64_t *pnum, int64_t *map,
2230 BlockDriverState **file)
2231 {
2232 int64_t total_size;
2233 int64_t n; /* bytes */
2234 int ret;
2235 int64_t local_map = 0;
2236 BlockDriverState *local_file = NULL;
2237 int64_t aligned_offset, aligned_bytes;
2238 uint32_t align;
2239 bool has_filtered_child;
2240
2241 assert(pnum);
2242 *pnum = 0;
2243 total_size = bdrv_getlength(bs);
2244 if (total_size < 0) {
2245 ret = total_size;
2246 goto early_out;
2247 }
2248
2249 if (offset >= total_size) {
2250 ret = BDRV_BLOCK_EOF;
2251 goto early_out;
2252 }
2253 if (!bytes) {
2254 ret = 0;
2255 goto early_out;
2256 }
2257
2258 n = total_size - offset;
2259 if (n < bytes) {
2260 bytes = n;
2261 }
2262
2263 /* Must be non-NULL or bdrv_getlength() would have failed */
2264 assert(bs->drv);
2265 has_filtered_child = bdrv_filter_child(bs);
2266 if (!bs->drv->bdrv_co_block_status && !has_filtered_child) {
2267 *pnum = bytes;
2268 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
2269 if (offset + bytes == total_size) {
2270 ret |= BDRV_BLOCK_EOF;
2271 }
2272 if (bs->drv->protocol_name) {
2273 ret |= BDRV_BLOCK_OFFSET_VALID;
2274 local_map = offset;
2275 local_file = bs;
2276 }
2277 goto early_out;
2278 }
2279
2280 bdrv_inc_in_flight(bs);
2281
2282 /* Round out to request_alignment boundaries */
2283 align = bs->bl.request_alignment;
2284 aligned_offset = QEMU_ALIGN_DOWN(offset, align);
2285 aligned_bytes = ROUND_UP(offset + bytes, align) - aligned_offset;
2286
2287 if (bs->drv->bdrv_co_block_status) {
2288 /*
2289 * Use the block-status cache only for protocol nodes: Format
2290 * drivers are generally quick to inquire the status, but protocol
2291 * drivers often need to get information from outside of qemu, so
2292 * we do not have control over the actual implementation. There
2293 * have been cases where inquiring the status took an unreasonably
2294 * long time, and we can do nothing in qemu to fix it.
2295 * This is especially problematic for images with large data areas,
2296 * because finding the few holes in them and giving them special
2297 * treatment does not gain much performance. Therefore, we try to
2298 * cache the last-identified data region.
2299 *
2300 * Second, limiting ourselves to protocol nodes allows us to assume
2301 * the block status for data regions to be DATA | OFFSET_VALID, and
2302 * that the host offset is the same as the guest offset.
2303 *
2304 * Note that it is possible that external writers zero parts of
2305 * the cached regions without the cache being invalidated, and so
2306 * we may report zeroes as data. This is not catastrophic,
2307 * however, because reporting zeroes as data is fine.
2308 */
2309 if (QLIST_EMPTY(&bs->children) &&
2310 bdrv_bsc_is_data(bs, aligned_offset, pnum))
2311 {
2312 ret = BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID;
2313 local_file = bs;
2314 local_map = aligned_offset;
2315 } else {
2316 ret = bs->drv->bdrv_co_block_status(bs, want_zero, aligned_offset,
2317 aligned_bytes, pnum, &local_map,
2318 &local_file);
2319
2320 /*
2321 * Note that checking QLIST_EMPTY(&bs->children) is also done when
2322 * the cache is queried above. Technically, we do not need to check
2323 * it here; the worst that can happen is that we fill the cache for
2324 * non-protocol nodes, and then it is never used. However, filling
2325 * the cache requires an RCU update, so double check here to avoid
2326 * such an update if possible.
2327 *
2328 * Check want_zero, because we only want to update the cache when we
2329 * have accurate information about what is zero and what is data.
2330 */
2331 if (want_zero &&
2332 ret == (BDRV_BLOCK_DATA | BDRV_BLOCK_OFFSET_VALID) &&
2333 QLIST_EMPTY(&bs->children))
2334 {
2335 /*
2336 * When a protocol driver reports BLOCK_OFFSET_VALID, the
2337 * returned local_map value must be the same as the offset we
2338 * have passed (aligned_offset), and local_bs must be the node
2339 * itself.
2340 * Assert this, because we follow this rule when reading from
2341 * the cache (see the `local_file = bs` and
2342 * `local_map = aligned_offset` assignments above), and the
2343 * result the cache delivers must be the same as the driver
2344 * would deliver.
2345 */
2346 assert(local_file == bs);
2347 assert(local_map == aligned_offset);
2348 bdrv_bsc_fill(bs, aligned_offset, *pnum);
2349 }
2350 }
2351 } else {
2352 /* Default code for filters */
2353
2354 local_file = bdrv_filter_bs(bs);
2355 assert(local_file);
2356
2357 *pnum = aligned_bytes;
2358 local_map = aligned_offset;
2359 ret = BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID;
2360 }
2361 if (ret < 0) {
2362 *pnum = 0;
2363 goto out;
2364 }
2365
2366 /*
2367 * The driver's result must be a non-zero multiple of request_alignment.
2368 * Clamp pnum and adjust map to original request.
2369 */
2370 assert(*pnum && QEMU_IS_ALIGNED(*pnum, align) &&
2371 align > offset - aligned_offset);
2372 if (ret & BDRV_BLOCK_RECURSE) {
2373 assert(ret & BDRV_BLOCK_DATA);
2374 assert(ret & BDRV_BLOCK_OFFSET_VALID);
2375 assert(!(ret & BDRV_BLOCK_ZERO));
2376 }
2377
2378 *pnum -= offset - aligned_offset;
2379 if (*pnum > bytes) {
2380 *pnum = bytes;
2381 }
2382 if (ret & BDRV_BLOCK_OFFSET_VALID) {
2383 local_map += offset - aligned_offset;
2384 }
2385
2386 if (ret & BDRV_BLOCK_RAW) {
2387 assert(ret & BDRV_BLOCK_OFFSET_VALID && local_file);
2388 ret = bdrv_co_block_status(local_file, want_zero, local_map,
2389 *pnum, pnum, &local_map, &local_file);
2390 goto out;
2391 }
2392
2393 if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
2394 ret |= BDRV_BLOCK_ALLOCATED;
2395 } else if (bs->drv->supports_backing) {
2396 BlockDriverState *cow_bs = bdrv_cow_bs(bs);
2397
2398 if (!cow_bs) {
2399 ret |= BDRV_BLOCK_ZERO;
2400 } else if (want_zero) {
2401 int64_t size2 = bdrv_getlength(cow_bs);
2402
2403 if (size2 >= 0 && offset >= size2) {
2404 ret |= BDRV_BLOCK_ZERO;
2405 }
2406 }
2407 }
2408
2409 if (want_zero && ret & BDRV_BLOCK_RECURSE &&
2410 local_file && local_file != bs &&
2411 (ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO) &&
2412 (ret & BDRV_BLOCK_OFFSET_VALID)) {
2413 int64_t file_pnum;
2414 int ret2;
2415
2416 ret2 = bdrv_co_block_status(local_file, want_zero, local_map,
2417 *pnum, &file_pnum, NULL, NULL);
2418 if (ret2 >= 0) {
2419 /* Ignore errors. This is just providing extra information, it
2420 * is useful but not necessary.
2421 */
2422 if (ret2 & BDRV_BLOCK_EOF &&
2423 (!file_pnum || ret2 & BDRV_BLOCK_ZERO)) {
2424 /*
2425 * It is valid for the format block driver to read
2426 * beyond the end of the underlying file's current
2427 * size; such areas read as zero.
2428 */
2429 ret |= BDRV_BLOCK_ZERO;
2430 } else {
2431 /* Limit request to the range reported by the protocol driver */
2432 *pnum = file_pnum;
2433 ret |= (ret2 & BDRV_BLOCK_ZERO);
2434 }
2435 }
2436 }
2437
2438 out:
2439 bdrv_dec_in_flight(bs);
2440 if (ret >= 0 && offset + *pnum == total_size) {
2441 ret |= BDRV_BLOCK_EOF;
2442 }
2443 early_out:
2444 if (file) {
2445 *file = local_file;
2446 }
2447 if (map) {
2448 *map = local_map;
2449 }
2450 return ret;
2451 }
2452
2453 int coroutine_fn
2454 bdrv_co_common_block_status_above(BlockDriverState *bs,
2455 BlockDriverState *base,
2456 bool include_base,
2457 bool want_zero,
2458 int64_t offset,
2459 int64_t bytes,
2460 int64_t *pnum,
2461 int64_t *map,
2462 BlockDriverState **file,
2463 int *depth)
2464 {
2465 int ret;
2466 BlockDriverState *p;
2467 int64_t eof = 0;
2468 int dummy;
2469 IO_CODE();
2470
2471 assert(!include_base || base); /* Can't include NULL base */
2472
2473 if (!depth) {
2474 depth = &dummy;
2475 }
2476 *depth = 0;
2477
2478 if (!include_base && bs == base) {
2479 *pnum = bytes;
2480 return 0;
2481 }
2482
2483 ret = bdrv_co_block_status(bs, want_zero, offset, bytes, pnum, map, file);
2484 ++*depth;
2485 if (ret < 0 || *pnum == 0 || ret & BDRV_BLOCK_ALLOCATED || bs == base) {
2486 return ret;
2487 }
2488
2489 if (ret & BDRV_BLOCK_EOF) {
2490 eof = offset + *pnum;
2491 }
2492
2493 assert(*pnum <= bytes);
2494 bytes = *pnum;
2495
2496 for (p = bdrv_filter_or_cow_bs(bs); include_base || p != base;
2497 p = bdrv_filter_or_cow_bs(p))
2498 {
2499 ret = bdrv_co_block_status(p, want_zero, offset, bytes, pnum, map,
2500 file);
2501 ++*depth;
2502 if (ret < 0) {
2503 return ret;
2504 }
2505 if (*pnum == 0) {
2506 /*
2507 * The top layer deferred to this layer, and because this layer is
2508 * short, any zeroes that we synthesize beyond EOF behave as if they
2509 * were allocated at this layer.
2510 *
2511 * We don't include BDRV_BLOCK_EOF into ret, as upper layer may be
2512 * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see
2513 * below.
2514 */
2515 assert(ret & BDRV_BLOCK_EOF);
2516 *pnum = bytes;
2517 if (file) {
2518 *file = p;
2519 }
2520 ret = BDRV_BLOCK_ZERO | BDRV_BLOCK_ALLOCATED;
2521 break;
2522 }
2523 if (ret & BDRV_BLOCK_ALLOCATED) {
2524 /*
2525 * We've found the node and the status, we must break.
2526 *
2527 * Drop BDRV_BLOCK_EOF, as it's not for upper layer, which may be
2528 * larger. We'll add BDRV_BLOCK_EOF if needed at function end, see
2529 * below.
2530 */
2531 ret &= ~BDRV_BLOCK_EOF;
2532 break;
2533 }
2534
2535 if (p == base) {
2536 assert(include_base);
2537 break;
2538 }
2539
2540 /*
2541 * OK, [offset, offset + *pnum) region is unallocated on this layer,
2542 * let's continue the diving.
2543 */
2544 assert(*pnum <= bytes);
2545 bytes = *pnum;
2546 }
2547
2548 if (offset + *pnum == eof) {
2549 ret |= BDRV_BLOCK_EOF;
2550 }
2551
2552 return ret;
2553 }
2554
2555 int bdrv_block_status_above(BlockDriverState *bs, BlockDriverState *base,
2556 int64_t offset, int64_t bytes, int64_t *pnum,
2557 int64_t *map, BlockDriverState **file)
2558 {
2559 IO_CODE();
2560 return bdrv_common_block_status_above(bs, base, false, true, offset, bytes,
2561 pnum, map, file, NULL);
2562 }
2563
2564 int bdrv_block_status(BlockDriverState *bs, int64_t offset, int64_t bytes,
2565 int64_t *pnum, int64_t *map, BlockDriverState **file)
2566 {
2567 IO_CODE();
2568 return bdrv_block_status_above(bs, bdrv_filter_or_cow_bs(bs),
2569 offset, bytes, pnum, map, file);
2570 }
2571
2572 /*
2573 * Check @bs (and its backing chain) to see if the range defined
2574 * by @offset and @bytes is known to read as zeroes.
2575 * Return 1 if that is the case, 0 otherwise and -errno on error.
2576 * This test is meant to be fast rather than accurate so returning 0
2577 * does not guarantee non-zero data.
2578 */
2579 int coroutine_fn bdrv_co_is_zero_fast(BlockDriverState *bs, int64_t offset,
2580 int64_t bytes)
2581 {
2582 int ret;
2583 int64_t pnum = bytes;
2584 IO_CODE();
2585
2586 if (!bytes) {
2587 return 1;
2588 }
2589
2590 ret = bdrv_co_common_block_status_above(bs, NULL, false, false, offset,
2591 bytes, &pnum, NULL, NULL, NULL);
2592
2593 if (ret < 0) {
2594 return ret;
2595 }
2596
2597 return (pnum == bytes) && (ret & BDRV_BLOCK_ZERO);
2598 }
2599
2600 int bdrv_is_allocated(BlockDriverState *bs, int64_t offset, int64_t bytes,
2601 int64_t *pnum)
2602 {
2603 int ret;
2604 int64_t dummy;
2605 IO_CODE();
2606
2607 ret = bdrv_common_block_status_above(bs, bs, true, false, offset,
2608 bytes, pnum ? pnum : &dummy, NULL,
2609 NULL, NULL);
2610 if (ret < 0) {
2611 return ret;
2612 }
2613 return !!(ret & BDRV_BLOCK_ALLOCATED);
2614 }
2615
2616 /*
2617 * Given an image chain: ... -> [BASE] -> [INTER1] -> [INTER2] -> [TOP]
2618 *
2619 * Return a positive depth if (a prefix of) the given range is allocated
2620 * in any image between BASE and TOP (BASE is only included if include_base
2621 * is set). Depth 1 is TOP, 2 is the first backing layer, and so forth.
2622 * BASE can be NULL to check if the given offset is allocated in any
2623 * image of the chain. Return 0 otherwise, or negative errno on
2624 * failure.
2625 *
2626 * 'pnum' is set to the number of bytes (including and immediately
2627 * following the specified offset) that are known to be in the same
2628 * allocated/unallocated state. Note that a subsequent call starting
2629 * at 'offset + *pnum' may return the same allocation status (in other
2630 * words, the result is not necessarily the maximum possible range);
2631 * but 'pnum' will only be 0 when end of file is reached.
2632 */
2633 int bdrv_is_allocated_above(BlockDriverState *top,
2634 BlockDriverState *base,
2635 bool include_base, int64_t offset,
2636 int64_t bytes, int64_t *pnum)
2637 {
2638 int depth;
2639 int ret = bdrv_common_block_status_above(top, base, include_base, false,
2640 offset, bytes, pnum, NULL, NULL,
2641 &depth);
2642 IO_CODE();
2643 if (ret < 0) {
2644 return ret;
2645 }
2646
2647 if (ret & BDRV_BLOCK_ALLOCATED) {
2648 return depth;
2649 }
2650 return 0;
2651 }
2652
2653 int coroutine_fn
2654 bdrv_co_readv_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
2655 {
2656 BlockDriver *drv = bs->drv;
2657 BlockDriverState *child_bs = bdrv_primary_bs(bs);
2658 int ret;
2659 IO_CODE();
2660
2661 ret = bdrv_check_qiov_request(pos, qiov->size, qiov, 0, NULL);
2662 if (ret < 0) {
2663 return ret;
2664 }
2665
2666 if (!drv) {
2667 return -ENOMEDIUM;
2668 }
2669
2670 bdrv_inc_in_flight(bs);
2671
2672 if (drv->bdrv_load_vmstate) {
2673 ret = drv->bdrv_load_vmstate(bs, qiov, pos);
2674 } else if (child_bs) {
2675 ret = bdrv_co_readv_vmstate(child_bs, qiov, pos);
2676 } else {
2677 ret = -ENOTSUP;
2678 }
2679
2680 bdrv_dec_in_flight(bs);
2681
2682 return ret;
2683 }
2684
2685 int coroutine_fn
2686 bdrv_co_writev_vmstate(BlockDriverState *bs, QEMUIOVector *qiov, int64_t pos)
2687 {
2688 BlockDriver *drv = bs->drv;
2689 BlockDriverState *child_bs = bdrv_primary_bs(bs);
2690 int ret;
2691 IO_CODE();
2692
2693 ret = bdrv_check_qiov_request(pos, qiov->size, qiov, 0, NULL);
2694 if (ret < 0) {
2695 return ret;
2696 }
2697
2698 if (!drv) {
2699 return -ENOMEDIUM;
2700 }
2701
2702 bdrv_inc_in_flight(bs);
2703
2704 if (drv->bdrv_save_vmstate) {
2705 ret = drv->bdrv_save_vmstate(bs, qiov, pos);
2706 } else if (child_bs) {
2707 ret = bdrv_co_writev_vmstate(child_bs, qiov, pos);
2708 } else {
2709 ret = -ENOTSUP;
2710 }
2711
2712 bdrv_dec_in_flight(bs);
2713
2714 return ret;
2715 }
2716
2717 int bdrv_save_vmstate(BlockDriverState *bs, const uint8_t *buf,
2718 int64_t pos, int size)
2719 {
2720 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size);
2721 int ret = bdrv_writev_vmstate(bs, &qiov, pos);
2722 IO_CODE();
2723
2724 return ret < 0 ? ret : size;
2725 }
2726
2727 int bdrv_load_vmstate(BlockDriverState *bs, uint8_t *buf,
2728 int64_t pos, int size)
2729 {
2730 QEMUIOVector qiov = QEMU_IOVEC_INIT_BUF(qiov, buf, size);
2731 int ret = bdrv_readv_vmstate(bs, &qiov, pos);
2732 IO_CODE();
2733
2734 return ret < 0 ? ret : size;
2735 }
2736
2737 /**************************************************************/
2738 /* async I/Os */
2739
2740 void bdrv_aio_cancel(BlockAIOCB *acb)
2741 {
2742 IO_CODE();
2743 qemu_aio_ref(acb);
2744 bdrv_aio_cancel_async(acb);
2745 while (acb->refcnt > 1) {
2746 if (acb->aiocb_info->get_aio_context) {
2747 aio_poll(acb->aiocb_info->get_aio_context(acb), true);
2748 } else if (acb->bs) {
2749 /* qemu_aio_ref and qemu_aio_unref are not thread-safe, so
2750 * assert that we're not using an I/O thread. Thread-safe
2751 * code should use bdrv_aio_cancel_async exclusively.
2752 */
2753 assert(bdrv_get_aio_context(acb->bs) == qemu_get_aio_context());
2754 aio_poll(bdrv_get_aio_context(acb->bs), true);
2755 } else {
2756 abort();
2757 }
2758 }
2759 qemu_aio_unref(acb);
2760 }
2761
2762 /* Async version of aio cancel. The caller is not blocked if the acb implements
2763 * cancel_async, otherwise we do nothing and let the request normally complete.
2764 * In either case the completion callback must be called. */
2765 void bdrv_aio_cancel_async(BlockAIOCB *acb)
2766 {
2767 IO_CODE();
2768 if (acb->aiocb_info->cancel_async) {
2769 acb->aiocb_info->cancel_async(acb);
2770 }
2771 }
2772
2773 /**************************************************************/
2774 /* Coroutine block device emulation */
2775
2776 int coroutine_fn bdrv_co_flush(BlockDriverState *bs)
2777 {
2778 BdrvChild *primary_child = bdrv_primary_child(bs);
2779 BdrvChild *child;
2780 int current_gen;
2781 int ret = 0;
2782 IO_CODE();
2783
2784 bdrv_inc_in_flight(bs);
2785
2786 if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs) ||
2787 bdrv_is_sg(bs)) {
2788 goto early_exit;
2789 }
2790
2791 qemu_co_mutex_lock(&bs->reqs_lock);
2792 current_gen = qatomic_read(&bs->write_gen);
2793
2794 /* Wait until any previous flushes are completed */
2795 while (bs->active_flush_req) {
2796 qemu_co_queue_wait(&bs->flush_queue, &bs->reqs_lock);
2797 }
2798
2799 /* Flushes reach this point in nondecreasing current_gen order. */
2800 bs->active_flush_req = true;
2801 qemu_co_mutex_unlock(&bs->reqs_lock);
2802
2803 /* Write back all layers by calling one driver function */
2804 if (bs->drv->bdrv_co_flush) {
2805 ret = bs->drv->bdrv_co_flush(bs);
2806 goto out;
2807 }
2808
2809 /* Write back cached data to the OS even with cache=unsafe */
2810 BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_OS);
2811 if (bs->drv->bdrv_co_flush_to_os) {
2812 ret = bs->drv->bdrv_co_flush_to_os(bs);
2813 if (ret < 0) {
2814 goto out;
2815 }
2816 }
2817
2818 /* But don't actually force it to the disk with cache=unsafe */
2819 if (bs->open_flags & BDRV_O_NO_FLUSH) {
2820 goto flush_children;
2821 }
2822
2823 /* Check if we really need to flush anything */
2824 if (bs->flushed_gen == current_gen) {
2825 goto flush_children;
2826 }
2827
2828 BLKDBG_EVENT(primary_child, BLKDBG_FLUSH_TO_DISK);
2829 if (!bs->drv) {
2830 /* bs->drv->bdrv_co_flush() might have ejected the BDS
2831 * (even in case of apparent success) */
2832 ret = -ENOMEDIUM;
2833 goto out;
2834 }
2835 if (bs->drv->bdrv_co_flush_to_disk) {
2836 ret = bs->drv->bdrv_co_flush_to_disk(bs);
2837 } else if (bs->drv->bdrv_aio_flush) {
2838 BlockAIOCB *acb;
2839 CoroutineIOCompletion co = {
2840 .coroutine = qemu_coroutine_self(),
2841 };
2842
2843 acb = bs->drv->bdrv_aio_flush(bs, bdrv_co_io_em_complete, &co);
2844 if (acb == NULL) {
2845 ret = -EIO;
2846 } else {
2847 qemu_coroutine_yield();
2848 ret = co.ret;
2849 }
2850 } else {
2851 /*
2852 * Some block drivers always operate in either writethrough or unsafe
2853 * mode and don't support bdrv_flush therefore. Usually qemu doesn't
2854 * know how the server works (because the behaviour is hardcoded or
2855 * depends on server-side configuration), so we can't ensure that
2856 * everything is safe on disk. Returning an error doesn't work because
2857 * that would break guests even if the server operates in writethrough
2858 * mode.
2859 *
2860 * Let's hope the user knows what he's doing.
2861 */
2862 ret = 0;
2863 }
2864
2865 if (ret < 0) {
2866 goto out;
2867 }
2868
2869 /* Now flush the underlying protocol. It will also have BDRV_O_NO_FLUSH
2870 * in the case of cache=unsafe, so there are no useless flushes.
2871 */
2872 flush_children:
2873 ret = 0;
2874 QLIST_FOREACH(child, &bs->children, next) {
2875 if (child->perm & (BLK_PERM_WRITE | BLK_PERM_WRITE_UNCHANGED)) {
2876 int this_child_ret = bdrv_co_flush(child->bs);
2877 if (!ret) {
2878 ret = this_child_ret;
2879 }
2880 }
2881 }
2882
2883 out:
2884 /* Notify any pending flushes that we have completed */
2885 if (ret == 0) {
2886 bs->flushed_gen = current_gen;
2887 }
2888
2889 qemu_co_mutex_lock(&bs->reqs_lock);
2890 bs->active_flush_req = false;
2891 /* Return value is ignored - it's ok if wait queue is empty */
2892 qemu_co_queue_next(&bs->flush_queue);
2893 qemu_co_mutex_unlock(&bs->reqs_lock);
2894
2895 early_exit:
2896 bdrv_dec_in_flight(bs);
2897 return ret;
2898 }
2899
2900 int coroutine_fn bdrv_co_pdiscard(BdrvChild *child, int64_t offset,
2901 int64_t bytes)
2902 {
2903 BdrvTrackedRequest req;
2904 int ret;
2905 int64_t max_pdiscard;
2906 int head, tail, align;
2907 BlockDriverState *bs = child->bs;
2908 IO_CODE();
2909
2910 if (!bs || !bs->drv || !bdrv_is_inserted(bs)) {
2911 return -ENOMEDIUM;
2912 }
2913
2914 if (bdrv_has_readonly_bitmaps(bs)) {
2915 return -EPERM;
2916 }
2917
2918 ret = bdrv_check_request(offset, bytes, NULL);
2919 if (ret < 0) {
2920 return ret;
2921 }
2922
2923 /* Do nothing if disabled. */
2924 if (!(bs->open_flags & BDRV_O_UNMAP)) {
2925 return 0;
2926 }
2927
2928 if (!bs->drv->bdrv_co_pdiscard && !bs->drv->bdrv_aio_pdiscard) {
2929 return 0;
2930 }
2931
2932 /* Invalidate the cached block-status data range if this discard overlaps */
2933 bdrv_bsc_invalidate_range(bs, offset, bytes);
2934
2935 /* Discard is advisory, but some devices track and coalesce
2936 * unaligned requests, so we must pass everything down rather than
2937 * round here. Still, most devices will just silently ignore
2938 * unaligned requests (by returning -ENOTSUP), so we must fragment
2939 * the request accordingly. */
2940 align = MAX(bs->bl.pdiscard_alignment, bs->bl.request_alignment);
2941 assert(align % bs->bl.request_alignment == 0);
2942 head = offset % align;
2943 tail = (offset + bytes) % align;
2944
2945 bdrv_inc_in_flight(bs);
2946 tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_DISCARD);
2947
2948 ret = bdrv_co_write_req_prepare(child, offset, bytes, &req, 0);
2949 if (ret < 0) {
2950 goto out;
2951 }
2952
2953 max_pdiscard = QEMU_ALIGN_DOWN(MIN_NON_ZERO(bs->bl.max_pdiscard, INT64_MAX),
2954 align);
2955 assert(max_pdiscard >= bs->bl.request_alignment);
2956
2957 while (bytes > 0) {
2958 int64_t num = bytes;
2959
2960 if (head) {
2961 /* Make small requests to get to alignment boundaries. */
2962 num = MIN(bytes, align - head);
2963 if (!QEMU_IS_ALIGNED(num, bs->bl.request_alignment)) {
2964 num %= bs->bl.request_alignment;
2965 }
2966 head = (head + num) % align;
2967 assert(num < max_pdiscard);
2968 } else if (tail) {
2969 if (num > align) {
2970 /* Shorten the request to the last aligned cluster. */
2971 num -= tail;
2972 } else if (!QEMU_IS_ALIGNED(tail, bs->bl.request_alignment) &&
2973 tail > bs->bl.request_alignment) {
2974 tail %= bs->bl.request_alignment;
2975 num -= tail;
2976 }
2977 }
2978 /* limit request size */
2979 if (num > max_pdiscard) {
2980 num = max_pdiscard;
2981 }
2982
2983 if (!bs->drv) {
2984 ret = -ENOMEDIUM;
2985 goto out;
2986 }
2987 if (bs->drv->bdrv_co_pdiscard) {
2988 ret = bs->drv->bdrv_co_pdiscard(bs, offset, num);
2989 } else {
2990 BlockAIOCB *acb;
2991 CoroutineIOCompletion co = {
2992 .coroutine = qemu_coroutine_self(),
2993 };
2994
2995 acb = bs->drv->bdrv_aio_pdiscard(bs, offset, num,
2996 bdrv_co_io_em_complete, &co);
2997 if (acb == NULL) {
2998 ret = -EIO;
2999 goto out;
3000 } else {
3001 qemu_coroutine_yield();
3002 ret = co.ret;
3003 }
3004 }
3005 if (ret && ret != -ENOTSUP) {
3006 goto out;
3007 }
3008
3009 offset += num;
3010 bytes -= num;
3011 }
3012 ret = 0;
3013 out:
3014 bdrv_co_write_req_finish(child, req.offset, req.bytes, &req, ret);
3015 tracked_request_end(&req);
3016 bdrv_dec_in_flight(bs);
3017 return ret;
3018 }
3019
3020 int coroutine_fn bdrv_co_ioctl(BlockDriverState *bs, int req, void *buf)
3021 {
3022 BlockDriver *drv = bs->drv;
3023 CoroutineIOCompletion co = {
3024 .coroutine = qemu_coroutine_self(),
3025 };
3026 BlockAIOCB *acb;
3027 IO_CODE();
3028
3029 bdrv_inc_in_flight(bs);
3030 if (!drv || (!drv->bdrv_aio_ioctl && !drv->bdrv_co_ioctl)) {
3031 co.ret = -ENOTSUP;
3032 goto out;
3033 }
3034
3035 if (drv->bdrv_co_ioctl) {
3036 co.ret = drv->bdrv_co_ioctl(bs, req, buf);
3037 } else {
3038 acb = drv->bdrv_aio_ioctl(bs, req, buf, bdrv_co_io_em_complete, &co);
3039 if (!acb) {
3040 co.ret = -ENOTSUP;
3041 goto out;
3042 }
3043 qemu_coroutine_yield();
3044 }
3045 out:
3046 bdrv_dec_in_flight(bs);
3047 return co.ret;
3048 }
3049
3050 void *qemu_blockalign(BlockDriverState *bs, size_t size)
3051 {
3052 IO_CODE();
3053 return qemu_memalign(bdrv_opt_mem_align(bs), size);
3054 }
3055
3056 void *qemu_blockalign0(BlockDriverState *bs, size_t size)
3057 {
3058 IO_CODE();
3059 return memset(qemu_blockalign(bs, size), 0, size);
3060 }
3061
3062 void *qemu_try_blockalign(BlockDriverState *bs, size_t size)
3063 {
3064 size_t align = bdrv_opt_mem_align(bs);
3065 IO_CODE();
3066
3067 /* Ensure that NULL is never returned on success */
3068 assert(align > 0);
3069 if (size == 0) {
3070 size = align;
3071 }
3072
3073 return qemu_try_memalign(align, size);
3074 }
3075
3076 void *qemu_try_blockalign0(BlockDriverState *bs, size_t size)
3077 {
3078 void *mem = qemu_try_blockalign(bs, size);
3079 IO_CODE();
3080
3081 if (mem) {
3082 memset(mem, 0, size);
3083 }
3084
3085 return mem;
3086 }
3087
3088 void bdrv_io_plug(BlockDriverState *bs)
3089 {
3090 BdrvChild *child;
3091 IO_CODE();
3092
3093 QLIST_FOREACH(child, &bs->children, next) {
3094 bdrv_io_plug(child->bs);
3095 }
3096
3097 if (qatomic_fetch_inc(&bs->io_plugged) == 0) {
3098 BlockDriver *drv = bs->drv;
3099 if (drv && drv->bdrv_io_plug) {
3100 drv->bdrv_io_plug(bs);
3101 }
3102 }
3103 }
3104
3105 void bdrv_io_unplug(BlockDriverState *bs)
3106 {
3107 BdrvChild *child;
3108 IO_CODE();
3109
3110 assert(bs->io_plugged);
3111 if (qatomic_fetch_dec(&bs->io_plugged) == 1) {
3112 BlockDriver *drv = bs->drv;
3113 if (drv && drv->bdrv_io_unplug) {
3114 drv->bdrv_io_unplug(bs);
3115 }
3116 }
3117
3118 QLIST_FOREACH(child, &bs->children, next) {
3119 bdrv_io_unplug(child->bs);
3120 }
3121 }
3122
3123 /* Helper that undoes bdrv_register_buf() when it fails partway through */
3124 static void bdrv_register_buf_rollback(BlockDriverState *bs,
3125 void *host,
3126 size_t size,
3127 BdrvChild *final_child)
3128 {
3129 BdrvChild *child;
3130
3131 QLIST_FOREACH(child, &bs->children, next) {
3132 if (child == final_child) {
3133 break;
3134 }
3135
3136 bdrv_unregister_buf(child->bs, host, size);
3137 }
3138
3139 if (bs->drv && bs->drv->bdrv_unregister_buf) {
3140 bs->drv->bdrv_unregister_buf(bs, host, size);
3141 }
3142 }
3143
3144 bool bdrv_register_buf(BlockDriverState *bs, void *host, size_t size,
3145 Error **errp)
3146 {
3147 BdrvChild *child;
3148
3149 GLOBAL_STATE_CODE();
3150 if (bs->drv && bs->drv->bdrv_register_buf) {
3151 if (!bs->drv->bdrv_register_buf(bs, host, size, errp)) {
3152 return false;
3153 }
3154 }
3155 QLIST_FOREACH(child, &bs->children, next) {
3156 if (!bdrv_register_buf(child->bs, host, size, errp)) {
3157 bdrv_register_buf_rollback(bs, host, size, child);
3158 return false;
3159 }
3160 }
3161 return true;
3162 }
3163
3164 void bdrv_unregister_buf(BlockDriverState *bs, void *host, size_t size)
3165 {
3166 BdrvChild *child;
3167
3168 GLOBAL_STATE_CODE();
3169 if (bs->drv && bs->drv->bdrv_unregister_buf) {
3170 bs->drv->bdrv_unregister_buf(bs, host, size);
3171 }
3172 QLIST_FOREACH(child, &bs->children, next) {
3173 bdrv_unregister_buf(child->bs, host, size);
3174 }
3175 }
3176
3177 static int coroutine_fn bdrv_co_copy_range_internal(
3178 BdrvChild *src, int64_t src_offset, BdrvChild *dst,
3179 int64_t dst_offset, int64_t bytes,
3180 BdrvRequestFlags read_flags, BdrvRequestFlags write_flags,
3181 bool recurse_src)
3182 {
3183 BdrvTrackedRequest req;
3184 int ret;
3185
3186 /* TODO We can support BDRV_REQ_NO_FALLBACK here */
3187 assert(!(read_flags & BDRV_REQ_NO_FALLBACK));
3188 assert(!(write_flags & BDRV_REQ_NO_FALLBACK));
3189 assert(!(read_flags & BDRV_REQ_NO_WAIT));
3190 assert(!(write_flags & BDRV_REQ_NO_WAIT));
3191
3192 if (!dst || !dst->bs || !bdrv_is_inserted(dst->bs)) {
3193 return -ENOMEDIUM;
3194 }
3195 ret = bdrv_check_request32(dst_offset, bytes, NULL, 0);
3196 if (ret) {
3197 return ret;
3198 }
3199 if (write_flags & BDRV_REQ_ZERO_WRITE) {
3200 return bdrv_co_pwrite_zeroes(dst, dst_offset, bytes, write_flags);
3201 }
3202
3203 if (!src || !src->bs || !bdrv_is_inserted(src->bs)) {
3204 return -ENOMEDIUM;
3205 }
3206 ret = bdrv_check_request32(src_offset, bytes, NULL, 0);
3207 if (ret) {
3208 return ret;
3209 }
3210
3211 if (!src->bs->drv->bdrv_co_copy_range_from
3212 || !dst->bs->drv->bdrv_co_copy_range_to
3213 || src->bs->encrypted || dst->bs->encrypted) {
3214 return -ENOTSUP;
3215 }
3216
3217 if (recurse_src) {
3218 bdrv_inc_in_flight(src->bs);
3219 tracked_request_begin(&req, src->bs, src_offset, bytes,
3220 BDRV_TRACKED_READ);
3221
3222 /* BDRV_REQ_SERIALISING is only for write operation */
3223 assert(!(read_flags & BDRV_REQ_SERIALISING));
3224 bdrv_wait_serialising_requests(&req);
3225
3226 ret = src->bs->drv->bdrv_co_copy_range_from(src->bs,
3227 src, src_offset,
3228 dst, dst_offset,
3229 bytes,
3230 read_flags, write_flags);
3231
3232 tracked_request_end(&req);
3233 bdrv_dec_in_flight(src->bs);
3234 } else {
3235 bdrv_inc_in_flight(dst->bs);
3236 tracked_request_begin(&req, dst->bs, dst_offset, bytes,
3237 BDRV_TRACKED_WRITE);
3238 ret = bdrv_co_write_req_prepare(dst, dst_offset, bytes, &req,
3239 write_flags);
3240 if (!ret) {
3241 ret = dst->bs->drv->bdrv_co_copy_range_to(dst->bs,
3242 src, src_offset,
3243 dst, dst_offset,
3244 bytes,
3245 read_flags, write_flags);
3246 }
3247 bdrv_co_write_req_finish(dst, dst_offset, bytes, &req, ret);
3248 tracked_request_end(&req);
3249 bdrv_dec_in_flight(dst->bs);
3250 }
3251
3252 return ret;
3253 }
3254
3255 /* Copy range from @src to @dst.
3256 *
3257 * See the comment of bdrv_co_copy_range for the parameter and return value
3258 * semantics. */
3259 int coroutine_fn bdrv_co_copy_range_from(BdrvChild *src, int64_t src_offset,
3260 BdrvChild *dst, int64_t dst_offset,
3261 int64_t bytes,
3262 BdrvRequestFlags read_flags,
3263 BdrvRequestFlags write_flags)
3264 {
3265 IO_CODE();
3266 trace_bdrv_co_copy_range_from(src, src_offset, dst, dst_offset, bytes,
3267 read_flags, write_flags);
3268 return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset,
3269 bytes, read_flags, write_flags, true);
3270 }
3271
3272 /* Copy range from @src to @dst.
3273 *
3274 * See the comment of bdrv_co_copy_range for the parameter and return value
3275 * semantics. */
3276 int coroutine_fn bdrv_co_copy_range_to(BdrvChild *src, int64_t src_offset,
3277 BdrvChild *dst, int64_t dst_offset,
3278 int64_t bytes,
3279 BdrvRequestFlags read_flags,
3280 BdrvRequestFlags write_flags)
3281 {
3282 IO_CODE();
3283 trace_bdrv_co_copy_range_to(src, src_offset, dst, dst_offset, bytes,
3284 read_flags, write_flags);
3285 return bdrv_co_copy_range_internal(src, src_offset, dst, dst_offset,
3286 bytes, read_flags, write_flags, false);
3287 }
3288
3289 int coroutine_fn bdrv_co_copy_range(BdrvChild *src, int64_t src_offset,
3290 BdrvChild *dst, int64_t dst_offset,
3291 int64_t bytes, BdrvRequestFlags read_flags,
3292 BdrvRequestFlags write_flags)
3293 {
3294 IO_CODE();
3295 return bdrv_co_copy_range_from(src, src_offset,
3296 dst, dst_offset,
3297 bytes, read_flags, write_flags);
3298 }
3299
3300 static void bdrv_parent_cb_resize(BlockDriverState *bs)
3301 {
3302 BdrvChild *c;
3303 QLIST_FOREACH(c, &bs->parents, next_parent) {
3304 if (c->klass->resize) {
3305 c->klass->resize(c);
3306 }
3307 }
3308 }
3309
3310 /**
3311 * Truncate file to 'offset' bytes (needed only for file protocols)
3312 *
3313 * If 'exact' is true, the file must be resized to exactly the given
3314 * 'offset'. Otherwise, it is sufficient for the node to be at least
3315 * 'offset' bytes in length.
3316 */
3317 int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset, bool exact,
3318 PreallocMode prealloc, BdrvRequestFlags flags,
3319 Error **errp)
3320 {
3321 BlockDriverState *bs = child->bs;
3322 BdrvChild *filtered, *backing;
3323 BlockDriver *drv = bs->drv;
3324 BdrvTrackedRequest req;
3325 int64_t old_size, new_bytes;
3326 int ret;
3327 IO_CODE();
3328
3329 /* if bs->drv == NULL, bs is closed, so there's nothing to do here */
3330 if (!drv) {
3331 error_setg(errp, "No medium inserted");
3332 return -ENOMEDIUM;
3333 }
3334 if (offset < 0) {
3335 error_setg(errp, "Image size cannot be negative");
3336 return -EINVAL;
3337 }
3338
3339 ret = bdrv_check_request(offset, 0, errp);
3340 if (ret < 0) {
3341 return ret;
3342 }
3343
3344 old_size = bdrv_getlength(bs);
3345 if (old_size < 0) {
3346 error_setg_errno(errp, -old_size, "Failed to get old image size");
3347 return old_size;
3348 }
3349
3350 if (bdrv_is_read_only(bs)) {
3351 error_setg(errp, "Image is read-only");
3352 return -EACCES;
3353 }
3354
3355 if (offset > old_size) {
3356 new_bytes = offset - old_size;
3357 } else {
3358 new_bytes = 0;
3359 }
3360
3361 bdrv_inc_in_flight(bs);
3362 tracked_request_begin(&req, bs, offset - new_bytes, new_bytes,
3363 BDRV_TRACKED_TRUNCATE);
3364
3365 /* If we are growing the image and potentially using preallocation for the
3366 * new area, we need to make sure that no write requests are made to it
3367 * concurrently or they might be overwritten by preallocation. */
3368 if (new_bytes) {
3369 bdrv_make_request_serialising(&req, 1);
3370 }
3371 ret = bdrv_co_write_req_prepare(child, offset - new_bytes, new_bytes, &req,
3372 0);
3373 if (ret < 0) {
3374 error_setg_errno(errp, -ret,
3375 "Failed to prepare request for truncation");
3376 goto out;
3377 }
3378
3379 filtered = bdrv_filter_child(bs);
3380 backing = bdrv_cow_child(bs);
3381
3382 /*
3383 * If the image has a backing file that is large enough that it would
3384 * provide data for the new area, we cannot leave it unallocated because
3385 * then the backing file content would become visible. Instead, zero-fill
3386 * the new area.
3387 *
3388 * Note that if the image has a backing file, but was opened without the
3389 * backing file, taking care of keeping things consistent with that backing
3390 * file is the user's responsibility.
3391 */
3392 if (new_bytes && backing) {
3393 int64_t backing_len;
3394
3395 backing_len = bdrv_getlength(backing->bs);
3396 if (backing_len < 0) {
3397 ret = backing_len;
3398 error_setg_errno(errp, -ret, "Could not get backing file size");
3399 goto out;
3400 }
3401
3402 if (backing_len > old_size) {
3403 flags |= BDRV_REQ_ZERO_WRITE;
3404 }
3405 }
3406
3407 if (drv->bdrv_co_truncate) {
3408 if (flags & ~bs->supported_truncate_flags) {
3409 error_setg(errp, "Block driver does not support requested flags");
3410 ret = -ENOTSUP;
3411 goto out;
3412 }
3413 ret = drv->bdrv_co_truncate(bs, offset, exact, prealloc, flags, errp);
3414 } else if (filtered) {
3415 ret = bdrv_co_truncate(filtered, offset, exact, prealloc, flags, errp);
3416 } else {
3417 error_setg(errp, "Image format driver does not support resize");
3418 ret = -ENOTSUP;
3419 goto out;
3420 }
3421 if (ret < 0) {
3422 goto out;
3423 }
3424
3425 ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS);
3426 if (ret < 0) {
3427 error_setg_errno(errp, -ret, "Could not refresh total sector count");
3428 } else {
3429 offset = bs->total_sectors * BDRV_SECTOR_SIZE;
3430 }
3431 /* It's possible that truncation succeeded but refresh_total_sectors
3432 * failed, but the latter doesn't affect how we should finish the request.
3433 * Pass 0 as the last parameter so that dirty bitmaps etc. are handled. */
3434 bdrv_co_write_req_finish(child, offset - new_bytes, new_bytes, &req, 0);
3435
3436 out:
3437 tracked_request_end(&req);
3438 bdrv_dec_in_flight(bs);
3439
3440 return ret;
3441 }
3442
3443 void bdrv_cancel_in_flight(BlockDriverState *bs)
3444 {
3445 GLOBAL_STATE_CODE();
3446 if (!bs || !bs->drv) {
3447 return;
3448 }
3449
3450 if (bs->drv->bdrv_cancel_in_flight) {
3451 bs->drv->bdrv_cancel_in_flight(bs);
3452 }
3453 }
3454
3455 int coroutine_fn
3456 bdrv_co_preadv_snapshot(BdrvChild *child, int64_t offset, int64_t bytes,
3457 QEMUIOVector *qiov, size_t qiov_offset)
3458 {
3459 BlockDriverState *bs = child->bs;
3460 BlockDriver *drv = bs->drv;
3461 int ret;
3462 IO_CODE();
3463
3464 if (!drv) {
3465 return -ENOMEDIUM;
3466 }
3467
3468 if (!drv->bdrv_co_preadv_snapshot) {
3469 return -ENOTSUP;
3470 }
3471
3472 bdrv_inc_in_flight(bs);
3473 ret = drv->bdrv_co_preadv_snapshot(bs, offset, bytes, qiov, qiov_offset);
3474 bdrv_dec_in_flight(bs);
3475
3476 return ret;
3477 }
3478
3479 int coroutine_fn
3480 bdrv_co_snapshot_block_status(BlockDriverState *bs,
3481 bool want_zero, int64_t offset, int64_t bytes,
3482 int64_t *pnum, int64_t *map,
3483 BlockDriverState **file)
3484 {
3485 BlockDriver *drv = bs->drv;
3486 int ret;
3487 IO_CODE();
3488
3489 if (!drv) {
3490 return -ENOMEDIUM;
3491 }
3492
3493 if (!drv->bdrv_co_snapshot_block_status) {
3494 return -ENOTSUP;
3495 }
3496
3497 bdrv_inc_in_flight(bs);
3498 ret = drv->bdrv_co_snapshot_block_status(bs, want_zero, offset, bytes,
3499 pnum, map, file);
3500 bdrv_dec_in_flight(bs);
3501
3502 return ret;
3503 }
3504
3505 int coroutine_fn
3506 bdrv_co_pdiscard_snapshot(BlockDriverState *bs, int64_t offset, int64_t bytes)
3507 {
3508 BlockDriver *drv = bs->drv;
3509 int ret;
3510 IO_CODE();
3511
3512 if (!drv) {
3513 return -ENOMEDIUM;
3514 }
3515
3516 if (!drv->bdrv_co_pdiscard_snapshot) {
3517 return -ENOTSUP;
3518 }
3519
3520 bdrv_inc_in_flight(bs);
3521 ret = drv->bdrv_co_pdiscard_snapshot(bs, offset, bytes);
3522 bdrv_dec_in_flight(bs);
3523
3524 return ret;
3525 }