]>
git.proxmox.com Git - mirror_qemu.git/blob - tests/qemu-iotests/257
4 # Test bitmap-sync backups (incremental, differential, and partials)
6 # Copyright (c) 2019 John Snow for Red Hat, Inc.
8 # This program is free software; you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation; either version 2 of the License, or
11 # (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
18 # You should have received a copy of the GNU General Public License
19 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 # owner=jsnow@redhat.com
27 from iotests
import log
, qemu_img
29 SIZE
= 64 * 1024 * 1024
30 GRANULARITY
= 64 * 1024
34 def __init__(self
, byte
, offset
, size
=GRANULARITY
):
39 def bits(self
, granularity
):
40 lower
= self
.offset
// granularity
41 upper
= (self
.offset
+ self
.size
- 1) // granularity
42 return set(range(lower
, upper
+ 1))
46 """Grouping of Pattern objects. Initialize with an iterable of Patterns."""
47 def __init__(self
, patterns
):
48 self
.patterns
= patterns
50 def bits(self
, granularity
):
51 """Calculate the unique bits dirtied by this pattern grouping"""
53 for pattern
in self
.patterns
:
54 res |
= pattern
.bits(granularity
)
61 Pattern('0x49', 0x0000000),
62 Pattern('0x6c', 0x0100000), # 1M
63 Pattern('0x6f', 0x2000000), # 32M
64 Pattern('0x76', 0x3ff0000)]), # 64M - 64K
66 # Batch 1: 6 clusters (3 new)
67 Pattern('0x65', 0x0000000), # Full overwrite
68 Pattern('0x77', 0x00f8000), # Partial-left (1M-32K)
69 Pattern('0x72', 0x2008000), # Partial-right (32M+32K)
70 Pattern('0x69', 0x3fe0000)]), # Adjacent-left (64M - 128K)
72 # Batch 2: 7 clusters (3 new)
73 Pattern('0x74', 0x0010000), # Adjacent-right
74 Pattern('0x69', 0x00e8000), # Partial-left (1M-96K)
75 Pattern('0x6e', 0x2018000), # Partial-right (32M+96K)
76 Pattern('0x67', 0x3fe0000,
77 2*GRANULARITY
)]), # Overwrite [(64M-128K)-64M)
79 # Batch 3: 8 clusters (5 new)
80 # Carefully chosen such that nothing re-dirties the one cluster
81 # that copies out successfully before failure in Group #1.
82 Pattern('0xaa', 0x0010000,
83 3*GRANULARITY
), # Overwrite and 2x Adjacent-right
84 Pattern('0xbb', 0x00d8000), # Partial-left (1M-160K)
85 Pattern('0xcc', 0x2028000), # Partial-right (32M+160K)
86 Pattern('0xdd', 0x3fc0000)]), # New; leaving a gap to the right
91 def __init__(self
, granularity
=GRANULARITY
):
93 self
.granularity
= granularity
95 def dirty_bits(self
, bits
):
96 self
._bits |
= set(bits
)
98 def dirty_group(self
, n
):
99 self
.dirty_bits(GROUPS
[n
].bits(self
.granularity
))
104 def clear_bits(self
, bits
):
105 self
._bits
-= set(bits
)
107 def clear_bit(self
, bit
):
108 self
.clear_bits({bit}
)
110 def clear_group(self
, n
):
111 self
.clear_bits(GROUPS
[n
].bits(self
.granularity
))
115 return sorted(self
.bits
)[0]
123 return len(self
.bits
)
125 def compare(self
, qmp_bitmap
):
127 Print a nice human-readable message checking that a bitmap as reported
128 by the QMP interface has as many bits set as we expect it to.
131 name
= qmp_bitmap
.get('name', '(anonymous)')
132 log("= Checking Bitmap {:s} =".format(name
))
135 have
= qmp_bitmap
['count'] // qmp_bitmap
['granularity']
137 log("expecting {:d} dirty sectors; have {:d}. {:s}".format(
138 want
, have
, "OK!" if want
== have
else "ERROR!"))
143 """Represents, vaguely, a drive attached to a VM.
144 Includes format, graph, and device information."""
146 def __init__(self
, path
, vm
=None):
153 def img_create(self
, fmt
, size
):
156 iotests
.qemu_img_create('-f', self
.fmt
, self
.path
, str(self
.size
))
158 def create_target(self
, name
, fmt
, size
):
159 basename
= os
.path
.basename(self
.path
)
160 file_node_name
= "file_{}".format(basename
)
163 log(vm
.command('blockdev-create', job_id
='bdc-file-job',
166 'filename': self
.path
,
169 vm
.run_job('bdc-file-job')
170 log(vm
.command('blockdev-add', driver
='file',
171 node_name
=file_node_name
, filename
=self
.path
))
173 log(vm
.command('blockdev-create', job_id
='bdc-fmt-job',
176 'file': file_node_name
,
179 vm
.run_job('bdc-fmt-job')
180 log(vm
.command('blockdev-add', driver
=fmt
,
182 file=file_node_name
))
187 def blockdev_backup(vm
, device
, target
, sync
, **kwargs
):
188 # Strip any arguments explicitly nulled by the caller:
189 kwargs
= {key
: val
for key
, val
in kwargs
.items() if val
is not None}
190 result
= vm
.qmp_log('blockdev-backup',
194 filter_node_name
='backup-top',
195 x_perf
={'max-workers': 1},
199 def blockdev_backup_mktarget(drive
, target_id
, filepath
, sync
, **kwargs
):
200 target_drive
= Drive(filepath
, vm
=drive
.vm
)
201 target_drive
.create_target(target_id
, drive
.fmt
, drive
.size
)
202 blockdev_backup(drive
.vm
, drive
.node
, target_id
, sync
, **kwargs
)
204 def reference_backup(drive
, n
, filepath
):
205 log("--- Reference Backup #{:d} ---\n".format(n
))
206 target_id
= "ref_target_{:d}".format(n
)
207 job_id
= "ref_backup_{:d}".format(n
)
208 blockdev_backup_mktarget(drive
, target_id
, filepath
, "full",
210 drive
.vm
.run_job(job_id
, auto_dismiss
=True)
213 def backup(drive
, n
, filepath
, sync
, **kwargs
):
214 log("--- Test Backup #{:d} ---\n".format(n
))
215 target_id
= "backup_target_{:d}".format(n
)
216 job_id
= "backup_{:d}".format(n
)
217 kwargs
.setdefault('auto-finalize', False)
218 blockdev_backup_mktarget(drive
, target_id
, filepath
, sync
,
219 job_id
=job_id
, **kwargs
)
222 def perform_writes(drive
, n
, filter_node_name
=None):
223 log("--- Write #{:d} ---\n".format(n
))
224 for pattern
in GROUPS
[n
].patterns
:
225 cmd
= "write -P{:s} 0x{:07x} 0x{:x}".format(
230 log(drive
.vm
.hmp_qemu_io(filter_node_name
or drive
.node
, cmd
))
231 bitmaps
= drive
.vm
.query_bitmaps()
232 log({'bitmaps': bitmaps
}, indent
=2)
237 def compare_images(image
, reference
, baseimg
=None, expected_match
=True):
239 Print a nice human-readable message comparing these images.
241 expected_ret
= 0 if expected_match
else 1
243 assert qemu_img("rebase", "-u", "-b", baseimg
, '-F', iotests
.imgfmt
,
245 ret
= qemu_img("compare", image
, reference
)
246 log('qemu_img compare "{:s}" "{:s}" ==> {:s}, {:s}'.format(
248 "Identical" if ret
== 0 else "Mismatch",
249 "OK!" if ret
== expected_ret
else "ERROR!"),
250 filters
=[iotests
.filter_testfiles
])
252 def test_bitmap_sync(bsync_mode
, msync_mode
='bitmap', failure
=None):
254 Test bitmap backup routines.
256 :param bsync_mode: Is the Bitmap Sync mode, and can be any of:
257 - on-success: This is the "incremental" style mode. Bitmaps are
258 synchronized to what was copied out only on success.
259 (Partial images must be discarded.)
260 - never: This is the "differential" style mode.
261 Bitmaps are never synchronized.
262 - always: This is a "best effort" style mode.
263 Bitmaps are always synchronized, regardless of failure.
264 (Partial images must be kept.)
266 :param msync_mode: The mirror sync mode to use for the first backup.
268 - bitmap: Backups based on bitmap manifest.
269 - full: Full backups.
270 - top: Full backups of the top layer only.
272 :param failure: Is the (optional) failure mode, and can be any of:
273 - None: No failure. Test the normative path. Default.
274 - simulated: Cancel the job right before it completes.
275 This also tests writes "during" the job.
276 - intermediate: This tests a job that fails mid-process and produces
277 an incomplete backup. Testing limitations prevent
278 testing competing writes.
280 with iotests
.FilePath(
281 'img', 'bsync1', 'bsync2', 'fbackup0', 'fbackup1', 'fbackup2') as \
282 (img_path
, bsync1
, bsync2
, fbackup0
, fbackup1
, fbackup2
), \
285 mode
= "Mode {:s}; Bitmap Sync {:s}".format(msync_mode
, bsync_mode
)
286 preposition
= "with" if failure
else "without"
287 cond
= "{:s} {:s}".format(preposition
,
288 "{:s} failure".format(failure
) if failure
290 log("\n=== {:s} {:s} ===\n".format(mode
, cond
))
292 log('--- Preparing image & VM ---\n')
293 drive0
= Drive(img_path
, vm
=vm
)
294 drive0
.img_create(iotests
.imgfmt
, SIZE
)
295 vm
.add_device("{},id=scsi0".format('virtio-scsi'))
300 'filename': drive0
.path
303 if failure
== 'intermediate':
305 'driver': 'blkdebug',
306 'image': file_config
,
308 'event': 'flush_to_disk',
320 'immediately': False,
325 drive0
.node
= 'drive0'
326 vm
.qmp_log('blockdev-add',
327 filters
=[iotests
.filter_qmp_testfiles
],
328 node_name
=drive0
.node
,
333 # 0 - Writes and Reference Backup
334 perform_writes(drive0
, 0)
335 reference_backup(drive0
, 0, fbackup0
)
336 log('--- Add Bitmap ---\n')
337 vm
.qmp_log("block-dirty-bitmap-add", node
=drive0
.node
,
338 name
="bitmap0", granularity
=GRANULARITY
)
340 ebitmap
= EmulatedBitmap()
342 # 1 - Writes and Reference Backup
343 bitmaps
= perform_writes(drive0
, 1)
344 ebitmap
.dirty_group(1)
345 bitmap
= vm
.get_bitmap(drive0
.node
, 'bitmap0', bitmaps
=bitmaps
)
346 ebitmap
.compare(bitmap
)
347 reference_backup(drive0
, 1, fbackup1
)
349 # 1 - Test Backup (w/ Optional induced failure)
350 if failure
== 'intermediate':
351 # Activate blkdebug induced failure for second-to-next read
352 log(vm
.hmp_qemu_io(drive0
.node
, 'flush'))
354 job
= backup(drive0
, 1, bsync1
, msync_mode
,
355 bitmap
="bitmap0", bitmap_mode
=bsync_mode
)
358 """Issue writes while the job is open to test bitmap divergence."""
359 # Note: when `failure` is 'intermediate', this isn't called.
361 bitmaps
= perform_writes(drive0
, 2, filter_node_name
='backup-top')
362 # Named bitmap (static, should be unchanged)
363 ebitmap
.compare(vm
.get_bitmap(drive0
.node
, 'bitmap0',
365 # Anonymous bitmap (dynamic, shows new writes)
366 anonymous
= EmulatedBitmap()
367 anonymous
.dirty_group(2)
368 anonymous
.compare(vm
.get_bitmap(drive0
.node
, '', recording
=True,
371 # Simulate the order in which this will happen:
372 # group 1 gets cleared first, then group two gets written.
373 if ((bsync_mode
== 'on-success' and not failure
) or
374 (bsync_mode
== 'always')):
376 ebitmap
.dirty_group(2)
378 vm
.run_job(job
, auto_dismiss
=True, auto_finalize
=False,
379 pre_finalize
=_callback
,
380 cancel
=(failure
== 'simulated'))
381 bitmaps
= vm
.query_bitmaps()
382 log({'bitmaps': bitmaps
}, indent
=2)
385 if bsync_mode
== 'always' and failure
== 'intermediate':
386 # TOP treats anything allocated as dirty, expect to see:
387 if msync_mode
== 'top':
388 ebitmap
.dirty_group(0)
390 # We manage to copy one sector (one bit) before the error.
391 ebitmap
.clear_bit(ebitmap
.first_bit
)
393 # Full returns all bits set except what was copied/skipped
394 if msync_mode
== 'full':
395 fail_bit
= ebitmap
.first_bit
397 ebitmap
.dirty_bits(range(fail_bit
, SIZE
// GRANULARITY
))
399 ebitmap
.compare(vm
.get_bitmap(drive0
.node
, 'bitmap0', bitmaps
=bitmaps
))
401 # 2 - Writes and Reference Backup
402 bitmaps
= perform_writes(drive0
, 3)
403 ebitmap
.dirty_group(3)
404 ebitmap
.compare(vm
.get_bitmap(drive0
.node
, 'bitmap0', bitmaps
=bitmaps
))
405 reference_backup(drive0
, 2, fbackup2
)
407 # 2 - Bitmap Backup (In failure modes, this is a recovery.)
408 job
= backup(drive0
, 2, bsync2
, "bitmap",
409 bitmap
="bitmap0", bitmap_mode
=bsync_mode
)
410 vm
.run_job(job
, auto_dismiss
=True, auto_finalize
=False)
411 bitmaps
= vm
.query_bitmaps()
412 log({'bitmaps': bitmaps
}, indent
=2)
414 if bsync_mode
!= 'never':
416 ebitmap
.compare(vm
.get_bitmap(drive0
.node
, 'bitmap0', bitmaps
=bitmaps
))
418 log('--- Cleanup ---\n')
419 vm
.qmp_log("block-dirty-bitmap-remove",
420 node
=drive0
.node
, name
="bitmap0")
421 bitmaps
= vm
.query_bitmaps()
422 log({'bitmaps': bitmaps
}, indent
=2)
426 log('--- Verification ---\n')
427 # 'simulated' failures will actually all pass here because we canceled
428 # while "pending". This is actually undefined behavior,
429 # don't rely on this to be true!
430 compare_images(bsync1
, fbackup1
, baseimg
=fbackup0
,
431 expected_match
=failure
!= 'intermediate')
432 if not failure
or bsync_mode
== 'always':
433 # Always keep the last backup on success or when using 'always'
437 compare_images(bsync2
, fbackup2
, baseimg
=base
)
438 compare_images(img_path
, fbackup2
)
441 def test_backup_api():
443 Test malformed and prohibited invocations of the backup API.
445 with iotests
.FilePath('img', 'bsync1') as (img_path
, backup_path
), \
448 log("\n=== API failure tests ===\n")
449 log('--- Preparing image & VM ---\n')
450 drive0
= Drive(img_path
, vm
=vm
)
451 drive0
.img_create(iotests
.imgfmt
, SIZE
)
452 vm
.add_device("{},id=scsi0".format('virtio-scsi'))
457 'filename': drive0
.path
460 drive0
.node
= 'drive0'
461 vm
.qmp_log('blockdev-add',
462 filters
=[iotests
.filter_qmp_testfiles
],
463 node_name
=drive0
.node
,
468 target0
= Drive(backup_path
, vm
=vm
)
469 target0
.create_target("backup_target", drive0
.fmt
, drive0
.size
)
472 vm
.qmp_log("block-dirty-bitmap-add", node
=drive0
.node
,
473 name
="bitmap0", granularity
=GRANULARITY
)
476 log('-- Testing invalid QMP commands --\n')
480 None: ['on-success', 'always', 'never', None],
481 'bitmap404': ['on-success', 'always', 'never', None],
482 'bitmap0': ['always', 'never']
485 None: ['on-success', 'always', 'never', None],
486 'bitmap404': ['on-success', 'always', 'never', None],
490 None: ['on-success', 'always', 'never'],
491 'bitmap404': ['on-success', 'always', 'never', None],
492 'bitmap0': ['never', None],
495 None: ['on-success', 'always', 'never'],
496 'bitmap404': ['on-success', 'always', 'never', None],
497 'bitmap0': ['never', None],
500 None: ['on-success', 'always', 'never'],
501 'bitmap404': ['on-success', 'always', 'never', None],
502 'bitmap0': ['on-success', 'always', 'never', None],
506 # Dicts, as always, are not stably-ordered prior to 3.7, so use tuples:
507 for sync_mode
in ('incremental', 'bitmap', 'full', 'top', 'none'):
508 log("-- Sync mode {:s} tests --\n".format(sync_mode
))
509 for bitmap
in (None, 'bitmap404', 'bitmap0'):
510 for policy
in error_cases
[sync_mode
][bitmap
]:
511 blockdev_backup(drive0
.vm
, drive0
.node
, "backup_target",
512 sync_mode
, job_id
='api_job',
513 bitmap
=bitmap
, bitmap_mode
=policy
)
518 for bsync_mode
in ("never", "on-success", "always"):
519 for failure
in ("simulated", "intermediate", None):
520 test_bitmap_sync(bsync_mode
, "bitmap", failure
)
522 for sync_mode
in ('full', 'top'):
523 for bsync_mode
in ('on-success', 'always'):
524 for failure
in ('simulated', 'intermediate', None):
525 test_bitmap_sync(bsync_mode
, sync_mode
, failure
)
529 if __name__
== '__main__':
530 iotests
.script_main(main
, supported_fmts
=['qcow2'],
531 supported_protocols
=['file'])