[mirror_qemu.git] / tests / qemu-iotests / 169

#!/usr/bin/env python3
#
# Tests for dirty bitmaps migration.
#
# Copyright (c) 2016-2017 Virtuozzo International GmbH. All rights reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#

import os
import iotests
import time
import itertools
import operator
import re
from iotests import qemu_img, qemu_img_create, Timeout


disk_a = os.path.join(iotests.test_dir, 'disk_a')
disk_b = os.path.join(iotests.test_dir, 'disk_b')
base_a = os.path.join(iotests.test_dir, 'base_a')
size = '1M'
mig_file = os.path.join(iotests.test_dir, 'mig_file')
mig_cmd = 'exec: cat > ' + mig_file
incoming_cmd = 'exec: cat ' + mig_file


class TestDirtyBitmapMigration(iotests.QMPTestCase):
    def tearDown(self):
        self.vm_a.shutdown()
        self.vm_b.shutdown()
        os.remove(disk_a)
        os.remove(disk_b)
        os.remove(mig_file)

    def setUp(self):
        qemu_img('create', '-f', iotests.imgfmt, disk_a, size)
        qemu_img('create', '-f', iotests.imgfmt, disk_b, size)

        self.vm_a = iotests.VM(path_suffix='a').add_drive(disk_a)
        self.vm_a.launch()

        self.vm_b = iotests.VM(path_suffix='b')

    def add_bitmap(self, vm, granularity, persistent):
        params = {'node': 'drive0',
                  'name': 'bitmap0',
                  'granularity': granularity}
        if persistent:
            params['persistent'] = True

        result = vm.qmp('block-dirty-bitmap-add', **params)
        self.assert_qmp(result, 'return', {});

    def get_bitmap_hash(self, vm):
        result = vm.qmp('x-debug-block-dirty-bitmap-sha256',
                        node='drive0', name='bitmap0')
        return result['return']['sha256']

    def check_bitmap(self, vm, sha256):
        result = vm.qmp('x-debug-block-dirty-bitmap-sha256',
                        node='drive0', name='bitmap0')
        if sha256:
            self.assert_qmp(result, 'return/sha256', sha256);
        else:
            self.assert_qmp(result, 'error/desc',
                            "Dirty bitmap 'bitmap0' not found");

    def do_test_migration_resume_source(self, persistent, migrate_bitmaps):
        granularity = 512

        # regions = ((start, count), ...)
        regions = ((0, 0x10000),
                   (0xf0000, 0x10000),
                   (0xa0201, 0x1000))

        mig_caps = [{'capability': 'events', 'state': True}]
        if migrate_bitmaps:
            mig_caps.append({'capability': 'dirty-bitmaps', 'state': True})

        result = self.vm_a.qmp('migrate-set-capabilities',
                               capabilities=mig_caps)
        self.assert_qmp(result, 'return', {})

        self.add_bitmap(self.vm_a, granularity, persistent)
        for r in regions:
            self.vm_a.hmp_qemu_io('drive0', 'write %d %d' % r)
        sha256 = self.get_bitmap_hash(self.vm_a)

        result = self.vm_a.qmp('migrate', uri=mig_cmd)
        while True:
            event = self.vm_a.event_wait('MIGRATION')
            if event['data']['status'] == 'completed':
                break
        while True:
            result = self.vm_a.qmp('query-status')
            if (result['return']['status'] == 'postmigrate'):
                break

        # test that bitmap is still here
        removed = (not migrate_bitmaps) and persistent
        self.check_bitmap(self.vm_a, False if removed else sha256)

        result = self.vm_a.qmp('cont')
        self.assert_qmp(result, 'return', {})

        # test that bitmap is still here after invalidation
        self.check_bitmap(self.vm_a, sha256)

        # shutdown and check that invalidation didn't fail
        self.vm_a.shutdown()

        # catch 'Could not reopen qcow2 layer: Bitmap already exists'
        # possible error
        log = self.vm_a.get_log()
        log = re.sub(r'^\[I \d+\.\d+\] OPENED\n', '', log)
        log = re.sub(r'^(wrote .* bytes at offset .*\n.*KiB.*ops.*sec.*\n){3}',
                     '', log)
        log = re.sub(r'\[I \+\d+\.\d+\] CLOSED\n?$', '', log)
        self.assertEqual(log, '')

        # test that bitmap is still persistent
        self.vm_a.launch()
        self.check_bitmap(self.vm_a, sha256 if persistent else False)

    def do_test_migration(self, persistent, migrate_bitmaps, online,
                          shared_storage, pre_shutdown):
        granularity = 512

        # regions = ((start, count), ...)
        regions = ((0, 0x10000),
                   (0xf0000, 0x10000),
                   (0xa0201, 0x1000))

        should_migrate = \
            (migrate_bitmaps and (persistent or not pre_shutdown)) or \
            (persistent and shared_storage)
        mig_caps = [{'capability': 'events', 'state': True}]
        if migrate_bitmaps:
            mig_caps.append({'capability': 'dirty-bitmaps', 'state': True})

        self.vm_b.add_incoming(incoming_cmd if online else "defer")
        self.vm_b.add_drive(disk_a if shared_storage else disk_b)

        if online:
            os.mkfifo(mig_file)
            self.vm_b.launch()
            result = self.vm_b.qmp('migrate-set-capabilities',
                                   capabilities=mig_caps)
            self.assert_qmp(result, 'return', {})

        self.add_bitmap(self.vm_a, granularity, persistent)
        for r in regions:
            self.vm_a.hmp_qemu_io('drive0', 'write %d %d' % r)
        sha256 = self.get_bitmap_hash(self.vm_a)

        if pre_shutdown:
            self.vm_a.shutdown()
            self.vm_a.launch()

        result = self.vm_a.qmp('migrate-set-capabilities',
                               capabilities=mig_caps)
        self.assert_qmp(result, 'return', {})

        result = self.vm_a.qmp('migrate', uri=mig_cmd)
        while True:
            event = self.vm_a.event_wait('MIGRATION')
            if event['data']['status'] == 'completed':
                break

        if not online:
            self.vm_a.shutdown()
            self.vm_b.launch()
            result = self.vm_b.qmp('migrate-set-capabilities',
                                   capabilities=mig_caps)
            self.assert_qmp(result, 'return', {})
            result = self.vm_b.qmp('migrate-incoming', uri=incoming_cmd)
            self.assert_qmp(result, 'return', {})

        while True:
            event = self.vm_b.event_wait('MIGRATION')
            if event['data']['status'] == 'completed':
                break

        self.check_bitmap(self.vm_b, sha256 if should_migrate else False)

        if should_migrate:
            self.vm_b.shutdown()

            # catch 'Could not reopen qcow2 layer: Bitmap already exists'
            # possible error
            log = self.vm_b.get_log()
            log = re.sub(r'^\[I \d+\.\d+\] OPENED\n', '', log)
            log = re.sub(r'\[I \+\d+\.\d+\] CLOSED\n?$', '', log)
            self.assertEqual(log, '')

            # recreate vm_b, as we don't want -incoming option (this will lead
            # to "cat" process left alive after test finish)
            self.vm_b = iotests.VM(path_suffix='b')
            self.vm_b.add_drive(disk_a if shared_storage else disk_b)
            self.vm_b.launch()
            self.check_bitmap(self.vm_b, sha256 if persistent else False)


def inject_test_case(klass, name, method, *args, **kwargs):
    mc = operator.methodcaller(method, *args, **kwargs)
    setattr(klass, 'test_' + method + name, lambda self: mc(self))

for cmb in list(itertools.product((True, False), repeat=5)):
    name = ('_' if cmb[0] else '_not_') + 'persistent_'
    name += ('_' if cmb[1] else '_not_') + 'migbitmap_'
    name += '_online' if cmb[2] else '_offline'
    name += '_shared' if cmb[3] else '_nonshared'
    if (cmb[4]):
        name += '__pre_shutdown'

    inject_test_case(TestDirtyBitmapMigration, name, 'do_test_migration',
                     *list(cmb))

for cmb in list(itertools.product((True, False), repeat=2)):
    name = ('_' if cmb[0] else '_not_') + 'persistent_'
    name += ('_' if cmb[1] else '_not_') + 'migbitmap'

    inject_test_case(TestDirtyBitmapMigration, name,
                     'do_test_migration_resume_source', *list(cmb))


class TestDirtyBitmapBackingMigration(iotests.QMPTestCase):
    def setUp(self):
        qemu_img_create('-f', iotests.imgfmt, base_a, size)
        qemu_img_create('-f', iotests.imgfmt, '-F', iotests.imgfmt,
                        '-b', base_a, disk_a, size)

        for f in (disk_a, base_a):
            qemu_img('bitmap', '--add', f, 'bmap0')

        blockdev = {
            'node-name': 'node0',
            'driver': iotests.imgfmt,
            'file': {
                'driver': 'file',
                'filename': disk_a
            },
            'backing': {
                'node-name': 'node0-base',
                'driver': iotests.imgfmt,
                'file': {
                    'driver': 'file',
                    'filename': base_a
                }
            }
        }

        self.vm = iotests.VM()
        self.vm.launch()

        result = self.vm.qmp('blockdev-add', **blockdev)
        self.assert_qmp(result, 'return', {})

        # Check that the bitmaps are there
        for node in self.vm.qmp('query-named-block-nodes', flat=True)['return']:
            if 'node0' in node['node-name']:
                self.assert_qmp(node, 'dirty-bitmaps[0]/name', 'bmap0')

        caps = [{'capability': 'events', 'state': True}]
        result = self.vm.qmp('migrate-set-capabilities', capabilities=caps)
        self.assert_qmp(result, 'return', {})

    def tearDown(self):
        self.vm.shutdown()
        for f in (disk_a, base_a):
            os.remove(f)

    def test_cont_on_source(self):
        """
        Continue the source after migration.
        """
        result = self.vm.qmp('migrate', uri=f'exec: cat > /dev/null')
        self.assert_qmp(result, 'return', {})

        with Timeout(10, 'Migration timeout'):
            self.vm.wait_migration('postmigrate')

        result = self.vm.qmp('cont')
        self.assert_qmp(result, 'return', {})


if __name__ == '__main__':
    iotests.main(supported_fmts=['qcow2'],
                 supported_protocols=['file'])
Commit	Line	Data
903cb1bf	1	#!/usr/bin/env python3
33dac6f3 VSO	2	#
	3	# Tests for dirty bitmaps migration.
	4	#
	5	# Copyright (c) 2016-2017 Virtuozzo International GmbH. All rights reserved.
	6	#
	7	# This program is free software; you can redistribute it and/or modify
	8	# it under the terms of the GNU General Public License as published by
	9	# the Free Software Foundation; either version 2 of the License, or
	10	# (at your option) any later version.
	11	#
	12	# This program is distributed in the hope that it will be useful,
	13	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	15	# GNU General Public License for more details.
	16	#
	17	# You should have received a copy of the GNU General Public License
	18	# along with this program. If not, see <http://www.gnu.org/licenses/>.
	19	#
	20
	21	import os
	22	import iotests
	23	import time
	24	import itertools
	25	import operator
b9247fc1	26	import re
edadc99a	27	from iotests import qemu_img, qemu_img_create, Timeout
33dac6f3 VSO	28
	29
	30	disk_a = os.path.join(iotests.test_dir, 'disk_a')
	31	disk_b = os.path.join(iotests.test_dir, 'disk_b')
edadc99a	32	base_a = os.path.join(iotests.test_dir, 'base_a')
33dac6f3 VSO	33	size = '1M'
33dac6f3 VSO	34	mig_file = os.path.join(iotests.test_dir, 'mig_file')
25bf2426 VSO	35	mig_cmd = 'exec: cat > ' + mig_file
25bf2426 VSO	36	incoming_cmd = 'exec: cat ' + mig_file
33dac6f3 VSO	37
	38
	39	class TestDirtyBitmapMigration(iotests.QMPTestCase):
	40	def tearDown(self):
	41	self.vm_a.shutdown()
	42	self.vm_b.shutdown()
	43	os.remove(disk_a)
	44	os.remove(disk_b)
	45	os.remove(mig_file)
	46
	47	def setUp(self):
	48	qemu_img('create', '-f', iotests.imgfmt, disk_a, size)
	49	qemu_img('create', '-f', iotests.imgfmt, disk_b, size)
	50
	51	self.vm_a = iotests.VM(path_suffix='a').add_drive(disk_a)
	52	self.vm_a.launch()
	53
	54	self.vm_b = iotests.VM(path_suffix='b')
33dac6f3 VSO	55
	56	def add_bitmap(self, vm, granularity, persistent):
	57	params = {'node': 'drive0',
	58	'name': 'bitmap0',
	59	'granularity': granularity}
	60	if persistent:
	61	params['persistent'] = True
33dac6f3 VSO	62
	63	result = vm.qmp('block-dirty-bitmap-add', **params)
	64	self.assert_qmp(result, 'return', {});
	65
	66	def get_bitmap_hash(self, vm):
	67	result = vm.qmp('x-debug-block-dirty-bitmap-sha256',
	68	node='drive0', name='bitmap0')
	69	return result['return']['sha256']
	70
	71	def check_bitmap(self, vm, sha256):
	72	result = vm.qmp('x-debug-block-dirty-bitmap-sha256',
	73	node='drive0', name='bitmap0')
	74	if sha256:
	75	self.assert_qmp(result, 'return/sha256', sha256);
	76	else:
	77	self.assert_qmp(result, 'error/desc',
	78	"Dirty bitmap 'bitmap0' not found");
	79
3e6d88f2 VSO	80	def do_test_migration_resume_source(self, persistent, migrate_bitmaps):
	81	granularity = 512
	82
	83	# regions = ((start, count), ...)
	84	regions = ((0, 0x10000),
	85	(0xf0000, 0x10000),
	86	(0xa0201, 0x1000))
	87
	88	mig_caps = [{'capability': 'events', 'state': True}]
	89	if migrate_bitmaps:
	90	mig_caps.append({'capability': 'dirty-bitmaps', 'state': True})
	91
	92	result = self.vm_a.qmp('migrate-set-capabilities',
	93	capabilities=mig_caps)
	94	self.assert_qmp(result, 'return', {})
	95
	96	self.add_bitmap(self.vm_a, granularity, persistent)
	97	for r in regions:
	98	self.vm_a.hmp_qemu_io('drive0', 'write %d %d' % r)
	99	sha256 = self.get_bitmap_hash(self.vm_a)
	100
	101	result = self.vm_a.qmp('migrate', uri=mig_cmd)
	102	while True:
	103	event = self.vm_a.event_wait('MIGRATION')
	104	if event['data']['status'] == 'completed':
	105	break
832d78ca VSO	106	while True:
	107	result = self.vm_a.qmp('query-status')
	108	if (result['return']['status'] == 'postmigrate'):
	109	break
3e6d88f2 VSO	110
	111	# test that bitmap is still here
	112	removed = (not migrate_bitmaps) and persistent
	113	self.check_bitmap(self.vm_a, False if removed else sha256)
	114
832d78ca VSO	115	result = self.vm_a.qmp('cont')
832d78ca VSO	116	self.assert_qmp(result, 'return', {})
3e6d88f2 VSO	117
	118	# test that bitmap is still here after invalidation
	119	self.check_bitmap(self.vm_a, sha256)
	120
	121	# shutdown and check that invalidation didn't fail
	122	self.vm_a.shutdown()
	123
	124	# catch 'Could not reopen qcow2 layer: Bitmap already exists'
	125	# possible error
	126	log = self.vm_a.get_log()
	127	log = re.sub(r'^\[I \d+\.\d+\] OPENED\n', '', log)
	128	log = re.sub(r'^(wrote .* bytes at offset .\n.KiB.ops.sec.*\n){3}',
	129	'', log)
	130	log = re.sub(r'\[I \+\d+\.\d+\] CLOSED\n?$', '', log)
	131	self.assertEqual(log, '')
	132
	133	# test that bitmap is still persistent
	134	self.vm_a.launch()
	135	self.check_bitmap(self.vm_a, sha256 if persistent else False)
	136
33dac6f3	137	def do_test_migration(self, persistent, migrate_bitmaps, online,
d8130f4c	138	shared_storage, pre_shutdown):
33dac6f3 VSO	139	granularity = 512
	140
	141	# regions = ((start, count), ...)
	142	regions = ((0, 0x10000),
	143	(0xf0000, 0x10000),
	144	(0xa0201, 0x1000))
	145
d8130f4c VSO	146	should_migrate = \
	147	(migrate_bitmaps and (persistent or not pre_shutdown)) or \
	148	(persistent and shared_storage)
25bf2426 VSO	149	mig_caps = [{'capability': 'events', 'state': True}]
	150	if migrate_bitmaps:
	151	mig_caps.append({'capability': 'dirty-bitmaps', 'state': True})
33dac6f3	152
25bf2426	153	self.vm_b.add_incoming(incoming_cmd if online else "defer")
33dac6f3 VSO	154	self.vm_b.add_drive(disk_a if shared_storage else disk_b)
	155
	156	if online:
	157	os.mkfifo(mig_file)
	158	self.vm_b.launch()
25bf2426 VSO	159	result = self.vm_b.qmp('migrate-set-capabilities',
	160	capabilities=mig_caps)
	161	self.assert_qmp(result, 'return', {})
33dac6f3 VSO	162
	163	self.add_bitmap(self.vm_a, granularity, persistent)
	164	for r in regions:
	165	self.vm_a.hmp_qemu_io('drive0', 'write %d %d' % r)
	166	sha256 = self.get_bitmap_hash(self.vm_a)
	167
d8130f4c VSO	168	if pre_shutdown:
	169	self.vm_a.shutdown()
	170	self.vm_a.launch()
	171
	172	result = self.vm_a.qmp('migrate-set-capabilities',
	173	capabilities=mig_caps)
	174	self.assert_qmp(result, 'return', {})
	175
25bf2426	176	result = self.vm_a.qmp('migrate', uri=mig_cmd)
33dac6f3 VSO	177	while True:
	178	event = self.vm_a.event_wait('MIGRATION')
	179	if event['data']['status'] == 'completed':
	180	break
	181
	182	if not online:
	183	self.vm_a.shutdown()
	184	self.vm_b.launch()
25bf2426 VSO	185	result = self.vm_b.qmp('migrate-set-capabilities',
	186	capabilities=mig_caps)
	187	self.assert_qmp(result, 'return', {})
	188	result = self.vm_b.qmp('migrate-incoming', uri=incoming_cmd)
	189	self.assert_qmp(result, 'return', {})
33dac6f3	190
25bf2426 VSO	191	while True:
	192	event = self.vm_b.event_wait('MIGRATION')
	193	if event['data']['status'] == 'completed':
	194	break
33dac6f3 VSO	195
	196	self.check_bitmap(self.vm_b, sha256 if should_migrate else False)
	197
	198	if should_migrate:
	199	self.vm_b.shutdown()
b9247fc1 VSO	200
	201	# catch 'Could not reopen qcow2 layer: Bitmap already exists'
	202	# possible error
	203	log = self.vm_b.get_log()
	204	log = re.sub(r'^\[I \d+\.\d+\] OPENED\n', '', log)
	205	log = re.sub(r'\[I \+\d+\.\d+\] CLOSED\n?$', '', log)
	206	self.assertEqual(log, '')
	207
25bf2426 VSO	208	# recreate vm_b, as we don't want -incoming option (this will lead
	209	# to "cat" process left alive after test finish)
	210	self.vm_b = iotests.VM(path_suffix='b')
	211	self.vm_b.add_drive(disk_a if shared_storage else disk_b)
33dac6f3 VSO	212	self.vm_b.launch()
	213	self.check_bitmap(self.vm_b, sha256 if persistent else False)
	214
	215
	216	def inject_test_case(klass, name, method, args, *kwargs):
	217	mc = operator.methodcaller(method, args, *kwargs)
c1a65cba	218	setattr(klass, 'test_' + method + name, lambda self: mc(self))
33dac6f3	219
d8130f4c	220	for cmb in list(itertools.product((True, False), repeat=5)):
33dac6f3 VSO	221	name = ('_' if cmb[0] else '_not_') + 'persistent_'
	222	name += ('_' if cmb[1] else '_not_') + 'migbitmap_'
	223	name += '_online' if cmb[2] else '_offline'
f7640f0d	224	name += '_shared' if cmb[3] else '_nonshared'
d8130f4c VSO	225	if (cmb[4]):
d8130f4c VSO	226	name += '__pre_shutdown'
33dac6f3 VSO	227
33dac6f3 VSO	228	inject_test_case(TestDirtyBitmapMigration, name, 'do_test_migration',
f7640f0d	229	*list(cmb))
33dac6f3	230
3e6d88f2 VSO	231	for cmb in list(itertools.product((True, False), repeat=2)):
	232	name = ('_' if cmb[0] else '_not_') + 'persistent_'
	233	name += ('_' if cmb[1] else '_not_') + 'migbitmap'
	234
	235	inject_test_case(TestDirtyBitmapMigration, name,
	236	'do_test_migration_resume_source', *list(cmb))
33dac6f3	237
edadc99a HR	238
	239	class TestDirtyBitmapBackingMigration(iotests.QMPTestCase):
	240	def setUp(self):
	241	qemu_img_create('-f', iotests.imgfmt, base_a, size)
	242	qemu_img_create('-f', iotests.imgfmt, '-F', iotests.imgfmt,
	243	'-b', base_a, disk_a, size)
	244
	245	for f in (disk_a, base_a):
	246	qemu_img('bitmap', '--add', f, 'bmap0')
	247
	248	blockdev = {
	249	'node-name': 'node0',
	250	'driver': iotests.imgfmt,
	251	'file': {
	252	'driver': 'file',
	253	'filename': disk_a
	254	},
	255	'backing': {
	256	'node-name': 'node0-base',
	257	'driver': iotests.imgfmt,
	258	'file': {
	259	'driver': 'file',
	260	'filename': base_a
	261	}
	262	}
	263	}
	264
	265	self.vm = iotests.VM()
	266	self.vm.launch()
	267
	268	result = self.vm.qmp('blockdev-add', **blockdev)
	269	self.assert_qmp(result, 'return', {})
	270
	271	# Check that the bitmaps are there
	272	for node in self.vm.qmp('query-named-block-nodes', flat=True)['return']:
	273	if 'node0' in node['node-name']:
	274	self.assert_qmp(node, 'dirty-bitmaps[0]/name', 'bmap0')
	275
	276	caps = [{'capability': 'events', 'state': True}]
	277	result = self.vm.qmp('migrate-set-capabilities', capabilities=caps)
	278	self.assert_qmp(result, 'return', {})
	279
	280	def tearDown(self):
	281	self.vm.shutdown()
	282	for f in (disk_a, base_a):
	283	os.remove(f)
	284
	285	def test_cont_on_source(self):
	286	"""
	287	Continue the source after migration.
	288	"""
	289	result = self.vm.qmp('migrate', uri=f'exec: cat > /dev/null')
	290	self.assert_qmp(result, 'return', {})
	291
	292	with Timeout(10, 'Migration timeout'):
	293	self.vm.wait_migration('postmigrate')
	294
	295	result = self.vm.qmp('cont')
	296	self.assert_qmp(result, 'return', {})
	297
	298
33dac6f3	299	if __name__ == '__main__':
103cbc77 HR	300	iotests.main(supported_fmts=['qcow2'],
103cbc77 HR	301	supported_protocols=['file'])