From 8cb3d98deed8993fda03d3effeeb7c397a5dc56f Mon Sep 17 00:00:00 2001 From: Jean-Louis Dupond Date: Mon, 12 Jun 2023 15:01:15 +0200 Subject: [PATCH] Add bitmap info to getQemuImageInfo We add bitmap info the the getQemuImageInfo output so this can be used within ovirt-engine. Signed-off-by: Jean-Louis Dupond --- lib/vdsm/api/vdsm-api.yml | 43 +++++++++++++++++++++++++ lib/vdsm/host/caps.py | 1 + lib/vdsm/storage/volume.py | 10 ++++++ tests/storage/blocksd_test.py | 18 +++++++++++ tests/storage/localfssd_test.py | 18 +++++++++++ tests/storage/sdm_add_bitmap_test.py | 6 ++++ tests/storage/sdm_clear_bitmaps_test.py | 15 +++++++++ tests/storage/sdm_copy_data_test.py | 7 ++++ tests/storage/sdm_merge_test.py | 7 ++++ tests/storage/sdm_remove_bitmap_test.py | 17 ++++++++++ 10 files changed, 142 insertions(+) diff --git a/lib/vdsm/api/vdsm-api.yml b/lib/vdsm/api/vdsm-api.yml index 2382ecfb7a..3055e5d938 100644 --- a/lib/vdsm/api/vdsm-api.yml +++ b/lib/vdsm/api/vdsm-api.yml @@ -557,6 +557,46 @@ types: 1.1: qemu versions starting with 1.1 which are supported by QCOW2 version 3. + Qcow2BitmapInfoFlags: &Qcow2BitmapInfoFlags + added: '4.5.5' + description: An enumeration of qcow compat version. + name: Qcow2BitmapInfoFlags + type: enum + values: + 'in-use': This flag is set by any process actively modifying the + qcow2 file, and cleared when the updated bitmap is flushed + to the qcow2 image. The presence of this flag in an + offline image means that the bitmap was not saved correctly + after its last usage, and may contain inconsistent data. + 'auto': The bitmap must reflect all changes of the virtual disk by + any application that would write to this qcow2 file. + + Qcow2BitmapInfo: &Qcow2BitmapInfo + added: '4.5.5' + description: Information about a Bitmap + name: Qcow2BitmapInfo + properties: + - description: The UUID of bitmap + name: name + type: *UUID + + - description: Granularity of the bitmap + name: granularity + type: uint + + - description: Bitmap flags + name: flags + defaultvalue: '[]' + type: *Qcow2BitmapInfoFlags + type: object + + Qcow2Bitmaps: &Qcow2Bitmaps + added: '4.5.5' + description: A list of qcow2 bitmaps + name: Qcow2Bitmaps + defaultvalue: '[]' + type: *Qcow2BitmapInfo + Qcow2Attributes: &Qcow2Attributes added: '4.1' description: Possible QCOW2 attributes which are allowed @@ -8131,6 +8171,9 @@ types: - description: The compat version. name: compat type: *Qcow2Compat + - description: Volume bitmaps + name: bitmaps + type: *Qcow2Bitmaps type: object VolumeSizeInfo: &VolumeSizeInfo diff --git a/lib/vdsm/host/caps.py b/lib/vdsm/host/caps.py index c6ae827bc8..87b87d5d3c 100644 --- a/lib/vdsm/host/caps.py +++ b/lib/vdsm/host/caps.py @@ -175,6 +175,7 @@ def get(): caps['measure_active'] = True caps['mailbox_events'] = config.getboolean("mailbox", "events_enable") caps['zerocopy_migrations'] = hasattr(libvirt, 'VIR_MIGRATE_ZEROCOPY') + caps['qemu_image_info_bitmaps'] = True return caps diff --git a/lib/vdsm/storage/volume.py b/lib/vdsm/storage/volume.py index 066fc69eb9..88551b9d16 100644 --- a/lib/vdsm/storage/volume.py +++ b/lib/vdsm/storage/volume.py @@ -5,6 +5,7 @@ import os.path import logging +from collections import namedtuple from contextlib import contextmanager from vdsm import utils @@ -27,6 +28,8 @@ log = logging.getLogger('storage.volume') +Qcow2BitmapInfo = namedtuple("Qcow2BitmapInfo", "name, granularity, flags") + def getBackingVolumePath(imgUUID, volUUID): # We used to return a relative path ..// but this caused @@ -289,6 +292,13 @@ def getQemuImageInfo(self): result["backingfile"] = info["backing-filename"] if "format-specific" in info: result["compat"] = info["format-specific"]["data"]["compat"] + if "bitmaps" in info["format-specific"]["data"]: + result["bitmaps"] = [ + Qcow2BitmapInfo(bitmap["name"], + bitmap["granularity"], + bitmap["flags"]) + for bitmap in info["format-specific"]["data"]["bitmaps"] + ] return result diff --git a/tests/storage/blocksd_test.py b/tests/storage/blocksd_test.py index 63dc6055c9..637ba1a6ef 100644 --- a/tests/storage/blocksd_test.py +++ b/tests/storage/blocksd_test.py @@ -32,6 +32,7 @@ from vdsm.storage.sdc import sdCache from vdsm.storage.sdm.api import merge as api_merge from vdsm.storage.spbackends import StoragePoolDiskBackend +from vdsm.storage.volume import Qcow2BitmapInfo from . import qemuio from . marks import requires_root @@ -1092,6 +1093,7 @@ def test_create_snapshot_cloning_bitmaps( top_vol.prepare() info = qemuimg.info(top_vol_path) + qemuInfo = top_vol.getQemuImageInfo() # Teardown top volume base_vol.teardown(sd_uuid, top_vol_uuid) @@ -1109,6 +1111,11 @@ def test_create_snapshot_cloning_bitmaps( }, ] + assert qemuInfo["bitmaps"] == [ + Qcow2BitmapInfo(bitmap_names[0], 65536, ["auto"]), + Qcow2BitmapInfo(bitmap_names[1], 65536, ["auto"]), + ] + @requires_root @pytest.mark.root @@ -1178,6 +1185,7 @@ def test_create_snapshot_with_new_bitmap( top.prepare() info = qemuimg.info(top.getVolumePath()) + qemuInfo = top.getQemuImageInfo() top.teardown(sd_id, top_id) assert info['format-specific']['data']['bitmaps'] == [ @@ -1193,6 +1201,11 @@ def test_create_snapshot_with_new_bitmap( }, ] + assert qemuInfo["bitmaps"] == [ + Qcow2BitmapInfo("old-bitmap", 65536, ["auto"]), + Qcow2BitmapInfo("new-bitmap", 65536, ["auto"]), + ] + @requires_root @pytest.mark.root @@ -1240,6 +1253,7 @@ def test_create_volume_with_new_bitmap( vol.prepare() info = qemuimg.info(vol.getVolumePath()) + qemuInfo = vol.getQemuImageInfo() vol.teardown(sd_id, vol_id) assert info['format-specific']['data']['bitmaps'] == [ @@ -1250,6 +1264,10 @@ def test_create_volume_with_new_bitmap( }, ] + assert qemuInfo["bitmaps"] == [ + Qcow2BitmapInfo("new-bitmap", 65536, ["auto"]), + ] + @requires_root @pytest.mark.root diff --git a/tests/storage/localfssd_test.py b/tests/storage/localfssd_test.py index 5d969fab1f..107f8ba4fb 100644 --- a/tests/storage/localfssd_test.py +++ b/tests/storage/localfssd_test.py @@ -20,6 +20,7 @@ from vdsm.storage import localFsSD from vdsm.storage import qemuimg from vdsm.storage import sd +from vdsm.storage.volume import Qcow2BitmapInfo from . import qemuio from . marks import requires_unprivileged_user @@ -1316,6 +1317,12 @@ def test_create_snapshot_cloning_bitmaps(user_domain, local_fallocate): }, ] + qemuInfo = vol.getQemuImageInfo() + assert qemuInfo["bitmaps"] == [ + Qcow2BitmapInfo(bitmap_names[0], 65536, ["auto"]), + Qcow2BitmapInfo(bitmap_names[1], 65536, ["auto"]), + ] + def test_create_snapshot_with_new_bitmap(user_domain, local_fallocate): if user_domain.getVersion() == 3: @@ -1374,6 +1381,12 @@ def test_create_snapshot_with_new_bitmap(user_domain, local_fallocate): }, ] + qemuInfo = top.getQemuImageInfo() + assert qemuInfo["bitmaps"] == [ + Qcow2BitmapInfo("old-bitmap", 65536, ["auto"]), + Qcow2BitmapInfo("new-bitmap", 65536, ["auto"]), + ] + def test_create_volume_with_new_bitmap(user_domain, local_fallocate): if user_domain.getVersion() == 3: @@ -1408,6 +1421,11 @@ def test_create_volume_with_new_bitmap(user_domain, local_fallocate): }, ] + qemuInfo = vol.getQemuImageInfo() + assert qemuInfo["bitmaps"] == [ + Qcow2BitmapInfo("new-bitmap", 65536, ["auto"]), + ] + def test_fail_add_bitmaps_to_v3_domain(user_domain, local_fallocate): if user_domain.getVersion() != 3: diff --git a/tests/storage/sdm_add_bitmap_test.py b/tests/storage/sdm_add_bitmap_test.py index 5c7714a812..932d1603e4 100644 --- a/tests/storage/sdm_add_bitmap_test.py +++ b/tests/storage/sdm_add_bitmap_test.py @@ -30,6 +30,7 @@ from vdsm.storage import qemuimg from vdsm.storage.sdm import volume_info from vdsm.storage.sdm.api import add_bitmap +from vdsm.storage.volume import Qcow2BitmapInfo def failure(*args, **kwargs): @@ -76,6 +77,11 @@ def test_add_bitmap(fake_scheduler, env_type): assert qcow2_data["bitmaps"][0]["name"] == bitmap assert env_vol.getMetaParam(sc.GENERATION) == generation + 1 + qemuInfo = env_vol.getQemuImageInfo() + assert qemuInfo["bitmaps"] == [ + Qcow2BitmapInfo(bitmap, 65536, ["auto"]), + ] + @pytest.mark.parametrize("env_type", ["file", "block"]) def test_vol_type_not_qcow(fake_scheduler, env_type): diff --git a/tests/storage/sdm_clear_bitmaps_test.py b/tests/storage/sdm_clear_bitmaps_test.py index e8ff2ec82b..7de23cfb77 100644 --- a/tests/storage/sdm_clear_bitmaps_test.py +++ b/tests/storage/sdm_clear_bitmaps_test.py @@ -79,6 +79,9 @@ def test_clear_bitmaps(fake_scheduler, env_type): assert "bitmaps" not in vol_info["format-specific"]["data"] assert top_vol.getMetaParam(sc.GENERATION) == generation + 1 + qemuInfo = top_vol.getQemuImageInfo() + assert 'bitmaps' not in qemuInfo + @pytest.mark.parametrize("env_type", ["file", "block"]) def test_clear_invalid_bitmaps(fake_scheduler, env_type): @@ -110,6 +113,9 @@ def test_clear_invalid_bitmaps(fake_scheduler, env_type): assert "bitmaps" not in vol_info["format-specific"]["data"] assert top_vol.getMetaParam(sc.GENERATION) == generation + 1 + qemuInfo = top_vol.getQemuImageInfo() + assert 'bitmaps' not in qemuInfo + @pytest.mark.parametrize("env_type", ["file", "block"]) def test_vol_type_not_qcow(fake_scheduler, env_type): @@ -179,6 +185,9 @@ def test_clear_missing_bitmaps(fake_scheduler, env_type): assert not bitmaps assert top_vol.getMetaParam(sc.GENERATION) == generation + 1 + qemuInfo = top_vol.getQemuImageInfo() + assert 'bitmaps' not in qemuInfo + @pytest.mark.parametrize("env_type", ["file", "block"]) def test_clear_bitmaps_from_vol_chain(fake_scheduler, env_type): @@ -214,6 +223,9 @@ def test_clear_bitmaps_from_vol_chain(fake_scheduler, env_type): assert not bitmaps assert leaf_vol.getMetaParam(sc.GENERATION) == generation + 1 + qemuInfo = leaf_vol.getQemuImageInfo() + assert 'bitmaps' not in qemuInfo + # Clear all the bitmaps from an internal volume internal_vol = env.chain[1] generation = internal_vol.getMetaParam(sc.GENERATION) @@ -234,3 +246,6 @@ def test_clear_bitmaps_from_vol_chain(fake_scheduler, env_type): bitmaps = vol_info["format-specific"]["data"].get("bitmaps", []) assert not bitmaps assert internal_vol.getMetaParam(sc.GENERATION) == generation + 1 + + qemuInfo = internal_vol.getQemuImageInfo() + assert 'bitmaps' not in qemuInfo diff --git a/tests/storage/sdm_copy_data_test.py b/tests/storage/sdm_copy_data_test.py index a590f81c1d..fe4ce36c4d 100644 --- a/tests/storage/sdm_copy_data_test.py +++ b/tests/storage/sdm_copy_data_test.py @@ -48,6 +48,7 @@ from vdsm.storage import volume from vdsm.storage import workarounds from vdsm.storage.sdm.api import copy_data +from vdsm.storage.volume import Qcow2BitmapInfo BACKENDS = userstorage.load_config("storage.py").BACKENDS DEFAULT_SIZE = MiB @@ -302,6 +303,12 @@ def test_volume_chain_copy_with_bitmaps( }, ] + qemuInfo = dst_vol.getQemuImageInfo() + assert qemuInfo["bitmaps"] == [ + Qcow2BitmapInfo(bitmaps[0], 65536, ["auto"]), + Qcow2BitmapInfo(bitmaps[1], 65536, ["auto"]), + ] + @pytest.mark.parametrize( "env_type, dst_fmt", [ diff --git a/tests/storage/sdm_merge_test.py b/tests/storage/sdm_merge_test.py index 2338a37acb..293bcc865f 100644 --- a/tests/storage/sdm_merge_test.py +++ b/tests/storage/sdm_merge_test.py @@ -38,6 +38,7 @@ from vdsm.storage import resourceManager as rm from vdsm.storage import volume from vdsm.storage.sdm.api import merge as api_merge +from vdsm.storage.volume import Qcow2BitmapInfo class FakeImage(object): @@ -275,3 +276,9 @@ def test_merge_subchain_with_bitmaps( "granularity": 65536 }, ] + + qemuInfo = base_vol.getQemuImageInfo() + assert qemuInfo["bitmaps"] == [ + Qcow2BitmapInfo(bitmap1_name, 65536, ["auto"]), + Qcow2BitmapInfo(bitmap2_name, 65536, ["auto"]), + ] diff --git a/tests/storage/sdm_remove_bitmap_test.py b/tests/storage/sdm_remove_bitmap_test.py index db7e04a979..9568bd63c5 100644 --- a/tests/storage/sdm_remove_bitmap_test.py +++ b/tests/storage/sdm_remove_bitmap_test.py @@ -87,6 +87,10 @@ def test_add_remove_bitmap(fake_scheduler, env_type): assert bitmap1 not in bitmaps and bitmap2 in bitmaps assert top_vol.getMetaParam(sc.GENERATION) == generation + 1 + qemuInfo = top_vol.getQemuImageInfo() + assert not any(bitmap[0] == bitmap1 for bitmap in qemuInfo["bitmaps"]) + assert any(bitmap[0] == bitmap2 for bitmap in qemuInfo["bitmaps"]) + @pytest.mark.parametrize("env_type", ["file", "block"]) def test_vol_type_not_qcow(fake_scheduler, env_type): @@ -143,6 +147,10 @@ def test_remove_bitmap_non_leaf_vol(fake_scheduler, env_type): assert bitmap1 not in bitmaps and bitmap2 in bitmaps assert base_vol.getMetaParam(sc.GENERATION) == generation + 1 + qemuInfo = base_vol.getQemuImageInfo() + assert not any(bitmap[0] == bitmap1 for bitmap in qemuInfo["bitmaps"]) + assert any(bitmap[0] == bitmap2 for bitmap in qemuInfo["bitmaps"]) + @pytest.mark.parametrize("env_type", ["file", "block"]) def test_remove_missing_bitmap(fake_scheduler, env_type): @@ -165,6 +173,9 @@ def test_remove_missing_bitmap(fake_scheduler, env_type): assert not bitmaps assert top_vol.getMetaParam(sc.GENERATION) == generation + 1 + qemuInfo = top_vol.getQemuImageInfo() + assert 'bitmaps' not in qemuInfo + @pytest.mark.parametrize("env_type", ["file", "block"]) def test_remove_inactive_bitmap(fake_scheduler, env_type): @@ -198,6 +209,9 @@ def test_remove_inactive_bitmap(fake_scheduler, env_type): assert not bitmaps assert base_vol.getMetaParam(sc.GENERATION) == generation + 1 + qemuInfo = base_vol.getQemuImageInfo() + assert 'bitmaps' not in qemuInfo + @pytest.mark.parametrize("env_type", ["file", "block"]) def test_remove_invalid_bitmap(fake_scheduler, env_type): @@ -233,3 +247,6 @@ def test_remove_invalid_bitmap(fake_scheduler, env_type): bitmaps = vol_info["format-specific"]["data"].get("bitmaps", []) assert not bitmaps assert base_vol.getMetaParam(sc.GENERATION) == generation + 1 + + qemuInfo = base_vol.getQemuImageInfo() + assert 'bitmaps' not in qemuInfo