From 37a13539249281670f6dfc012378ca69d6a93fef Mon Sep 17 00:00:00 2001 From: Alessio Siniscalchi Date: Fri, 17 May 2024 12:44:38 +0200 Subject: [PATCH 1/4] added api constraint flag --- .../654a874249a8_added_api_constraint_flag.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 alembic/versions/654a874249a8_added_api_constraint_flag.py diff --git a/alembic/versions/654a874249a8_added_api_constraint_flag.py b/alembic/versions/654a874249a8_added_api_constraint_flag.py new file mode 100644 index 0000000..d2d93de --- /dev/null +++ b/alembic/versions/654a874249a8_added_api_constraint_flag.py @@ -0,0 +1,25 @@ +"""added api constraint flag. + +Revision ID: 654a874249a8 +Revises: 7366df17e57c +Create Date: 2024-05-17 12:16:18.206589 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = '654a874249a8' +down_revision = '7366df17e57c' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "resources", sa.Column("api_enforce_constraints", sa.Boolean) + ) + + +def downgrade() -> None: + op.drop_column("resources", "api_enforce_constraints") From 0a5db11c88026919c0814f6f8849f5e4b6477e70 Mon Sep 17 00:00:00 2001 From: Alessio Siniscalchi Date: Fri, 17 May 2024 12:54:20 +0200 Subject: [PATCH 2/4] added, used and tested new boolean flag in catalogue manager --- cads_catalogue/database.py | 1 + cads_catalogue/manager.py | 1 + .../cads-forms-json/reanalysis-era5-land/metadata.json | 1 + .../reanalysis-era5-single-levels/metadata.json | 1 + tests/data/dumped_resources1.txt | 2 ++ tests/data/dumped_resources2.txt | 2 ++ tests/data/dumped_resources3.txt | 1 + tests/data/dumped_resources4.txt | 7 +++++++ tests/data/dumped_resources5.txt | 8 ++++++++ tests/data/dumped_resources6.txt | 8 ++++++++ tests/data/dumped_resources7.txt | 8 ++++++++ tests/test_01_utils.py | 4 ++-- tests/test_40_manager.py | 1 + 13 files changed, 43 insertions(+), 2 deletions(-) diff --git a/cads_catalogue/database.py b/cads_catalogue/database.py index 0c60baa..079a174 100644 --- a/cads_catalogue/database.py +++ b/cads_catalogue/database.py @@ -182,6 +182,7 @@ class Resource(BaseModel): # internal functionality related adaptor = sa.Column(sa.String) adaptor_properties_hash = sa.Column(sa.String) + api_enforce_constraints = sa.Column(sa.Boolean, default=False) disabled_reason = sa.Column(sa.String) sources_hash = sa.Column(sa.String) related_resources_keywords: List[str] = sa.Column( diff --git a/cads_catalogue/manager.py b/cads_catalogue/manager.py index 64a9f22..0ae23e9 100644 --- a/cads_catalogue/manager.py +++ b/cads_catalogue/manager.py @@ -277,6 +277,7 @@ def load_resource_metadata_file(folder_path: str | pathlib.Path) -> dict[str, An data = json.load(fp) metadata["abstract"] = utils.normalize_abstract(data["abstract"]) # required + metadata["api_enforce_constraints"] = data.get("api_enforce_constraints", False) metadata["begin_date"] = data.get("begin_date") metadata["citation"] = data.get("citation") metadata["contactemail"] = data.get("contactemail") diff --git a/tests/data/cads-forms-json/reanalysis-era5-land/metadata.json b/tests/data/cads-forms-json/reanalysis-era5-land/metadata.json index 9dc4a53..26e947e 100644 --- a/tests/data/cads-forms-json/reanalysis-era5-land/metadata.json +++ b/tests/data/cads-forms-json/reanalysis-era5-land/metadata.json @@ -48,6 +48,7 @@ "Provider: Copernicus C3S" ], "qos_tags": ["tag1", "tag2", "tag3"], + "api_enforce_constraints": true, "title": "ERA5-Land hourly data from 1950 to present", "abstract": "ERA5-Land is a reanalysis dataset providing a consistent view of the evolution of land variables over several decades at an enhanced resolution compared to ERA5. ERA5-Land has been produced by replaying the land component of the ECMWF ERA5 climate reanalysis. Reanalysis combines model data with observations from across the world into a globally complete and consistent dataset using the laws of physics. Reanalysis produces data that goes several decades back in time, providing an accurate description of the climate of the past." } \ No newline at end of file diff --git a/tests/data/cads-forms-json/reanalysis-era5-single-levels/metadata.json b/tests/data/cads-forms-json/reanalysis-era5-single-levels/metadata.json index de51aea..2e50ac0 100644 --- a/tests/data/cads-forms-json/reanalysis-era5-single-levels/metadata.json +++ b/tests/data/cads-forms-json/reanalysis-era5-single-levels/metadata.json @@ -23,6 +23,7 @@ "bboxW": 0, "bboxS": -89, "bboxE": 360, + "api_enforce_constraints": false, "begin_date": "1959-01-01", "end_date": "2023-02-11", "file_format": "grib", diff --git a/tests/data/dumped_resources1.txt b/tests/data/dumped_resources1.txt index 2324468..26301d3 100644 --- a/tests/data/dumped_resources1.txt +++ b/tests/data/dumped_resources1.txt @@ -23,6 +23,7 @@ "record_update": "2024-04-12 10:15:31.425675+02:00", "resource_update": "2023-02-17", "abstract": "ERA5-Land is a reanalysis dataset providing a consistent view of the evolution of land variables over several decades at an enhanced resolution compared to ERA5. ERA5-Land has been produced by replaying the land component of the ECMWF ERA5 climate reanalysis. Reanalysis combines model data with observations from across the world into a globally complete and consistent dataset using the laws of physics. Reanalysis produces data that goes several decades back in time, providing an accurate description of the climate of the past.", + "api_enforce_constraints": true, "citation": null, "contactemail": "https://support.ecmwf.int", "description": [], @@ -80,6 +81,7 @@ "record_update": "2024-04-12 10:15:31.463810+02:00", "resource_update": "2023-02-17", "abstract": "ERA5-Land is a reanalysis dataset providing a consistent view of the evolution of land variables over several decades at an enhanced resolution compared to ERA5. ERA5-Land has been produced by replaying the land component of the ECMWF ERA5 climate reanalysis. Reanalysis combines model data with observations from across the world into a globally complete and consistent dataset using the laws of physics. Reanalysis produces data that goes several decades back in time, providing an accurate description of the climate of the past.", + "api_enforce_constraints": false, "citation": null, "contactemail": "https://support.ecmwf.int", "description": [], diff --git a/tests/data/dumped_resources2.txt b/tests/data/dumped_resources2.txt index 07d15d1..ec18c8e 100644 --- a/tests/data/dumped_resources2.txt +++ b/tests/data/dumped_resources2.txt @@ -3,6 +3,7 @@ "resource_id": 1, "resource_uid": "reanalysis-era5-land", "constraints": "an url", + "api_enforce_constraints": true, "form": "an url for form.json", "layout": "an url for layout.json", "previewimage": "an url", @@ -65,6 +66,7 @@ "previewimage": "an url", "adaptor": null, "adaptor_properties_hash": null, + "api_enforce_constraints": false, "disabled_reason": null, "sources_hash": null, "related_resources_keywords": [], diff --git a/tests/data/dumped_resources3.txt b/tests/data/dumped_resources3.txt index 42a27b3..305b8f0 100644 --- a/tests/data/dumped_resources3.txt +++ b/tests/data/dumped_resources3.txt @@ -2,6 +2,7 @@ { "resource_id": 1, "resource_uid": "reanalysis-era5-land", + "api_enforce_constraints": true, "constraints": "an url", "form": "an url", "layout": "an url", diff --git a/tests/data/dumped_resources4.txt b/tests/data/dumped_resources4.txt index e3c4735..469020d 100644 --- a/tests/data/dumped_resources4.txt +++ b/tests/data/dumped_resources4.txt @@ -2,6 +2,7 @@ { "resource_id": 4, "resource_uid": "cams-global-reanalysis-eac4", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", @@ -898,6 +899,7 @@ { "resource_id": 3, "resource_uid": "cams-global-reanalysis-eac4-monthly", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", @@ -1424,6 +1426,7 @@ { "resource_id": 5, "resource_uid": "derived-near-surface-meteorological-variables", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", @@ -1477,6 +1480,7 @@ { "resource_id": 1, "resource_uid": "reanalysis-era5-land", + "api_enforce_constraints": true, "constraints": "an url", "form": "an url", "layout": "an url", @@ -1534,6 +1538,7 @@ { "resource_id": 6, "resource_uid": "reanalysis-era5-land-monthly-means", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", @@ -1587,6 +1592,7 @@ { "resource_id": 7, "resource_uid": "reanalysis-era5-pressure-levels", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", @@ -1640,6 +1646,7 @@ { "resource_id": 8, "resource_uid": "satellite-surface-radiation-budget", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", diff --git a/tests/data/dumped_resources5.txt b/tests/data/dumped_resources5.txt index 30073a3..822eb4c 100644 --- a/tests/data/dumped_resources5.txt +++ b/tests/data/dumped_resources5.txt @@ -2,6 +2,7 @@ { "resource_id": 4, "resource_uid": "cams-global-reanalysis-eac4", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", @@ -898,6 +899,7 @@ { "resource_id": 3, "resource_uid": "cams-global-reanalysis-eac4-monthly", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", @@ -1424,6 +1426,7 @@ { "resource_id": 5, "resource_uid": "derived-near-surface-meteorological-variables", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", @@ -1477,6 +1480,7 @@ { "resource_id": 1, "resource_uid": "reanalysis-era5-land", + "api_enforce_constraints": true, "constraints": "an url", "form": "an url", "layout": "an url", @@ -1534,6 +1538,7 @@ { "resource_id": 6, "resource_uid": "reanalysis-era5-land-monthly-means", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", @@ -1587,6 +1592,7 @@ { "resource_id": 7, "resource_uid": "reanalysis-era5-pressure-levels", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", @@ -1640,6 +1646,7 @@ { "resource_id": 9, "resource_uid": "reanalysis-era5-single-levels", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", @@ -1693,6 +1700,7 @@ { "resource_id": 8, "resource_uid": "satellite-surface-radiation-budget", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", diff --git a/tests/data/dumped_resources6.txt b/tests/data/dumped_resources6.txt index 30073a3..822eb4c 100644 --- a/tests/data/dumped_resources6.txt +++ b/tests/data/dumped_resources6.txt @@ -2,6 +2,7 @@ { "resource_id": 4, "resource_uid": "cams-global-reanalysis-eac4", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", @@ -898,6 +899,7 @@ { "resource_id": 3, "resource_uid": "cams-global-reanalysis-eac4-monthly", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", @@ -1424,6 +1426,7 @@ { "resource_id": 5, "resource_uid": "derived-near-surface-meteorological-variables", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", @@ -1477,6 +1480,7 @@ { "resource_id": 1, "resource_uid": "reanalysis-era5-land", + "api_enforce_constraints": true, "constraints": "an url", "form": "an url", "layout": "an url", @@ -1534,6 +1538,7 @@ { "resource_id": 6, "resource_uid": "reanalysis-era5-land-monthly-means", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", @@ -1587,6 +1592,7 @@ { "resource_id": 7, "resource_uid": "reanalysis-era5-pressure-levels", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", @@ -1640,6 +1646,7 @@ { "resource_id": 9, "resource_uid": "reanalysis-era5-single-levels", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", @@ -1693,6 +1700,7 @@ { "resource_id": 8, "resource_uid": "satellite-surface-radiation-budget", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", diff --git a/tests/data/dumped_resources7.txt b/tests/data/dumped_resources7.txt index 62c25af..f575525 100644 --- a/tests/data/dumped_resources7.txt +++ b/tests/data/dumped_resources7.txt @@ -2,6 +2,7 @@ { "resource_id": 4, "resource_uid": "cams-global-reanalysis-eac4", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", @@ -898,6 +899,7 @@ { "resource_id": 3, "resource_uid": "cams-global-reanalysis-eac4-monthly", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", @@ -1424,6 +1426,7 @@ { "resource_id": 5, "resource_uid": "derived-near-surface-meteorological-variables", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", @@ -1477,6 +1480,7 @@ { "resource_id": 1, "resource_uid": "reanalysis-era5-land", + "api_enforce_constraints": true, "constraints": "an url", "form": "an url", "layout": "an url", @@ -1534,6 +1538,7 @@ { "resource_id": 6, "resource_uid": "reanalysis-era5-land-monthly-means", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", @@ -1587,6 +1592,7 @@ { "resource_id": 7, "resource_uid": "reanalysis-era5-pressure-levels", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", @@ -1640,6 +1646,7 @@ { "resource_id": 9, "resource_uid": "reanalysis-era5-single-levels", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", @@ -1693,6 +1700,7 @@ { "resource_id": 8, "resource_uid": "satellite-surface-radiation-budget", + "api_enforce_constraints": false, "constraints": "an url", "form": "an url", "layout": "an url", diff --git a/tests/test_01_utils.py b/tests/test_01_utils.py index faabe20..0c9bd0c 100644 --- a/tests/test_01_utils.py +++ b/tests/test_01_utils.py @@ -43,11 +43,11 @@ def test_folders2hash() -> None: ) assert ( utils.folders2hash([test_file_path_1]).hexdigest() - == "23d3f7b29edc9b0e75fd270f08364cd4" + == "e8fa178d247cf902502dfeca284bdd49" ) assert ( utils.folders2hash([test_file_path_1, test_file_path_2]).hexdigest() - == "523ebd3dee5705bf0d1c402dc129cc51" + == "afd7020282c5977b199dcc569fbb9370" ) diff --git a/tests/test_40_manager.py b/tests/test_40_manager.py index 857c49c..e3e2cf6 100644 --- a/tests/test_40_manager.py +++ b/tests/test_40_manager.py @@ -1557,6 +1557,7 @@ def test_load_resource_from_folder() -> None: " the world into a globally complete and consistent dataset using the laws of " "physics. Reanalysis produces data that goes several decades back in time, " "providing an accurate description of the climate of the past.", + "api_enforce_constraints": True, "begin_date": "1950-01-01", "citation": None, "contactemail": "https://support.ecmwf.int", From 0cdae716650396a523159f2f5b42c90aaabda2ea Mon Sep 17 00:00:00 2001 From: Alessio Siniscalchi Date: Fri, 17 May 2024 13:32:21 +0200 Subject: [PATCH 3/4] style --- .../versions/654a874249a8_added_api_constraint_flag.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/alembic/versions/654a874249a8_added_api_constraint_flag.py b/alembic/versions/654a874249a8_added_api_constraint_flag.py index d2d93de..a8766f4 100644 --- a/alembic/versions/654a874249a8_added_api_constraint_flag.py +++ b/alembic/versions/654a874249a8_added_api_constraint_flag.py @@ -5,20 +5,20 @@ Create Date: 2024-05-17 12:16:18.206589 """ + import sqlalchemy as sa + from alembic import op # revision identifiers, used by Alembic. -revision = '654a874249a8' -down_revision = '7366df17e57c' +revision = "654a874249a8" +down_revision = "7366df17e57c" branch_labels = None depends_on = None def upgrade() -> None: - op.add_column( - "resources", sa.Column("api_enforce_constraints", sa.Boolean) - ) + op.add_column("resources", sa.Column("api_enforce_constraints", sa.Boolean)) def downgrade() -> None: From 274a36fefb726f0ada26391d6ddbc3222ce3e868 Mon Sep 17 00:00:00 2001 From: Alessio Siniscalchi Date: Fri, 17 May 2024 13:32:47 +0200 Subject: [PATCH 4/4] update validation to include recently added fields --- cads_catalogue/validations.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cads_catalogue/validations.py b/cads_catalogue/validations.py index 8ea05f9..c1ae58e 100644 --- a/cads_catalogue/validations.py +++ b/cads_catalogue/validations.py @@ -218,6 +218,7 @@ def validate_metadata_json(dataset_folder): if required_field not in data or not data.get(required_field): logger.error(f"required field not found or empty: '{required_field}'") optional_fields = [ + "api_enforce_constraints", "bboxE", "bboxN", "bboxS", @@ -234,6 +235,8 @@ def validate_metadata_json(dataset_folder): "ds_responsible_organisation_role", "end_date", "file_format", + "format_version", + "high_priority_terms", "hidden", "inspire_theme", "keywords", @@ -241,6 +244,7 @@ def validate_metadata_json(dataset_folder): "lineage", "portal", "publication_date", + "qos_tags", "related_resources_keywords", "representative_fraction", "responsible_individual",