added api constraint flag

alembic/versions/654a874249a8_added_api_constraint_flag.py

@@ -0,0 +1,25 @@
+"""added api constraint flag.
+
+Revision ID: 654a874249a8
+Revises: 7366df17e57c
+Create Date: 2024-05-17 12:16:18.206589
+
+"""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "654a874249a8"
+down_revision = "7366df17e57c"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column("resources", sa.Column("api_enforce_constraints", sa.Boolean))
+
+
+def downgrade() -> None:
+    op.drop_column("resources", "api_enforce_constraints")

cads_catalogue/database.py

@@ -182,6 +182,7 @@ class Resource(BaseModel):
     # internal functionality related
     adaptor = sa.Column(sa.String)
     adaptor_properties_hash = sa.Column(sa.String)
+    api_enforce_constraints = sa.Column(sa.Boolean, default=False)
     disabled_reason = sa.Column(sa.String)
     sources_hash = sa.Column(sa.String)
     related_resources_keywords: List[str] = sa.Column(

cads_catalogue/manager.py

@@ -277,6 +277,7 @@ def load_resource_metadata_file(folder_path: str | pathlib.Path) -> dict[str, An
         data = json.load(fp)
 
     metadata["abstract"] = utils.normalize_abstract(data["abstract"])  # required
+    metadata["api_enforce_constraints"] = data.get("api_enforce_constraints", False)
     metadata["begin_date"] = data.get("begin_date")
     metadata["citation"] = data.get("citation")
     metadata["contactemail"] = data.get("contactemail")

cads_catalogue/validations.py

@@ -218,6 +218,7 @@ def validate_metadata_json(dataset_folder):
         if required_field not in data or not data.get(required_field):
             logger.error(f"required field not found or empty: '{required_field}'")
     optional_fields = [
+        "api_enforce_constraints",
         "bboxE",
         "bboxN",
         "bboxS",
@@ -234,13 +235,16 @@ def validate_metadata_json(dataset_folder):
         "ds_responsible_organisation_role",
         "end_date",
         "file_format",
+        "format_version",
+        "high_priority_terms",
         "hidden",
         "inspire_theme",
         "keywords",
         "licences",
         "lineage",
         "portal",
         "publication_date",
+        "qos_tags",
         "related_resources_keywords",
         "representative_fraction",
         "responsible_individual",

tests/data/cads-forms-json/reanalysis-era5-land/metadata.json

@@ -48,6 +48,7 @@
     "Provider: Copernicus C3S"
   ],
   "qos_tags": ["tag1", "tag2", "tag3"],
+  "api_enforce_constraints": true,
   "title": "ERA5-Land hourly data from 1950 to present",
   "abstract": "ERA5-Land is a reanalysis dataset providing a consistent view of the evolution of land variables over several decades at an enhanced resolution compared to ERA5. ERA5-Land has been produced by replaying the land component of the ECMWF ERA5 climate reanalysis. Reanalysis combines model data with observations from across the world into a globally complete and consistent dataset using the laws of physics. Reanalysis produces data that goes several decades back in time, providing an accurate description of the climate of the past."
 }

tests/data/cads-forms-json/reanalysis-era5-single-levels/metadata.json

@@ -23,6 +23,7 @@
   "bboxW": 0,
   "bboxS": -89,
   "bboxE": 360,
+  "api_enforce_constraints": false,
   "begin_date": "1959-01-01",
   "end_date": "2023-02-11",
   "file_format": "grib",

tests/data/dumped_resources1.txt

@@ -23,6 +23,7 @@
         "record_update": "2024-04-12 10:15:31.425675+02:00",
         "resource_update": "2023-02-17",
         "abstract": "ERA5-Land is a reanalysis dataset providing a consistent view of the evolution of land variables over several decades at an enhanced resolution compared to ERA5. ERA5-Land has been produced by replaying the land component of the ECMWF ERA5 climate reanalysis. Reanalysis combines model data with observations from across the world into a globally complete and consistent dataset using the laws of physics. Reanalysis produces data that goes several decades back in time, providing an accurate description of the climate of the past.",
+        "api_enforce_constraints": true,
         "citation": null,
         "contactemail": "https://support.ecmwf.int",
         "description": [],
@@ -80,6 +81,7 @@
         "record_update": "2024-04-12 10:15:31.463810+02:00",
         "resource_update": "2023-02-17",
         "abstract": "ERA5-Land is a reanalysis dataset providing a consistent view of the evolution of land variables over several decades at an enhanced resolution compared to ERA5. ERA5-Land has been produced by replaying the land component of the ECMWF ERA5 climate reanalysis. Reanalysis combines model data with observations from across the world into a globally complete and consistent dataset using the laws of physics. Reanalysis produces data that goes several decades back in time, providing an accurate description of the climate of the past.",
+        "api_enforce_constraints": false,
         "citation": null,
         "contactemail": "https://support.ecmwf.int",
         "description": [],

tests/data/dumped_resources2.txt

@@ -3,6 +3,7 @@
         "resource_id": 1,
         "resource_uid": "reanalysis-era5-land",
         "constraints": "an url",
+        "api_enforce_constraints": true,
         "form": "an url for form.json",
         "layout": "an url for layout.json",
         "previewimage": "an url",
@@ -65,6 +66,7 @@
         "previewimage": "an url",
         "adaptor": null,
         "adaptor_properties_hash": null,
+        "api_enforce_constraints": false,
         "disabled_reason": null,
         "sources_hash": null,
         "related_resources_keywords": [],

tests/data/dumped_resources3.txt

@@ -2,6 +2,7 @@
     {
         "resource_id": 1,
         "resource_uid": "reanalysis-era5-land",
+        "api_enforce_constraints": true,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",

tests/data/dumped_resources4.txt

@@ -2,6 +2,7 @@
     {
         "resource_id": 4,
         "resource_uid": "cams-global-reanalysis-eac4",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -898,6 +899,7 @@
     {
         "resource_id": 3,
         "resource_uid": "cams-global-reanalysis-eac4-monthly",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -1424,6 +1426,7 @@
     {
         "resource_id": 5,
         "resource_uid": "derived-near-surface-meteorological-variables",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -1477,6 +1480,7 @@
     {
         "resource_id": 1,
         "resource_uid": "reanalysis-era5-land",
+        "api_enforce_constraints": true,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -1534,6 +1538,7 @@
     {
         "resource_id": 6,
         "resource_uid": "reanalysis-era5-land-monthly-means",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -1587,6 +1592,7 @@
     {
         "resource_id": 7,
         "resource_uid": "reanalysis-era5-pressure-levels",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -1640,6 +1646,7 @@
     {
         "resource_id": 8,
         "resource_uid": "satellite-surface-radiation-budget",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",

tests/data/dumped_resources5.txt

@@ -2,6 +2,7 @@
     {
         "resource_id": 4,
         "resource_uid": "cams-global-reanalysis-eac4",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -898,6 +899,7 @@
     {
         "resource_id": 3,
         "resource_uid": "cams-global-reanalysis-eac4-monthly",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -1424,6 +1426,7 @@
     {
         "resource_id": 5,
         "resource_uid": "derived-near-surface-meteorological-variables",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -1477,6 +1480,7 @@
     {
         "resource_id": 1,
         "resource_uid": "reanalysis-era5-land",
+        "api_enforce_constraints": true,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -1534,6 +1538,7 @@
     {
         "resource_id": 6,
         "resource_uid": "reanalysis-era5-land-monthly-means",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -1587,6 +1592,7 @@
     {
         "resource_id": 7,
         "resource_uid": "reanalysis-era5-pressure-levels",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -1640,6 +1646,7 @@
     {
         "resource_id": 9,
         "resource_uid": "reanalysis-era5-single-levels",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -1693,6 +1700,7 @@
     {
         "resource_id": 8,
         "resource_uid": "satellite-surface-radiation-budget",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",

tests/data/dumped_resources6.txt

@@ -2,6 +2,7 @@
     {
         "resource_id": 4,
         "resource_uid": "cams-global-reanalysis-eac4",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -898,6 +899,7 @@
     {
         "resource_id": 3,
         "resource_uid": "cams-global-reanalysis-eac4-monthly",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -1424,6 +1426,7 @@
     {
         "resource_id": 5,
         "resource_uid": "derived-near-surface-meteorological-variables",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -1477,6 +1480,7 @@
     {
         "resource_id": 1,
         "resource_uid": "reanalysis-era5-land",
+        "api_enforce_constraints": true,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -1534,6 +1538,7 @@
     {
         "resource_id": 6,
         "resource_uid": "reanalysis-era5-land-monthly-means",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -1587,6 +1592,7 @@
     {
         "resource_id": 7,
         "resource_uid": "reanalysis-era5-pressure-levels",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -1640,6 +1646,7 @@
     {
         "resource_id": 9,
         "resource_uid": "reanalysis-era5-single-levels",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -1693,6 +1700,7 @@
     {
         "resource_id": 8,
         "resource_uid": "satellite-surface-radiation-budget",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",

tests/data/dumped_resources7.txt

@@ -2,6 +2,7 @@
     {
         "resource_id": 4,
         "resource_uid": "cams-global-reanalysis-eac4",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -898,6 +899,7 @@
     {
         "resource_id": 3,
         "resource_uid": "cams-global-reanalysis-eac4-monthly",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -1424,6 +1426,7 @@
     {
         "resource_id": 5,
         "resource_uid": "derived-near-surface-meteorological-variables",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -1477,6 +1480,7 @@
     {
         "resource_id": 1,
         "resource_uid": "reanalysis-era5-land",
+        "api_enforce_constraints": true,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -1534,6 +1538,7 @@
     {
         "resource_id": 6,
         "resource_uid": "reanalysis-era5-land-monthly-means",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -1587,6 +1592,7 @@
     {
         "resource_id": 7,
         "resource_uid": "reanalysis-era5-pressure-levels",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -1640,6 +1646,7 @@
     {
         "resource_id": 9,
         "resource_uid": "reanalysis-era5-single-levels",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",
@@ -1693,6 +1700,7 @@
     {
         "resource_id": 8,
         "resource_uid": "satellite-surface-radiation-budget",
+        "api_enforce_constraints": false,
         "constraints": "an url",
         "form": "an url",
         "layout": "an url",

tests/test_01_utils.py

@@ -43,11 +43,11 @@ def test_folders2hash() -> None:
     )
     assert (
         utils.folders2hash([test_file_path_1]).hexdigest()
-        == "23d3f7b29edc9b0e75fd270f08364cd4"
+        == "e8fa178d247cf902502dfeca284bdd49"
     )
     assert (
         utils.folders2hash([test_file_path_1, test_file_path_2]).hexdigest()
-        == "523ebd3dee5705bf0d1c402dc129cc51"
+        == "afd7020282c5977b199dcc569fbb9370"
     )
 
 

tests/test_40_manager.py

@@ -1557,6 +1557,7 @@ def test_load_resource_from_folder() -> None:
         " the world into a globally complete and consistent dataset using the laws of "
         "physics. Reanalysis produces data that goes several decades back in time, "
         "providing an accurate description of the climate of the past.",
+        "api_enforce_constraints": True,
         "begin_date": "1950-01-01",
         "citation": None,
         "contactemail": "https://support.ecmwf.int",