Merge pull request #188 from dataforgoodfr/load_zones

Load zones

backend/bloom/tasks/create_update_excursions_segments.py

@@ -44,7 +44,7 @@ def add_excursion(session: Session, vessel_id: int, departure_at: datetime,
 
     if result:
         arrival_port_id = result["arrival_port_id"]
-        arrival_position = to_shape(result["arrival_position"])
+        arrival_position = to_shape(result["arrival_position"]) if result["arrival_position"] else None
     else:
         arrival_port_id = None
         arrival_position = None
@@ -226,7 +226,7 @@ def get_port(x, session):
                 for a in df.index:
                     if (df["port"].iloc[a] >= 0):
                         if (open_ongoing_excursion):
-                            close_excursion(session, ongoing_excursion_id, df["port"].iloc[a],
+                            close_excursion(session, ongoing_excursion_id, int(df["port"].iloc[a]),
                                             df["end_latitude"].iloc[a],
                                             df["end_longitude"].iloc[a],
                                             df["timestamp_end"].iloc[a])  # put the close excursion function here
@@ -306,9 +306,9 @@ def get_time_of_departure():
                 new_rels.append(RelSegmentZone(segment_id=segment.id, zone_id=zone.id))
                 if zone.category == "amp":
                     segment.in_amp_zone = True
-                elif zone.category == "coastal":
+                elif zone.category.startswith("Fishing coastal waters"):
                     segment.in_costal_waters = True
-                elif zone.category == "territorial":
+                elif zone.category == "Territorial seas":
                     segment.in_territorial_waters = True
             # Mise à jour de l'excursion avec le temps passé dans chaque type de zone
             excursion = excursions.get(segment.excursion_id,

backend/bloom/tasks/load_dim_zone_amp_from_csv.py

@@ -2,66 +2,62 @@
 from time import perf_counter
 
 import pandas as pd
+from shapely import wkb
+
 from bloom.config import settings
 from bloom.container import UseCases
 from bloom.domain.zone import Zone
-from bloom.infra.database.errors import DBException
 from bloom.logger import logger
-from pydantic import ValidationError
-from shapely import wkb
+
+FIC_ZONE = ["french_metropolitan_mpas.csv", "fishing_coastal_waters.csv", "territorial_seas.csv"]
 
 
 def map_to_domain(row: pd.Series) -> Zone:
     isna = row.isna()
 
+    json_data = {}
+    for k in ["index", "wdpaid", "desig_eng", "desig_type", "iucn_cat", "parent_iso", "iso3", "benificiaries",
+              "source", "reference"]:
+        try:
+            value = row[k] if not isna[k] else None
+            json_data[k] = value
+        except:
+            pass
+
     return Zone(
-        category="amp",
-        sub_category=None,
+        category=row["category"],
+        sub_category=row["sub_category"] if not isna["sub_category"] else None,
         name=row["name"],
         geometry=row["geometry"],
         centroid=row["geometry"].centroid,
-        json_data={k: row[k] if not isna[k] else None for k in
-                   ["index", "desig_eng", "desig_type", "iucn_cat", "parent_iso", "iso3", "benificiaries"]},
+        json_data=json_data,
     )
 
 
-def run(csv_file_name: str):
+def run():
     use_cases = UseCases()
     db = use_cases.db()
     zone_repository = use_cases.zone_repository()
 
-    total = 0
-    try:
-        df = pd.read_csv(csv_file_name, sep=",")
-        df = df.rename(columns={"Geometry": "geometry",
-                                "Index": "index", "WDPAID": "wdpaid",
-                                "Name": "name",
-                                "DESIG_ENG": "desig_eng",
-                                "DESIG_TYPE": "desig_type",
-                                "IUCN_CAT": "iucn_cat",
-                                "PARENT_ISO": "parent_iso",
-                                "ISO3": "iso3",
-                                "Benificiaries": "benificiaries"})
-        df["geometry"] = df["geometry"].apply(wkb.loads)
-        zones = df.apply(map_to_domain, axis=1)
-        with db.session() as session:
+    with db.session() as session:
+        for fic_csv in FIC_ZONE:
+            file_name = Path(settings.data_folder).joinpath(fic_csv)
+            logger.info(f"Chargement des données du fichier {file_name}")
+
+            total = 0
+            df = pd.read_csv(file_name, sep=",")
+            df["geometry"] = df["geometry"].apply(wkb.loads)
+            zones = df.apply(map_to_domain, axis=1)
             zones = zone_repository.batch_create_zone(session, list(zones))
-            session.commit()
             total = len(zones)
-            print(zones)
-    except ValidationError as e:
-        logger.error("Erreur de validation des données de bateau")
-        logger.error(e.errors())
-    except DBException:
-        logger.error("Erreur d'insertion en base")
-    logger.info(f"{total} zone(s) créés")
+            logger.info(f"{total} zone(s) créés")
+        session.commit()
 
 
 if __name__ == "__main__":
     time_start = perf_counter()
-    file_name = Path(settings.data_folder).joinpath("./zones_subset.csv")
-    logger.info(f"DEBUT - Chargement des données des zones AMP depuis le fichier {file_name}")
-    run(file_name)
+    logger.info("DEBUT - Chargement des données des zones")
+    run()
     time_end = perf_counter()
     duration = time_end - time_start
-    logger.info(f"FIN - Chargement des données des zones AMP en {duration:.2f}s")
+    logger.info(f"FIN - Chargement des données des zones en {duration:.2f}s")