Last PR corrections

bloom/README.md → README.md

@@ -12,6 +12,20 @@
 
 **BLOOM** is a non-profit organization founded in 2005 that works to preserve the marine environment and species from unnecessary destruction and to increase social benefits in the fishing sector. **BLOOM** wages awareness and advocacy campaigns in order to accelerate the adoption of concrete solutions for the ocean, humans and the climate. **BLOOM** carries out scientific research projects, independent studies and evaluations that highlight crucial and unaddressed issues such as the financing mechanisms of the fishing sector. **BLOOM**’s actions are meant for the general public as well as policy-makers and economic stakeholders.
 
+## Installing Trawl Watch with Docker Compose
+
+* Ensure [Docker](https://docs.docker.com/get-docker/) is installed.
+* git clone https://github.com/dataforgoodfr/12_bloom.git
+* cd 12_bloom
+* docker compose build
+* docker compose pull
+* copy and paste bloom/env.template at the same level than docker-compose.yaml and rename it .env
+* docker compose run --service-ports bloom /bin/bash
+* streamlit run Trawlwatcher.py
+* working mmsi : 261084090
+
+
+
 ## Installing Trawl Watch with `poetry`
 
 ### Prerequisites:

bloom/Dockerfile

@@ -1,4 +1,5 @@
-FROM python:3.10-slim-bullseye
+ARG IMAGE_PYTHON=${IMAGE_PYTHON:-python:3.10-slim-bullseye}
+FROM ${IMAGE_PYTHON}
 
 RUN apt-get update
 RUN apt-get -y install wget gcc g++
@@ -7,7 +8,6 @@ RUN apt-get install -y rsyslog
 # Define working directory
 WORKDIR /source_code
 COPY bloom/ ./bloom/
-COPY data/ ./data/
 COPY app.py .
 COPY container.py .
 COPY docker-env/rsyslog.conf /etc/rsyslog.conf

bloom/Trawlwatcher.py

@@ -3,6 +3,8 @@
 import streamlit as st
 from dotenv import load_dotenv
 
+from bloom.config import settings
+
 
 st.set_page_config(
     page_title="Bloom Trawlwatcher Demo app",
@@ -11,17 +13,18 @@
 )
 
 # FILL IN YOUR CREDENTIALS .env file HERE !!
-env_path = Path('.') / '.env.template'
-if not env_path.is_file():
-    raise FileNotFoundError(f"Couldn't find .env file at {env_path.absolute()}")
-load_dotenv(env_path)
+
+#env_path = Path('.') / '.env.template'
+#if not env_path.is_file():
+#    raise FileNotFoundError(f"Couldn't find .env file at {env_path.absolute()}")
+#load_dotenv(env_path)
 
 def local_css(file_name: str) -> None:
     with Path.open(file_name) as f:
         st.markdown(f"<style>{f.read()}</style>", unsafe_allow_html=True)
         return
 
-local_css(Path("styles.css"))
+local_css(Path(__file__).parent.joinpath("./styles.css"))
 
 st.write("![](https://upload.wikimedia.org/wikipedia/fr/e/e8/Logo_BLOOM.jpg)")
 st.write("## Welcome to Bloom Trawlwatcher Demo app! 🐟")

bloom/alembic/env.py

@@ -5,6 +5,8 @@
 
 from alembic import context
 
+from bloom.config import settings
+
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
@@ -18,30 +20,7 @@
 # for 'autogenerate' support
 target_metadata = None
 
-# other values from the config, defined by the needs of env.py,
-# can be acquired:
-# ... etc.
-
-postgres_user = os.environ.get("POSTGRES_USER")
-postgres_password = os.environ.get("POSTGRES_PASSWORD")
-postgres_hostname = os.environ.get("POSTGRES_HOSTNAME")
-postgres_db = os.environ.get("POSTGRES_DB")
-postgres_port = os.environ.get("POSTGRES_PORT")
-
-db_url = (
-    "postgresql://"
-    + postgres_user
-    + ":"
-    + postgres_password
-    + "@"
-    + postgres_hostname
-    + ":"
-    + postgres_port
-    + "/"
-    + postgres_db
-)
-
-config.set_main_option("sqlalchemy.url", db_url)
+config.set_main_option("sqlalchemy.url", settings.db_url)
 
 
 def run_migrations_offline() -> None:

bloom/alembic/init_script/load_amp_data.py

@@ -6,32 +6,13 @@
 import pandas as pd
 from shapely import wkb
 from sqlalchemy import create_engine
+from bloom.config import settings
+
 
 logging.basicConfig()
 logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO)
 
-postgres_user = os.environ.get("POSTGRES_USER")
-postgres_password = os.environ.get("POSTGRES_PASSWORD")
-postgres_hostname = os.environ.get("POSTGRES_HOSTNAME")
-postgres_db = os.environ.get("POSTGRES_DB")
-postgres_port = os.environ.get("POSTGRES_PORT")
-
-
-# The db url is configured with the db connexion variables declared in the db.yaml file.
-db_url = (
-    "postgresql://"
-    + postgres_user
-    + ":"
-    + postgres_password
-    + "@"
-    + postgres_hostname
-    + ":"
-    + postgres_port
-    + "/"
-    + postgres_db
-)
-
-engine = create_engine(db_url, echo=False)
+engine = create_engine(settings.db_url, echo=False)
 
 df = pd.read_csv(
     Path(os.path.dirname(__file__)).joinpath("../../data/zones_subset_02022024.csv"),

bloom/alembic/init_script/load_positions_data.py

@@ -4,31 +4,12 @@
 
 import pandas as pd
 from sqlalchemy import create_engine
+from bloom.config import settings
 
 logging.basicConfig()
 logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO)
 
-postgres_user = os.environ.get("POSTGRES_USER")
-postgres_password = os.environ.get("POSTGRES_PASSWORD")
-postgres_hostname = os.environ.get("POSTGRES_HOSTNAME")
-postgres_db = os.environ.get("POSTGRES_DB")
-postgres_port = os.environ.get("POSTGRES_PORT")
-
-# The db url is configured with the db connexion variables declared in the db.yaml file.
-db_url = (
-    "postgresql://"
-    + postgres_user
-    + ":"
-    + postgres_password
-    + "@"
-    + postgres_hostname
-    + ":"
-    + postgres_port
-    + "/"
-    + postgres_db
-)
-
-engine = create_engine(db_url)
+engine = create_engine(settings.db_url)
 
 df = pd.read_csv(
     Path(os.path.dirname(__file__)).joinpath("../../data/spire_positions_subset_02022024.csv"),

bloom/alembic/init_script/load_vessels_data.py

@@ -4,30 +4,12 @@
 
 import pandas as pd
 from sqlalchemy import create_engine
+from bloom.config import settings
 
 logging.basicConfig()
 logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO)
 
-postgres_user = os.environ.get("POSTGRES_USER")
-postgres_password = os.environ.get("POSTGRES_PASSWORD")
-postgres_hostname = os.environ.get("POSTGRES_HOSTNAME")
-postgres_db = os.environ.get("POSTGRES_DB")
-postgres_port = os.environ.get("POSTGRES_PORT")
-
-# The db url is configured with the db connexion variables declared in the db.yaml file.
-db_url = (
-    "postgresql://"
-    + postgres_user
-    + ":"
-    + postgres_password
-    + "@"
-    + postgres_hostname
-    + ":"
-    + postgres_port
-    + "/"
-    + postgres_db
-)
-engine = create_engine(db_url)
+engine = create_engine(settings.db_url)
 df = pd.read_csv(
     Path(os.path.dirname(__file__)).joinpath("../../data/chalutiers_pelagiques.csv"),
     sep=";",

bloom/bloom/config.py

@@ -1,29 +1,111 @@
 import os
+from pathlib import Path
 
 from pydantic import BaseSettings
 
-class Settings(BaseSettings):
-    postgres_user = os.environ.get("POSTGRES_USER")
-    postgres_password = os.environ.get("POSTGRES_PASSWORD")
-    postgres_hostname = os.environ.get("POSTGRES_HOSTNAME")
-    postgres_port = os.environ.get("POSTGRES_PORT")
-    postgres_db = os.environ.get("POSTGRES_DB")
-
-    print("db_url: ", "postgresql://"+postgres_user+":"+postgres_password+"@"+postgres_hostname+":"+postgres_port+"/"+postgres_db)
+def extract_values_from_env(config:dict,allow_extend:bool=False) -> dict:
+    """ function that extrat key=value pairs from a file
+    Parameters:
+    - config: dict to extend/update with new key/value pairs found in environment
+    - allow_extend: allows to extend extracted keys with new keys that are not in
+                    actuel config if True, restrict to already existing keys in config of False
+    Returns a dict contains key/value
+    """
+    for k,v in os.environ.items():
+        # Processing of indirect affectation via [ATTR]_FILE=VALUE_PATH => ATTR=VALUE
+        if k.lower() in [f"{k}_FILE".lower() for k in config.keys()]\
+            and ( k.removesuffix('_FILE').lower() in config.keys() or  allow_extend == True)\
+            and Path(v).is_file():
+                with Path.open(v, mode='r') as file:
+                    config[k.removesuffix('_FILE').lower()]=file.readline().strip()
+        # Processing of direct affectation via ATTR=VALUE
+        # if extracted key already exist in config OR if allowed to add new keys to config
+        # Then adding/updating key/value
+        if k.lower() in [k.lower() for k in config.keys()] or allow_extend == True:
+            config[k.lower()]=v
+    return config
 
-    db_url = (
-        "postgresql://"
-        + postgres_user
-        + ":"
-        + postgres_password
-        + "@"
-        + postgres_hostname
-        + ":"
-        + postgres_port
-        + "/"
-        + postgres_db
-    )
+def extract_values_from_file(filename:str,config:dict,
+                             allow_extend:bool=False,
+                             env_priority:bool=True
+                             )-> dict:
+    """ function that extrat key=value pairs from a file
+    Parameters:
+    - filename: filename/filepath from which to extract key/value pairs found in .env.* file
+    - config: dict to extend/update with new key/value pairs
+    - allow_extend: allows to extend extracted keys with new keys that are not in actuel
+                    config if True, restrict to already existing keys in config of False
+    Returns a dict contains key/value
+    """ 
+    filepath=Path(Path(__file__).parent).joinpath(filename)
+    with Path.open(filepath) as file:
+        for line in file:
+            # Split line at first occurence of '='.
+            # This allows to have values containing '=' character
+            split=line.strip().split('=',1)
+            # if extraction contains 2 items and strictly 2 items
+            split_succeed=2
+            if(len(split)==split_succeed):
+                k=split[0]
+                v=split[1]
+                # Processing of indirect affectation via [ATTR]_FILE=VALUE_PATH => ATTR=VALUE
+                if k.lower() in [f"{k}_FILE".lower() for k in config.keys()]\
+                    and ( k.removesuffix('_FILE').lower() in config.keys() or  allow_extend == True)\
+                    and Path(v).is_file():
+                        with Path(v).open( mode='r') as file_value:
+                            config[k.removesuffix('_FILE').lower()]=file_value.readline().strip()
+                # if extracted key already exist in config OR if allowed to add new keys to
+                # config then adding/updating key/value
+                if k.lower() in [k.lower() for k in config.keys()] or allow_extend == True:
+                    config[k.lower()]=v
+            # If env priority True, then overriding all values with ENV values before ending
+            if env_priority:
+                extract_values_from_env(config,allow_extend=False)
+    return config
 
+
+class Settings(BaseSettings):
+    # Déclaration des attributs/paramètres disponibles au sein de la class settings
+    postgres_user:str = None
+    postgres_password:str = None
+    postgres_hostname:str = None
+    postgres_port:str = None
+    postgres_db:str = None
     srid: int = 4326
+    db_url:str = None
+    spire_token:str = None
+
+    def __init__(self):
+        super().__init__(self)
+        # Default values
+        srid: int = 4326
+
+        # Si le fichier de configuration à charger est précisé par la variable d'environnement
+        # BLOOM_CONFIG alors on charge ce fichier, sinon par défaut c'est ../.env
+        BLOOM_CONFIG=os.getenv('BLOOM_CONFIG',Path(__file__).parent.joinpath(".env"))
+
+        # Ici on charge les paramètres à partir du fichier BLOOM_CONFIG
+        # et on mets à jour directement les valeurs des paramètres en tant qu'attribut de la
+        # la classe courante Settings en attanquant le self.__dict__
+        # Ces variables sont systmétiquement convertie en lower case
+        # 
+        # allow_extend=False précise que seuls les attributs déjà existants dans la config
+        # passée en paramètres (ici self.__dict__) sont mis à jour. Pas de nouveau paramètres
+        # Cela singifie que pour rendre accessible un nouveau paramètre il faut le déclaré
+        # dans la liste des attributs de la classe Settings
+        #
+        # env_priority=true signifie que si un paramètres est présent dans la classe Settings,
+        # mas aussi dans le fichier BLOOM_CONFIG ainsi qu'en tant que variable d'environnement
+        # alors c'est la valeur de la variable d'environnement qui sera chargée au final
+        # La priorité est donnée aux valeur de l'environnement selon le standard Docker
+        if Path(BLOOM_CONFIG).exists():
+            extract_values_from_file(BLOOM_CONFIG,self.__dict__,allow_extend=False,env_priority=True)
+        else:
+            extract_values_from_env(self.__dict__,allow_extend=False)
+
+        self.db_url = ( f"postgresql://{self.postgres_user}:"
+                  f"{self.postgres_password}@{self.postgres_hostname}:"
+                  f"{self.postgres_port}/{self.postgres_db}")
+
 
 settings = Settings()

bloom/bloom/infra/repositories/repository_vessel.py

@@ -23,7 +23,7 @@ def __init__(
         session_factory: Callable,
     ) -> Callable[..., AbstractContextManager]:
         self.session_factory = session_factory
-        self.vessels_path = Path.joinpath(Path.cwd(), "data/chalutiers_pelagiques.csv")
+        self.vessels_path = Path(settings.data_folder).joinpath("./chalutiers_pelagiques.csv")
 
     def load_vessel_metadata(self) -> list[Vessel]:
         with self.session_factory() as session:

bloom/bloom/usecase/GetVesselsFromSpire.py

@@ -10,6 +10,7 @@
 from bloom.infra.http.spire_api_utils import Paging
 from bloom.infra.repositories.repository_vessel import RepositoryVessel
 from bloom.logger import logger
+from bloom.config import settings
 
 
 class GetVesselsFromSpire:
@@ -19,7 +20,7 @@ def __init__(
     ) -> None:
         self.vessel_repository: RepositoryVessel = vessel_repository
 
-        spire_token = os.environ.get("SPIRE_TOKEN")
+        spire_token = settings.spire_token
 
         self.transport = RequestsHTTPTransport(
             url="https://api.spire.com/graphql",