From 16f02f58d94bbb4dadc51a752617e30829f33e77 Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Mon, 4 Mar 2024 11:47:52 +0100 Subject: [PATCH 01/23] feat: add BLOOM_CONFIG management in bloom.config --- bloom/bloom/config.py | 119 +++++++++++++++++++++++++++++++++++------- 1 file changed, 99 insertions(+), 20 deletions(-) diff --git a/bloom/bloom/config.py b/bloom/bloom/config.py index c0f0e1ea..6d89434c 100644 --- a/bloom/bloom/config.py +++ b/bloom/bloom/config.py @@ -1,29 +1,108 @@ import os +from pathlib import Path from pydantic import BaseSettings -class Settings(BaseSettings): - postgres_user = os.environ.get("POSTGRES_USER") - postgres_password = os.environ.get("POSTGRES_PASSWORD") - postgres_hostname = os.environ.get("POSTGRES_HOSTNAME") - postgres_port = os.environ.get("POSTGRES_PORT") - postgres_db = os.environ.get("POSTGRES_DB") - - print("db_url: ", "postgresql://"+postgres_user+":"+postgres_password+"@"+postgres_hostname+":"+postgres_port+"/"+postgres_db) +def extract_values_from_env(config:dict,allow_extend:bool=False) -> dict: + """ function that extrat key=value pairs from a file + Parameters: + - config: dict to extend/update with new key/value pairs found in environment + - allow_extend: allows to extend extracted keys with new keys that are not in + actuel config if True, restrict to already existing keys in config of False + Returns a dict contains key/value + """ + for k,v in os.environ.items(): + # Processing of indirect affectation via [ATTR]_FILE=VALUE_PATH => ATTR=VALUE + if k.lower() in [f"{k}_FILE".lower() for k in config.keys()]\ + and ( k.removesuffix('_FILE').lower() in config.keys() or allow_extend == True)\ + and Path(v).is_file(): + with Path.open(v, mode='r') as file: + config[k.removesuffix('_FILE').lower()]=file.readline().strip() + # Processing of direct affectation via ATTR=VALUE + # if extracted key already exist in config OR if allowed to add new keys to config + # Then adding/updating key/value + if k.lower() in [k.lower() for k in config.keys()] or allow_extend == True: + config[k.lower()]=v + return config - db_url = ( - "postgresql://" - + postgres_user - + ":" - + postgres_password - + "@" - + postgres_hostname - + ":" - + postgres_port - + "/" - + postgres_db - ) +def extract_values_from_file(filename:str,config:dict, + allow_extend:bool=False, + env_priority:bool=True + )-> dict: + """ function that extrat key=value pairs from a file + Parameters: + - filename: filename/filepath from which to extract key/value pairs found in .env.* file + - config: dict to extend/update with new key/value pairs + - allow_extend: allows to extend extracted keys with new keys that are not in actuel + config if True, restrict to already existing keys in config of False + Returns a dict contains key/value + """ + filepath=Path(Path(__file__).parent).joinpath(filename) + with Path.open(filepath) as file: + for line in file: + # Split line at first occurence of '='. + # This allows to have values containing '=' character + split=line.strip().split('=',1) + # if extraction contains 2 items and strictly 2 items + split_succeed=2 + if(len(split)==split_succeed): + k=split[0] + v=split[1] + # Processing of indirect affectation via [ATTR]_FILE=VALUE_PATH => ATTR=VALUE + if k.lower() in [f"{k}_FILE".lower() for k in config.keys()]\ + and ( k.removesuffix('_FILE').lower() in config.keys() or allow_extend == True)\ + and Path(v).is_file(): + with Path(v).open( mode='r') as file_value: + config[k.removesuffix('_FILE').lower()]=file_value.readline().strip() + # if extracted key already exist in config OR if allowed to add new keys to + # config then adding/updating key/value + if k.lower() in [k.lower() for k in config.keys()] or allow_extend == True: + config[k.lower()]=v + # If env priority True, then overriding all values with ENV values before ending + if env_priority: + extract_values_from_env(config,allow_extend=False) + return config + +class Settings(BaseSettings): + # Déclaration des attributs/paramètres disponibles au sein de la class settings + postgres_user:str = None + postgres_password:str = None + postgres_hostname:str = None + postgres_port:str = None + postgres_db:str = None srid: int = 4326 + db_url:str = None + spire_toekn:str = None + + def __init__(self): + super().__init__(self) + # Default values + srid: int = 4326 + + # Si le fichier de configuration à charger est précisé par la variable d'environnement + # BLOOM_CONFIG alors on charge ce fichier, sinon par défaut c'est ../.env + BLOOM_CONFIG=os.getenv('BLOOM_CONFIG',"../../.env") + + # Ici on charge les paramètres à partir du fichier BLOOM_CONFIG + # et on mets à jour directement les valeurs des paramètres en tant qu'attribut de la + # la classe courante Settings en attanquant le self.__dict__ + # Ces variables sont systmétiquement convertie en lower case + # + # allow_extend=False précise que seuls les attributs déjà existants dans la config + # passée en paramètres (ici self.__dict__) sont mis à jour. Pas de nouveau paramètres + # Cela singifie que pour rendre accessible un nouveau paramètre il faut le déclaré + # dans la liste des attributs de la classe Settings + # + # env_priority=true signifie que si un paramètres est présent dans la classe Settings, + # mas aussi dans le fichier BLOOM_CONFIG ainsi qu'en tant que variable d'environnement + # alors c'est la valeur de la variable d'environnement qui sera chargée au final + # La priorité est donnée aux valeur de l'environnement selon le standard Docker + extract_values_from_file(APP_CONFIG,self.__dict__,allow_extend=False,env_priority=True) + + self.db_url = ( f"postgresql://{self.postgres_user}:" + f"{self.postgres_password}@{self.postgres_host}:" + f"{self.postgres_port}/{self.postgres_db}") + settings = Settings() From 0a34a68c5377b10fb8cc2ab59484ce67b4cc8c03 Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Mon, 4 Mar 2024 14:52:24 +0100 Subject: [PATCH 02/23] =?UTF-8?q?fix:=20some=20mixed=20values,=20d=C3=A9fi?= =?UTF-8?q?ne=20standard=20POSTGRES=5FHOSTNAME=20=3D>=20POSTGRES=5FHOST?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bloom/.env.template | 2 +- bloom/Makefile | 4 ++-- bloom/docs/notes/database.initialisation.md | 2 +- bloom/docs/notes/development.environment.md | 2 +- bloom/tox.ini | 2 +- docker-compose.yaml | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bloom/.env.template b/bloom/.env.template index 32778649..42ee3f6e 100644 --- a/bloom/.env.template +++ b/bloom/.env.template @@ -1,5 +1,5 @@ # these values are used in the local docker env. You can use "localhost" hostname if you run the application without docker -POSTGRES_HOSTNAME=postgres_bloom +POSTGRES_HOST=postgres_bloom POSTGRES_USER=bloom_user POSTGRES_PASSWORD=bloom POSTGRES_DB=bloom_db diff --git a/bloom/Makefile b/bloom/Makefile index 1db006e2..47286bf2 100644 --- a/bloom/Makefile +++ b/bloom/Makefile @@ -48,7 +48,7 @@ launch-production-app: $(BLOOM_PRODUCTION_DOCKER) --name bloom-production-app --rm d4g/bloom:${VERSION} /venv/bin/python3 app.py dump-dev-db: - $(BLOOM_DEV_DOCKER) --rm postgres:latest sh -c 'export PGPASSWORD=$$POSTGRES_PASSWORD && pg_dump -Fc $$POSTGRES_DB -h $$POSTGRES_HOSTNAME -p $$POSTGRES_PORT -U $$POSTGRES_USER> /source_code/bloom_$(shell date +%Y%m%d_%H%M).dump' + $(BLOOM_DEV_DOCKER) --rm postgres:latest sh -c 'export PGPASSWORD=$$POSTGRES_PASSWORD && pg_dump -Fc $$POSTGRES_DB -h $$POSTGRES_HOST -p $$POSTGRES_PORT -U $$POSTGRES_USER> /source_code/bloom_$(shell date +%Y%m%d_%H%M).dump' dump-db: - @docker run --mount type=bind,source="$(shell pwd)",target=/source_code --env-file ./.env.test --network=bloom_net --rm postgres:latest sh -c 'export PGPASSWORD=$$POSTGRES_PASSWORD && pg_dump -Fc $$POSTGRES_DB -h $$POSTGRES_HOSTNAME -p $$POSTGRES_PORT -U $$POSTGRES_USER> /source_code/bloom_$(shell date +%Y%m%d_%H%M).dump' \ No newline at end of file + @docker run --mount type=bind,source="$(shell pwd)",target=/source_code --env-file ./.env.test --network=bloom_net --rm postgres:latest sh -c 'export PGPASSWORD=$$POSTGRES_PASSWORD && pg_dump -Fc $$POSTGRES_DB -h $$POSTGRES_HOST -p $$POSTGRES_PORT -U $$POSTGRES_USER> /source_code/bloom_$(shell date +%Y%m%d_%H%M).dump' \ No newline at end of file diff --git a/bloom/docs/notes/database.initialisation.md b/bloom/docs/notes/database.initialisation.md index 51cb8e08..205c55e2 100644 --- a/bloom/docs/notes/database.initialisation.md +++ b/bloom/docs/notes/database.initialisation.md @@ -15,4 +15,4 @@ The second step is to load the [distance-from-port-v20201104.tiff](https://globa - install psql and raster2pgsql. - install raster type in db with postgis-raster using `create extension postgis_raster` -- adapt this command for each file : `raster2pgsql -t auto -I -C -M /PATH_TO/distance-from-shore.tif public.distance_shore | PGPASSWORD='POSTGRES_PASSWORD' psql -h POSTGRES_HOSTNAME -d POSTGRES_DB -U POSTGRES_USER -p POSTGRES_PORT` +- adapt this command for each file : `raster2pgsql -t auto -I -C -M /PATH_TO/distance-from-shore.tif public.distance_shore | PGPASSWORD='POSTGRES_PASSWORD' psql -h POSTGRES_HOST -d POSTGRES_DB -U POSTGRES_USER -p POSTGRES_PORT` diff --git a/bloom/docs/notes/development.environment.md b/bloom/docs/notes/development.environment.md index 9d3ebcf5..2ab32d34 100644 --- a/bloom/docs/notes/development.environment.md +++ b/bloom/docs/notes/development.environment.md @@ -15,7 +15,7 @@ If you work with Mac m1, the containers may not work as expected ## Local Database -You will need to source the .env.template file but before you should modify it according to your local configuration, for example using POSTGRES_HOSTNAME=localhost +You will need to source the .env.template file but before you should modify it according to your local configuration, for example using POSTGRES_HOST=localhost ## Development Database diff --git a/bloom/tox.ini b/bloom/tox.ini index f764c039..12dac3ab 100644 --- a/bloom/tox.ini +++ b/bloom/tox.ini @@ -16,7 +16,7 @@ commands = setenv = POSTGRES_USER=bloom_user POSTGRES_PASSWORD=bloom - POSTGRES_HOSTNAME=postgres + POSTGRES_HOST=postgres POSTGRES_DB=bloom_db POSTGRES_PORT=5432 SLACK_URL=https://hooks.slack.com/services/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 diff --git a/docker-compose.yaml b/docker-compose.yaml index 17bfdeef..8e285ca1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,7 +1,7 @@ x-common-infos: # Env variables stored in a .env file at same level than docker-compose.yaml environment: &common-env - POSTGRES_HOSTNAME: ${POSTGRES_HOSTNAME} + POSTGRES_HOST: ${POSTGRES_HOST} POSTGRES_PORT: ${POSTGRES_PORT} POSTGRES_DB: ${POSTGRES_DB} POSTGRES_USER: ${POSTGRES_USER} From 23ff12dc747d9917e0d8816a9c13d694f54395a8 Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Mon, 4 Mar 2024 14:58:52 +0100 Subject: [PATCH 03/23] fix: conformisation of POSTGRES_HOSTNAME instead of POSTGRES_HOST --- bloom/.env.template | 2 +- bloom/Makefile | 4 ++-- bloom/bloom/config.py | 2 +- bloom/docs/notes/database.initialisation.md | 2 +- bloom/docs/notes/development.environment.md | 2 +- bloom/tox.ini | 2 +- docker-compose.yaml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bloom/.env.template b/bloom/.env.template index 42ee3f6e..32778649 100644 --- a/bloom/.env.template +++ b/bloom/.env.template @@ -1,5 +1,5 @@ # these values are used in the local docker env. You can use "localhost" hostname if you run the application without docker -POSTGRES_HOST=postgres_bloom +POSTGRES_HOSTNAME=postgres_bloom POSTGRES_USER=bloom_user POSTGRES_PASSWORD=bloom POSTGRES_DB=bloom_db diff --git a/bloom/Makefile b/bloom/Makefile index 47286bf2..1db006e2 100644 --- a/bloom/Makefile +++ b/bloom/Makefile @@ -48,7 +48,7 @@ launch-production-app: $(BLOOM_PRODUCTION_DOCKER) --name bloom-production-app --rm d4g/bloom:${VERSION} /venv/bin/python3 app.py dump-dev-db: - $(BLOOM_DEV_DOCKER) --rm postgres:latest sh -c 'export PGPASSWORD=$$POSTGRES_PASSWORD && pg_dump -Fc $$POSTGRES_DB -h $$POSTGRES_HOST -p $$POSTGRES_PORT -U $$POSTGRES_USER> /source_code/bloom_$(shell date +%Y%m%d_%H%M).dump' + $(BLOOM_DEV_DOCKER) --rm postgres:latest sh -c 'export PGPASSWORD=$$POSTGRES_PASSWORD && pg_dump -Fc $$POSTGRES_DB -h $$POSTGRES_HOSTNAME -p $$POSTGRES_PORT -U $$POSTGRES_USER> /source_code/bloom_$(shell date +%Y%m%d_%H%M).dump' dump-db: - @docker run --mount type=bind,source="$(shell pwd)",target=/source_code --env-file ./.env.test --network=bloom_net --rm postgres:latest sh -c 'export PGPASSWORD=$$POSTGRES_PASSWORD && pg_dump -Fc $$POSTGRES_DB -h $$POSTGRES_HOST -p $$POSTGRES_PORT -U $$POSTGRES_USER> /source_code/bloom_$(shell date +%Y%m%d_%H%M).dump' \ No newline at end of file + @docker run --mount type=bind,source="$(shell pwd)",target=/source_code --env-file ./.env.test --network=bloom_net --rm postgres:latest sh -c 'export PGPASSWORD=$$POSTGRES_PASSWORD && pg_dump -Fc $$POSTGRES_DB -h $$POSTGRES_HOSTNAME -p $$POSTGRES_PORT -U $$POSTGRES_USER> /source_code/bloom_$(shell date +%Y%m%d_%H%M).dump' \ No newline at end of file diff --git a/bloom/bloom/config.py b/bloom/bloom/config.py index 6d89434c..8d297d0a 100644 --- a/bloom/bloom/config.py +++ b/bloom/bloom/config.py @@ -101,7 +101,7 @@ def __init__(self): extract_values_from_file(APP_CONFIG,self.__dict__,allow_extend=False,env_priority=True) self.db_url = ( f"postgresql://{self.postgres_user}:" - f"{self.postgres_password}@{self.postgres_host}:" + f"{self.postgres_password}@{self.postgres_hostname}:" f"{self.postgres_port}/{self.postgres_db}") diff --git a/bloom/docs/notes/database.initialisation.md b/bloom/docs/notes/database.initialisation.md index 205c55e2..51cb8e08 100644 --- a/bloom/docs/notes/database.initialisation.md +++ b/bloom/docs/notes/database.initialisation.md @@ -15,4 +15,4 @@ The second step is to load the [distance-from-port-v20201104.tiff](https://globa - install psql and raster2pgsql. - install raster type in db with postgis-raster using `create extension postgis_raster` -- adapt this command for each file : `raster2pgsql -t auto -I -C -M /PATH_TO/distance-from-shore.tif public.distance_shore | PGPASSWORD='POSTGRES_PASSWORD' psql -h POSTGRES_HOST -d POSTGRES_DB -U POSTGRES_USER -p POSTGRES_PORT` +- adapt this command for each file : `raster2pgsql -t auto -I -C -M /PATH_TO/distance-from-shore.tif public.distance_shore | PGPASSWORD='POSTGRES_PASSWORD' psql -h POSTGRES_HOSTNAME -d POSTGRES_DB -U POSTGRES_USER -p POSTGRES_PORT` diff --git a/bloom/docs/notes/development.environment.md b/bloom/docs/notes/development.environment.md index 2ab32d34..9d3ebcf5 100644 --- a/bloom/docs/notes/development.environment.md +++ b/bloom/docs/notes/development.environment.md @@ -15,7 +15,7 @@ If you work with Mac m1, the containers may not work as expected ## Local Database -You will need to source the .env.template file but before you should modify it according to your local configuration, for example using POSTGRES_HOST=localhost +You will need to source the .env.template file but before you should modify it according to your local configuration, for example using POSTGRES_HOSTNAME=localhost ## Development Database diff --git a/bloom/tox.ini b/bloom/tox.ini index 12dac3ab..f764c039 100644 --- a/bloom/tox.ini +++ b/bloom/tox.ini @@ -16,7 +16,7 @@ commands = setenv = POSTGRES_USER=bloom_user POSTGRES_PASSWORD=bloom - POSTGRES_HOST=postgres + POSTGRES_HOSTNAME=postgres POSTGRES_DB=bloom_db POSTGRES_PORT=5432 SLACK_URL=https://hooks.slack.com/services/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 diff --git a/docker-compose.yaml b/docker-compose.yaml index 8e285ca1..17bfdeef 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,7 +1,7 @@ x-common-infos: # Env variables stored in a .env file at same level than docker-compose.yaml environment: &common-env - POSTGRES_HOST: ${POSTGRES_HOST} + POSTGRES_HOSTNAME: ${POSTGRES_HOSTNAME} POSTGRES_PORT: ${POSTGRES_PORT} POSTGRES_DB: ${POSTGRES_DB} POSTGRES_USER: ${POSTGRES_USER} From 8d7891d12826d1f7f8900a53d621823669dcb83b Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Mon, 4 Mar 2024 15:04:12 +0100 Subject: [PATCH 04/23] feat: paramterizing PYTHON IMAGE for action testing multiple python versions --- bloom/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bloom/Dockerfile b/bloom/Dockerfile index a23d2922..a5c17fd7 100644 --- a/bloom/Dockerfile +++ b/bloom/Dockerfile @@ -1,4 +1,5 @@ -FROM python:3.10-slim-bullseye +ARG IMAGE_PYTHON=${IMAGE_PYTHON:-python:3.10-slim-bullseye} +FROM ${IMAGE_PYTHON} RUN apt-get update RUN apt-get -y install wget gcc g++ From 1d6e5c2f739d6e0efadc926dd6e9e09fe2e14638 Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Mon, 4 Mar 2024 15:04:48 +0100 Subject: [PATCH 05/23] fix: docker remove data from container, must be mounted via volume --- bloom/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/bloom/Dockerfile b/bloom/Dockerfile index a5c17fd7..2530ccb2 100644 --- a/bloom/Dockerfile +++ b/bloom/Dockerfile @@ -8,7 +8,6 @@ RUN apt-get install -y rsyslog # Define working directory WORKDIR /source_code COPY bloom/ ./bloom/ -COPY data/ ./data/ COPY app.py . COPY container.py . COPY docker-env/rsyslog.conf /etc/rsyslog.conf From e9c6aaa2d2ea1820016c680cf9c737d044189808 Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Mon, 4 Mar 2024 15:05:22 +0100 Subject: [PATCH 06/23] fix: conformization of settings acces via bloom.config --- bloom/alembic/env.py | 27 +++---------------- bloom/alembic/init_script/load_amp_data.py | 25 +++-------------- .../init_script/load_positions_data.py | 23 ++-------------- .../alembic/init_script/load_vessels_data.py | 22 ++------------- 4 files changed, 10 insertions(+), 87 deletions(-) diff --git a/bloom/alembic/env.py b/bloom/alembic/env.py index 1d2331a6..8d6eb49c 100644 --- a/bloom/alembic/env.py +++ b/bloom/alembic/env.py @@ -5,6 +5,8 @@ from alembic import context +from bloom.config import settings + # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config @@ -18,30 +20,7 @@ # for 'autogenerate' support target_metadata = None -# other values from the config, defined by the needs of env.py, -# can be acquired: -# ... etc. - -postgres_user = os.environ.get("POSTGRES_USER") -postgres_password = os.environ.get("POSTGRES_PASSWORD") -postgres_hostname = os.environ.get("POSTGRES_HOSTNAME") -postgres_db = os.environ.get("POSTGRES_DB") -postgres_port = os.environ.get("POSTGRES_PORT") - -db_url = ( - "postgresql://" - + postgres_user - + ":" - + postgres_password - + "@" - + postgres_hostname - + ":" - + postgres_port - + "/" - + postgres_db -) - -config.set_main_option("sqlalchemy.url", db_url) +config.set_main_option("sqlalchemy.url", settings.db_url) def run_migrations_offline() -> None: diff --git a/bloom/alembic/init_script/load_amp_data.py b/bloom/alembic/init_script/load_amp_data.py index 86326de1..00a1d627 100644 --- a/bloom/alembic/init_script/load_amp_data.py +++ b/bloom/alembic/init_script/load_amp_data.py @@ -6,32 +6,13 @@ import pandas as pd from shapely import wkb from sqlalchemy import create_engine +from bloom.config import settings + logging.basicConfig() logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO) -postgres_user = os.environ.get("POSTGRES_USER") -postgres_password = os.environ.get("POSTGRES_PASSWORD") -postgres_hostname = os.environ.get("POSTGRES_HOSTNAME") -postgres_db = os.environ.get("POSTGRES_DB") -postgres_port = os.environ.get("POSTGRES_PORT") - - -# The db url is configured with the db connexion variables declared in the db.yaml file. -db_url = ( - "postgresql://" - + postgres_user - + ":" - + postgres_password - + "@" - + postgres_hostname - + ":" - + postgres_port - + "/" - + postgres_db -) - -engine = create_engine(db_url, echo=False) +engine = create_engine(settings.db_url, echo=False) df = pd.read_csv( Path(os.path.dirname(__file__)).joinpath("../../data/zones_subset_02022024.csv"), diff --git a/bloom/alembic/init_script/load_positions_data.py b/bloom/alembic/init_script/load_positions_data.py index 6db0e4e3..cddae0ff 100644 --- a/bloom/alembic/init_script/load_positions_data.py +++ b/bloom/alembic/init_script/load_positions_data.py @@ -4,31 +4,12 @@ import pandas as pd from sqlalchemy import create_engine +from bloom.config import settings logging.basicConfig() logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO) -postgres_user = os.environ.get("POSTGRES_USER") -postgres_password = os.environ.get("POSTGRES_PASSWORD") -postgres_hostname = os.environ.get("POSTGRES_HOSTNAME") -postgres_db = os.environ.get("POSTGRES_DB") -postgres_port = os.environ.get("POSTGRES_PORT") - -# The db url is configured with the db connexion variables declared in the db.yaml file. -db_url = ( - "postgresql://" - + postgres_user - + ":" - + postgres_password - + "@" - + postgres_hostname - + ":" - + postgres_port - + "/" - + postgres_db -) - -engine = create_engine(db_url) +engine = create_engine(settings.db_url) df = pd.read_csv( Path(os.path.dirname(__file__)).joinpath("../../data/spire_positions_subset_02022024.csv"), diff --git a/bloom/alembic/init_script/load_vessels_data.py b/bloom/alembic/init_script/load_vessels_data.py index 03190585..6f3a1848 100644 --- a/bloom/alembic/init_script/load_vessels_data.py +++ b/bloom/alembic/init_script/load_vessels_data.py @@ -4,30 +4,12 @@ import pandas as pd from sqlalchemy import create_engine +from bloom.config import settings logging.basicConfig() logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO) -postgres_user = os.environ.get("POSTGRES_USER") -postgres_password = os.environ.get("POSTGRES_PASSWORD") -postgres_hostname = os.environ.get("POSTGRES_HOSTNAME") -postgres_db = os.environ.get("POSTGRES_DB") -postgres_port = os.environ.get("POSTGRES_PORT") - -# The db url is configured with the db connexion variables declared in the db.yaml file. -db_url = ( - "postgresql://" - + postgres_user - + ":" - + postgres_password - + "@" - + postgres_hostname - + ":" - + postgres_port - + "/" - + postgres_db -) -engine = create_engine(db_url) +engine = create_engine(settings.db_url) df = pd.read_csv( Path(os.path.dirname(__file__)).joinpath("../../data/chalutiers_pelagiques.csv"), sep=";", From 070fdbb49f4cf62cf93e9cf701becaf3b5ac895c Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Mon, 4 Mar 2024 15:07:14 +0100 Subject: [PATCH 07/23] feat: add init container in docker compose file --- docker-compose.yaml | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 17bfdeef..4d213de7 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -9,8 +9,11 @@ x-common-infos: services: bloom: - container_name: bloom-test - build: ./bloom/ + container_name: bloom + image: d4g/bloom:${VERSION:-latest} + build: + context: ./bloom/ + command: streamlit run Trawlwatcher.py volumes: - ./bloom:/source_code @@ -21,8 +24,8 @@ services: networks: - bloom_net depends_on: - postgres: - condition: service_healthy # The service is working and still running + init: + condition: service_completed_successfully postgres: container_name: postgres_bloom @@ -40,6 +43,31 @@ services: retries: 140 start_period: 0s + init: + container_name: init_bloom + hostname: init_bloom + image: d4g/bloom:${VERSION:-latest} + + # Nominal start: + command: alembic check + # Debug start: + #command: bash + #tty: true + #stdin_open: true + + environment: + <<: *common-env + volumes: + - ./bloom/data:/source_code/data + - ./bloom/alembic:/source_code/alembic + - ./bloom/alembic.ini:/source_code/alembic.ini + #- ./bloom/bloom:/source_code/bloom + networks: + - bloom_net + depends_on: + postgres: + condition: service_healthy # The service is working and still running + networks: bloom_net: name: bloom_net \ No newline at end of file From 8dd74aec71b5025a68708f134db1d3e52755db13 Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Mon, 4 Mar 2024 15:07:32 +0100 Subject: [PATCH 08/23] fix: rollback to postgis image for postgres container, the trawlwatcher if in private repository and maybe not conform with alembic --- docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 4d213de7..a59ba9af 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -29,7 +29,7 @@ services: postgres: container_name: postgres_bloom - image: docker.pkg.github.com/wbarillon/docker-packages/trawlwatcher_local_db:0.2 + image: ${POSTGIS_IMAGE:-postgis/postgis:14-3.3-alpine} environment: <<: *common-env ports: From 5dc327cb3117d43f47b6f80b09dd1b347c2df26b Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Mon, 4 Mar 2024 15:08:27 +0100 Subject: [PATCH 09/23] feat: add bloom config managerment for futur use (deployment) --- bloom/bloom/config.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bloom/bloom/config.py b/bloom/bloom/config.py index 8d297d0a..4315f9c2 100644 --- a/bloom/bloom/config.py +++ b/bloom/bloom/config.py @@ -82,7 +82,7 @@ def __init__(self): # Si le fichier de configuration à charger est précisé par la variable d'environnement # BLOOM_CONFIG alors on charge ce fichier, sinon par défaut c'est ../.env - BLOOM_CONFIG=os.getenv('BLOOM_CONFIG',"../../.env") + BLOOM_CONFIG=os.getenv('BLOOM_CONFIG',Path(__file__).parent.joinpath(".env")) # Ici on charge les paramètres à partir du fichier BLOOM_CONFIG # et on mets à jour directement les valeurs des paramètres en tant qu'attribut de la @@ -98,7 +98,10 @@ def __init__(self): # mas aussi dans le fichier BLOOM_CONFIG ainsi qu'en tant que variable d'environnement # alors c'est la valeur de la variable d'environnement qui sera chargée au final # La priorité est donnée aux valeur de l'environnement selon le standard Docker - extract_values_from_file(APP_CONFIG,self.__dict__,allow_extend=False,env_priority=True) + if Path(BLOOM_CONFIG).exists(): + extract_values_from_file(BLOOM_CONFIG,self.__dict__,allow_extend=False,env_priority=True) + else: + extract_values_from_env(self.__dict__,allow_extend=False) self.db_url = ( f"postgresql://{self.postgres_user}:" f"{self.postgres_password}@{self.postgres_hostname}:" From 108451258e4ab7e398deec9e885749ab8c1f7751 Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Mon, 4 Mar 2024 15:07:32 +0100 Subject: [PATCH 10/23] fix: rollback to postgis image for postgres container, the trawlwatcher if in private repository and maybe not conform with alembic --- docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 17bfdeef..6a61dc6a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -26,7 +26,7 @@ services: postgres: container_name: postgres_bloom - image: docker.pkg.github.com/wbarillon/docker-packages/trawlwatcher_local_db:0.2 + image: ${POSTGIS_IMAGE:-postgis/postgis:14-3.3-alpine} environment: <<: *common-env ports: From 3799aa6bab9c4712e5f7ab5aaa3ba5120ec88dd3 Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Mon, 4 Mar 2024 11:47:52 +0100 Subject: [PATCH 11/23] feat: add BLOOM_CONFIG management in bloom.config --- bloom/bloom/config.py | 119 +++++++++++++++++++++++++++++++++++------- 1 file changed, 99 insertions(+), 20 deletions(-) diff --git a/bloom/bloom/config.py b/bloom/bloom/config.py index c0f0e1ea..6d89434c 100644 --- a/bloom/bloom/config.py +++ b/bloom/bloom/config.py @@ -1,29 +1,108 @@ import os +from pathlib import Path from pydantic import BaseSettings -class Settings(BaseSettings): - postgres_user = os.environ.get("POSTGRES_USER") - postgres_password = os.environ.get("POSTGRES_PASSWORD") - postgres_hostname = os.environ.get("POSTGRES_HOSTNAME") - postgres_port = os.environ.get("POSTGRES_PORT") - postgres_db = os.environ.get("POSTGRES_DB") - - print("db_url: ", "postgresql://"+postgres_user+":"+postgres_password+"@"+postgres_hostname+":"+postgres_port+"/"+postgres_db) +def extract_values_from_env(config:dict,allow_extend:bool=False) -> dict: + """ function that extrat key=value pairs from a file + Parameters: + - config: dict to extend/update with new key/value pairs found in environment + - allow_extend: allows to extend extracted keys with new keys that are not in + actuel config if True, restrict to already existing keys in config of False + Returns a dict contains key/value + """ + for k,v in os.environ.items(): + # Processing of indirect affectation via [ATTR]_FILE=VALUE_PATH => ATTR=VALUE + if k.lower() in [f"{k}_FILE".lower() for k in config.keys()]\ + and ( k.removesuffix('_FILE').lower() in config.keys() or allow_extend == True)\ + and Path(v).is_file(): + with Path.open(v, mode='r') as file: + config[k.removesuffix('_FILE').lower()]=file.readline().strip() + # Processing of direct affectation via ATTR=VALUE + # if extracted key already exist in config OR if allowed to add new keys to config + # Then adding/updating key/value + if k.lower() in [k.lower() for k in config.keys()] or allow_extend == True: + config[k.lower()]=v + return config - db_url = ( - "postgresql://" - + postgres_user - + ":" - + postgres_password - + "@" - + postgres_hostname - + ":" - + postgres_port - + "/" - + postgres_db - ) +def extract_values_from_file(filename:str,config:dict, + allow_extend:bool=False, + env_priority:bool=True + )-> dict: + """ function that extrat key=value pairs from a file + Parameters: + - filename: filename/filepath from which to extract key/value pairs found in .env.* file + - config: dict to extend/update with new key/value pairs + - allow_extend: allows to extend extracted keys with new keys that are not in actuel + config if True, restrict to already existing keys in config of False + Returns a dict contains key/value + """ + filepath=Path(Path(__file__).parent).joinpath(filename) + with Path.open(filepath) as file: + for line in file: + # Split line at first occurence of '='. + # This allows to have values containing '=' character + split=line.strip().split('=',1) + # if extraction contains 2 items and strictly 2 items + split_succeed=2 + if(len(split)==split_succeed): + k=split[0] + v=split[1] + # Processing of indirect affectation via [ATTR]_FILE=VALUE_PATH => ATTR=VALUE + if k.lower() in [f"{k}_FILE".lower() for k in config.keys()]\ + and ( k.removesuffix('_FILE').lower() in config.keys() or allow_extend == True)\ + and Path(v).is_file(): + with Path(v).open( mode='r') as file_value: + config[k.removesuffix('_FILE').lower()]=file_value.readline().strip() + # if extracted key already exist in config OR if allowed to add new keys to + # config then adding/updating key/value + if k.lower() in [k.lower() for k in config.keys()] or allow_extend == True: + config[k.lower()]=v + # If env priority True, then overriding all values with ENV values before ending + if env_priority: + extract_values_from_env(config,allow_extend=False) + return config + +class Settings(BaseSettings): + # Déclaration des attributs/paramètres disponibles au sein de la class settings + postgres_user:str = None + postgres_password:str = None + postgres_hostname:str = None + postgres_port:str = None + postgres_db:str = None srid: int = 4326 + db_url:str = None + spire_toekn:str = None + + def __init__(self): + super().__init__(self) + # Default values + srid: int = 4326 + + # Si le fichier de configuration à charger est précisé par la variable d'environnement + # BLOOM_CONFIG alors on charge ce fichier, sinon par défaut c'est ../.env + BLOOM_CONFIG=os.getenv('BLOOM_CONFIG',"../../.env") + + # Ici on charge les paramètres à partir du fichier BLOOM_CONFIG + # et on mets à jour directement les valeurs des paramètres en tant qu'attribut de la + # la classe courante Settings en attanquant le self.__dict__ + # Ces variables sont systmétiquement convertie en lower case + # + # allow_extend=False précise que seuls les attributs déjà existants dans la config + # passée en paramètres (ici self.__dict__) sont mis à jour. Pas de nouveau paramètres + # Cela singifie que pour rendre accessible un nouveau paramètre il faut le déclaré + # dans la liste des attributs de la classe Settings + # + # env_priority=true signifie que si un paramètres est présent dans la classe Settings, + # mas aussi dans le fichier BLOOM_CONFIG ainsi qu'en tant que variable d'environnement + # alors c'est la valeur de la variable d'environnement qui sera chargée au final + # La priorité est donnée aux valeur de l'environnement selon le standard Docker + extract_values_from_file(APP_CONFIG,self.__dict__,allow_extend=False,env_priority=True) + + self.db_url = ( f"postgresql://{self.postgres_user}:" + f"{self.postgres_password}@{self.postgres_host}:" + f"{self.postgres_port}/{self.postgres_db}") + settings = Settings() From 0b6f5e4a82429ed3501ba27507ceac2007fd4ee4 Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Mon, 4 Mar 2024 14:52:24 +0100 Subject: [PATCH 12/23] =?UTF-8?q?fix:=20some=20mixed=20values,=20d=C3=A9fi?= =?UTF-8?q?ne=20standard=20POSTGRES=5FHOSTNAME=20=3D>=20POSTGRES=5FHOST?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bloom/.env.template | 2 +- bloom/Makefile | 4 ++-- bloom/docs/notes/database.initialisation.md | 2 +- bloom/docs/notes/development.environment.md | 2 +- bloom/tox.ini | 2 +- docker-compose.yaml | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bloom/.env.template b/bloom/.env.template index 32778649..42ee3f6e 100644 --- a/bloom/.env.template +++ b/bloom/.env.template @@ -1,5 +1,5 @@ # these values are used in the local docker env. You can use "localhost" hostname if you run the application without docker -POSTGRES_HOSTNAME=postgres_bloom +POSTGRES_HOST=postgres_bloom POSTGRES_USER=bloom_user POSTGRES_PASSWORD=bloom POSTGRES_DB=bloom_db diff --git a/bloom/Makefile b/bloom/Makefile index 1db006e2..47286bf2 100644 --- a/bloom/Makefile +++ b/bloom/Makefile @@ -48,7 +48,7 @@ launch-production-app: $(BLOOM_PRODUCTION_DOCKER) --name bloom-production-app --rm d4g/bloom:${VERSION} /venv/bin/python3 app.py dump-dev-db: - $(BLOOM_DEV_DOCKER) --rm postgres:latest sh -c 'export PGPASSWORD=$$POSTGRES_PASSWORD && pg_dump -Fc $$POSTGRES_DB -h $$POSTGRES_HOSTNAME -p $$POSTGRES_PORT -U $$POSTGRES_USER> /source_code/bloom_$(shell date +%Y%m%d_%H%M).dump' + $(BLOOM_DEV_DOCKER) --rm postgres:latest sh -c 'export PGPASSWORD=$$POSTGRES_PASSWORD && pg_dump -Fc $$POSTGRES_DB -h $$POSTGRES_HOST -p $$POSTGRES_PORT -U $$POSTGRES_USER> /source_code/bloom_$(shell date +%Y%m%d_%H%M).dump' dump-db: - @docker run --mount type=bind,source="$(shell pwd)",target=/source_code --env-file ./.env.test --network=bloom_net --rm postgres:latest sh -c 'export PGPASSWORD=$$POSTGRES_PASSWORD && pg_dump -Fc $$POSTGRES_DB -h $$POSTGRES_HOSTNAME -p $$POSTGRES_PORT -U $$POSTGRES_USER> /source_code/bloom_$(shell date +%Y%m%d_%H%M).dump' \ No newline at end of file + @docker run --mount type=bind,source="$(shell pwd)",target=/source_code --env-file ./.env.test --network=bloom_net --rm postgres:latest sh -c 'export PGPASSWORD=$$POSTGRES_PASSWORD && pg_dump -Fc $$POSTGRES_DB -h $$POSTGRES_HOST -p $$POSTGRES_PORT -U $$POSTGRES_USER> /source_code/bloom_$(shell date +%Y%m%d_%H%M).dump' \ No newline at end of file diff --git a/bloom/docs/notes/database.initialisation.md b/bloom/docs/notes/database.initialisation.md index 51cb8e08..205c55e2 100644 --- a/bloom/docs/notes/database.initialisation.md +++ b/bloom/docs/notes/database.initialisation.md @@ -15,4 +15,4 @@ The second step is to load the [distance-from-port-v20201104.tiff](https://globa - install psql and raster2pgsql. - install raster type in db with postgis-raster using `create extension postgis_raster` -- adapt this command for each file : `raster2pgsql -t auto -I -C -M /PATH_TO/distance-from-shore.tif public.distance_shore | PGPASSWORD='POSTGRES_PASSWORD' psql -h POSTGRES_HOSTNAME -d POSTGRES_DB -U POSTGRES_USER -p POSTGRES_PORT` +- adapt this command for each file : `raster2pgsql -t auto -I -C -M /PATH_TO/distance-from-shore.tif public.distance_shore | PGPASSWORD='POSTGRES_PASSWORD' psql -h POSTGRES_HOST -d POSTGRES_DB -U POSTGRES_USER -p POSTGRES_PORT` diff --git a/bloom/docs/notes/development.environment.md b/bloom/docs/notes/development.environment.md index 9d3ebcf5..2ab32d34 100644 --- a/bloom/docs/notes/development.environment.md +++ b/bloom/docs/notes/development.environment.md @@ -15,7 +15,7 @@ If you work with Mac m1, the containers may not work as expected ## Local Database -You will need to source the .env.template file but before you should modify it according to your local configuration, for example using POSTGRES_HOSTNAME=localhost +You will need to source the .env.template file but before you should modify it according to your local configuration, for example using POSTGRES_HOST=localhost ## Development Database diff --git a/bloom/tox.ini b/bloom/tox.ini index f764c039..12dac3ab 100644 --- a/bloom/tox.ini +++ b/bloom/tox.ini @@ -16,7 +16,7 @@ commands = setenv = POSTGRES_USER=bloom_user POSTGRES_PASSWORD=bloom - POSTGRES_HOSTNAME=postgres + POSTGRES_HOST=postgres POSTGRES_DB=bloom_db POSTGRES_PORT=5432 SLACK_URL=https://hooks.slack.com/services/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 diff --git a/docker-compose.yaml b/docker-compose.yaml index 6a61dc6a..86968790 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,7 +1,7 @@ x-common-infos: # Env variables stored in a .env file at same level than docker-compose.yaml environment: &common-env - POSTGRES_HOSTNAME: ${POSTGRES_HOSTNAME} + POSTGRES_HOST: ${POSTGRES_HOST} POSTGRES_PORT: ${POSTGRES_PORT} POSTGRES_DB: ${POSTGRES_DB} POSTGRES_USER: ${POSTGRES_USER} From 6bf3d853141296f65c60c6ec39ad947ac08c9a83 Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Mon, 4 Mar 2024 14:58:52 +0100 Subject: [PATCH 13/23] fix: conformisation of POSTGRES_HOSTNAME instead of POSTGRES_HOST --- bloom/.env.template | 2 +- bloom/Makefile | 4 ++-- bloom/bloom/config.py | 2 +- bloom/docs/notes/database.initialisation.md | 2 +- bloom/docs/notes/development.environment.md | 2 +- bloom/tox.ini | 2 +- docker-compose.yaml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bloom/.env.template b/bloom/.env.template index 42ee3f6e..32778649 100644 --- a/bloom/.env.template +++ b/bloom/.env.template @@ -1,5 +1,5 @@ # these values are used in the local docker env. You can use "localhost" hostname if you run the application without docker -POSTGRES_HOST=postgres_bloom +POSTGRES_HOSTNAME=postgres_bloom POSTGRES_USER=bloom_user POSTGRES_PASSWORD=bloom POSTGRES_DB=bloom_db diff --git a/bloom/Makefile b/bloom/Makefile index 47286bf2..1db006e2 100644 --- a/bloom/Makefile +++ b/bloom/Makefile @@ -48,7 +48,7 @@ launch-production-app: $(BLOOM_PRODUCTION_DOCKER) --name bloom-production-app --rm d4g/bloom:${VERSION} /venv/bin/python3 app.py dump-dev-db: - $(BLOOM_DEV_DOCKER) --rm postgres:latest sh -c 'export PGPASSWORD=$$POSTGRES_PASSWORD && pg_dump -Fc $$POSTGRES_DB -h $$POSTGRES_HOST -p $$POSTGRES_PORT -U $$POSTGRES_USER> /source_code/bloom_$(shell date +%Y%m%d_%H%M).dump' + $(BLOOM_DEV_DOCKER) --rm postgres:latest sh -c 'export PGPASSWORD=$$POSTGRES_PASSWORD && pg_dump -Fc $$POSTGRES_DB -h $$POSTGRES_HOSTNAME -p $$POSTGRES_PORT -U $$POSTGRES_USER> /source_code/bloom_$(shell date +%Y%m%d_%H%M).dump' dump-db: - @docker run --mount type=bind,source="$(shell pwd)",target=/source_code --env-file ./.env.test --network=bloom_net --rm postgres:latest sh -c 'export PGPASSWORD=$$POSTGRES_PASSWORD && pg_dump -Fc $$POSTGRES_DB -h $$POSTGRES_HOST -p $$POSTGRES_PORT -U $$POSTGRES_USER> /source_code/bloom_$(shell date +%Y%m%d_%H%M).dump' \ No newline at end of file + @docker run --mount type=bind,source="$(shell pwd)",target=/source_code --env-file ./.env.test --network=bloom_net --rm postgres:latest sh -c 'export PGPASSWORD=$$POSTGRES_PASSWORD && pg_dump -Fc $$POSTGRES_DB -h $$POSTGRES_HOSTNAME -p $$POSTGRES_PORT -U $$POSTGRES_USER> /source_code/bloom_$(shell date +%Y%m%d_%H%M).dump' \ No newline at end of file diff --git a/bloom/bloom/config.py b/bloom/bloom/config.py index 6d89434c..8d297d0a 100644 --- a/bloom/bloom/config.py +++ b/bloom/bloom/config.py @@ -101,7 +101,7 @@ def __init__(self): extract_values_from_file(APP_CONFIG,self.__dict__,allow_extend=False,env_priority=True) self.db_url = ( f"postgresql://{self.postgres_user}:" - f"{self.postgres_password}@{self.postgres_host}:" + f"{self.postgres_password}@{self.postgres_hostname}:" f"{self.postgres_port}/{self.postgres_db}") diff --git a/bloom/docs/notes/database.initialisation.md b/bloom/docs/notes/database.initialisation.md index 205c55e2..51cb8e08 100644 --- a/bloom/docs/notes/database.initialisation.md +++ b/bloom/docs/notes/database.initialisation.md @@ -15,4 +15,4 @@ The second step is to load the [distance-from-port-v20201104.tiff](https://globa - install psql and raster2pgsql. - install raster type in db with postgis-raster using `create extension postgis_raster` -- adapt this command for each file : `raster2pgsql -t auto -I -C -M /PATH_TO/distance-from-shore.tif public.distance_shore | PGPASSWORD='POSTGRES_PASSWORD' psql -h POSTGRES_HOST -d POSTGRES_DB -U POSTGRES_USER -p POSTGRES_PORT` +- adapt this command for each file : `raster2pgsql -t auto -I -C -M /PATH_TO/distance-from-shore.tif public.distance_shore | PGPASSWORD='POSTGRES_PASSWORD' psql -h POSTGRES_HOSTNAME -d POSTGRES_DB -U POSTGRES_USER -p POSTGRES_PORT` diff --git a/bloom/docs/notes/development.environment.md b/bloom/docs/notes/development.environment.md index 2ab32d34..9d3ebcf5 100644 --- a/bloom/docs/notes/development.environment.md +++ b/bloom/docs/notes/development.environment.md @@ -15,7 +15,7 @@ If you work with Mac m1, the containers may not work as expected ## Local Database -You will need to source the .env.template file but before you should modify it according to your local configuration, for example using POSTGRES_HOST=localhost +You will need to source the .env.template file but before you should modify it according to your local configuration, for example using POSTGRES_HOSTNAME=localhost ## Development Database diff --git a/bloom/tox.ini b/bloom/tox.ini index 12dac3ab..f764c039 100644 --- a/bloom/tox.ini +++ b/bloom/tox.ini @@ -16,7 +16,7 @@ commands = setenv = POSTGRES_USER=bloom_user POSTGRES_PASSWORD=bloom - POSTGRES_HOST=postgres + POSTGRES_HOSTNAME=postgres POSTGRES_DB=bloom_db POSTGRES_PORT=5432 SLACK_URL=https://hooks.slack.com/services/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 diff --git a/docker-compose.yaml b/docker-compose.yaml index 86968790..6a61dc6a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,7 +1,7 @@ x-common-infos: # Env variables stored in a .env file at same level than docker-compose.yaml environment: &common-env - POSTGRES_HOST: ${POSTGRES_HOST} + POSTGRES_HOSTNAME: ${POSTGRES_HOSTNAME} POSTGRES_PORT: ${POSTGRES_PORT} POSTGRES_DB: ${POSTGRES_DB} POSTGRES_USER: ${POSTGRES_USER} From fe010bbc75a04b4ec19b2f506f55c36108ac792a Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Mon, 4 Mar 2024 15:08:27 +0100 Subject: [PATCH 14/23] feat: add bloom config managerment for futur use (deployment) --- bloom/bloom/config.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bloom/bloom/config.py b/bloom/bloom/config.py index 8d297d0a..4315f9c2 100644 --- a/bloom/bloom/config.py +++ b/bloom/bloom/config.py @@ -82,7 +82,7 @@ def __init__(self): # Si le fichier de configuration à charger est précisé par la variable d'environnement # BLOOM_CONFIG alors on charge ce fichier, sinon par défaut c'est ../.env - BLOOM_CONFIG=os.getenv('BLOOM_CONFIG',"../../.env") + BLOOM_CONFIG=os.getenv('BLOOM_CONFIG',Path(__file__).parent.joinpath(".env")) # Ici on charge les paramètres à partir du fichier BLOOM_CONFIG # et on mets à jour directement les valeurs des paramètres en tant qu'attribut de la @@ -98,7 +98,10 @@ def __init__(self): # mas aussi dans le fichier BLOOM_CONFIG ainsi qu'en tant que variable d'environnement # alors c'est la valeur de la variable d'environnement qui sera chargée au final # La priorité est donnée aux valeur de l'environnement selon le standard Docker - extract_values_from_file(APP_CONFIG,self.__dict__,allow_extend=False,env_priority=True) + if Path(BLOOM_CONFIG).exists(): + extract_values_from_file(BLOOM_CONFIG,self.__dict__,allow_extend=False,env_priority=True) + else: + extract_values_from_env(self.__dict__,allow_extend=False) self.db_url = ( f"postgresql://{self.postgres_user}:" f"{self.postgres_password}@{self.postgres_hostname}:" From 75c1a43d5a3e242e4ea103c5ece10a0eeb2b6c80 Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Mon, 4 Mar 2024 15:07:14 +0100 Subject: [PATCH 15/23] feat: add init container in docker compose file --- docker-compose.yaml | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 6a61dc6a..a59ba9af 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -9,8 +9,11 @@ x-common-infos: services: bloom: - container_name: bloom-test - build: ./bloom/ + container_name: bloom + image: d4g/bloom:${VERSION:-latest} + build: + context: ./bloom/ + command: streamlit run Trawlwatcher.py volumes: - ./bloom:/source_code @@ -21,8 +24,8 @@ services: networks: - bloom_net depends_on: - postgres: - condition: service_healthy # The service is working and still running + init: + condition: service_completed_successfully postgres: container_name: postgres_bloom @@ -40,6 +43,31 @@ services: retries: 140 start_period: 0s + init: + container_name: init_bloom + hostname: init_bloom + image: d4g/bloom:${VERSION:-latest} + + # Nominal start: + command: alembic check + # Debug start: + #command: bash + #tty: true + #stdin_open: true + + environment: + <<: *common-env + volumes: + - ./bloom/data:/source_code/data + - ./bloom/alembic:/source_code/alembic + - ./bloom/alembic.ini:/source_code/alembic.ini + #- ./bloom/bloom:/source_code/bloom + networks: + - bloom_net + depends_on: + postgres: + condition: service_healthy # The service is working and still running + networks: bloom_net: name: bloom_net \ No newline at end of file From 154415662f516c9f1c6f6efbc837f253bb477bb7 Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Mon, 4 Mar 2024 14:52:24 +0100 Subject: [PATCH 16/23] fix: remove requirements.txt & rearrange pytoml + typo --- bloom/bloom/config.py | 2 +- bloom/pyproject.toml | 36 ++++++++++++++++++------------------ bloom/requirements.txt | 24 ------------------------ 3 files changed, 19 insertions(+), 43 deletions(-) delete mode 100644 bloom/requirements.txt diff --git a/bloom/bloom/config.py b/bloom/bloom/config.py index 4315f9c2..392d28b3 100644 --- a/bloom/bloom/config.py +++ b/bloom/bloom/config.py @@ -73,7 +73,7 @@ class Settings(BaseSettings): postgres_db:str = None srid: int = 4326 db_url:str = None - spire_toekn:str = None + spire_token:str = None def __init__(self): super().__init__(self) diff --git a/bloom/pyproject.toml b/bloom/pyproject.toml index 3bd53d6b..1a36901f 100644 --- a/bloom/pyproject.toml +++ b/bloom/pyproject.toml @@ -9,33 +9,33 @@ description = "Bloom scrapping application" authors = ["Your Name "] [tool.poetry.dependencies] -python = "^3.10" -poetry = "^1.8" +alembic = "^1.10.2" autopep8 = "1.5.4" -pandas = "1.4.4" -selenium = "4.18.1" -undetected-chromedriver = "3.4.6" +dependency-injector = "^4.41.0" +dependency_injection = "1.2.0" +folium = "^0.14.0" +geoalchemy2 = "^0.13.1" +geopandas = "0.14.3" +geopy = "^2.4.0" +gql = "^3.4.0" +matplotlib = "^3.8.0" openpyxl = "^3.1.0" +pandas = "1.4.4" +poetry = "^1.8" +psycopg2-binary = "^2.9.6" pydantic = "1.10.14" +python = "^3.10" +python-dotenv = "^1.0.0" pyyaml = "^6.0" -shapely = "2.0.1" -geopandas = "0.14.3" -sqlalchemy = "^2.0" -alembic = "^1.10.2" -geoalchemy2 = "^0.13.1" -psycopg2-binary = "^2.9.6" -dependency-injector = "^4.41.0" -dependency_injection = "1.2.0" requests = "^2.31" -gql = "^3.4.0" requests-toolbelt = "^1.0" +selenium = "4.18.1" +shapely = "2.0.1" slack-sdk = "^3.21.3" -folium = "^0.14.0" -matplotlib = "^3.8.0" -geopy = "^2.4.0" +sqlalchemy = "^2.0" streamlit = "^1.27.2" streamlit-folium = "^0.15.0" -python-dotenv = "^1.0.0" +undetected-chromedriver = "3.4.6" [tool.poetry.group.dev.dependencies] diff --git a/bloom/requirements.txt b/bloom/requirements.txt deleted file mode 100644 index 79dc9d1b..00000000 --- a/bloom/requirements.txt +++ /dev/null @@ -1,24 +0,0 @@ -pandas==2.2.1 -selenium==4.18.1 -undetected-chromedriver==3.4.6 -openpyxl==3.1.0 -pydantic==1.10.14 -pyyaml==6.0 -shapely==2.0.1 -geopandas==0.14.3 -sqlalchemy==2.0 -alembic==1.10.2 -geoalchemy2==0.13.1 -psycopg2-binary==2.9.6 -dependency-injector==4.41.0 -dependency_injection==1.2.0 -requests==2.31 -gql==3.4.0 -requests-toolbelt==1.0 -slack-sdk==3.21.3 -folium==0.14.0 -matplotlib==3.8.0 -geopy==2.4.0 -streamlit==1.27.2 -streamlit-folium==0.15.0 -python-dotenv==1.0.0 \ No newline at end of file From 27ebb4bc7e3fafd3aa87b3ebca79ec53c9e308ca Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Mon, 4 Mar 2024 15:05:22 +0100 Subject: [PATCH 17/23] fix: conformization of settings acces via bloom.config --- bloom/alembic/env.py | 27 +++---------------- bloom/alembic/init_script/load_amp_data.py | 25 +++-------------- .../init_script/load_positions_data.py | 23 ++-------------- .../alembic/init_script/load_vessels_data.py | 22 ++------------- 4 files changed, 10 insertions(+), 87 deletions(-) diff --git a/bloom/alembic/env.py b/bloom/alembic/env.py index 1d2331a6..8d6eb49c 100644 --- a/bloom/alembic/env.py +++ b/bloom/alembic/env.py @@ -5,6 +5,8 @@ from alembic import context +from bloom.config import settings + # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config @@ -18,30 +20,7 @@ # for 'autogenerate' support target_metadata = None -# other values from the config, defined by the needs of env.py, -# can be acquired: -# ... etc. - -postgres_user = os.environ.get("POSTGRES_USER") -postgres_password = os.environ.get("POSTGRES_PASSWORD") -postgres_hostname = os.environ.get("POSTGRES_HOSTNAME") -postgres_db = os.environ.get("POSTGRES_DB") -postgres_port = os.environ.get("POSTGRES_PORT") - -db_url = ( - "postgresql://" - + postgres_user - + ":" - + postgres_password - + "@" - + postgres_hostname - + ":" - + postgres_port - + "/" - + postgres_db -) - -config.set_main_option("sqlalchemy.url", db_url) +config.set_main_option("sqlalchemy.url", settings.db_url) def run_migrations_offline() -> None: diff --git a/bloom/alembic/init_script/load_amp_data.py b/bloom/alembic/init_script/load_amp_data.py index 86326de1..00a1d627 100644 --- a/bloom/alembic/init_script/load_amp_data.py +++ b/bloom/alembic/init_script/load_amp_data.py @@ -6,32 +6,13 @@ import pandas as pd from shapely import wkb from sqlalchemy import create_engine +from bloom.config import settings + logging.basicConfig() logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO) -postgres_user = os.environ.get("POSTGRES_USER") -postgres_password = os.environ.get("POSTGRES_PASSWORD") -postgres_hostname = os.environ.get("POSTGRES_HOSTNAME") -postgres_db = os.environ.get("POSTGRES_DB") -postgres_port = os.environ.get("POSTGRES_PORT") - - -# The db url is configured with the db connexion variables declared in the db.yaml file. -db_url = ( - "postgresql://" - + postgres_user - + ":" - + postgres_password - + "@" - + postgres_hostname - + ":" - + postgres_port - + "/" - + postgres_db -) - -engine = create_engine(db_url, echo=False) +engine = create_engine(settings.db_url, echo=False) df = pd.read_csv( Path(os.path.dirname(__file__)).joinpath("../../data/zones_subset_02022024.csv"), diff --git a/bloom/alembic/init_script/load_positions_data.py b/bloom/alembic/init_script/load_positions_data.py index 6db0e4e3..cddae0ff 100644 --- a/bloom/alembic/init_script/load_positions_data.py +++ b/bloom/alembic/init_script/load_positions_data.py @@ -4,31 +4,12 @@ import pandas as pd from sqlalchemy import create_engine +from bloom.config import settings logging.basicConfig() logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO) -postgres_user = os.environ.get("POSTGRES_USER") -postgres_password = os.environ.get("POSTGRES_PASSWORD") -postgres_hostname = os.environ.get("POSTGRES_HOSTNAME") -postgres_db = os.environ.get("POSTGRES_DB") -postgres_port = os.environ.get("POSTGRES_PORT") - -# The db url is configured with the db connexion variables declared in the db.yaml file. -db_url = ( - "postgresql://" - + postgres_user - + ":" - + postgres_password - + "@" - + postgres_hostname - + ":" - + postgres_port - + "/" - + postgres_db -) - -engine = create_engine(db_url) +engine = create_engine(settings.db_url) df = pd.read_csv( Path(os.path.dirname(__file__)).joinpath("../../data/spire_positions_subset_02022024.csv"), diff --git a/bloom/alembic/init_script/load_vessels_data.py b/bloom/alembic/init_script/load_vessels_data.py index 03190585..6f3a1848 100644 --- a/bloom/alembic/init_script/load_vessels_data.py +++ b/bloom/alembic/init_script/load_vessels_data.py @@ -4,30 +4,12 @@ import pandas as pd from sqlalchemy import create_engine +from bloom.config import settings logging.basicConfig() logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO) -postgres_user = os.environ.get("POSTGRES_USER") -postgres_password = os.environ.get("POSTGRES_PASSWORD") -postgres_hostname = os.environ.get("POSTGRES_HOSTNAME") -postgres_db = os.environ.get("POSTGRES_DB") -postgres_port = os.environ.get("POSTGRES_PORT") - -# The db url is configured with the db connexion variables declared in the db.yaml file. -db_url = ( - "postgresql://" - + postgres_user - + ":" - + postgres_password - + "@" - + postgres_hostname - + ":" - + postgres_port - + "/" - + postgres_db -) -engine = create_engine(db_url) +engine = create_engine(settings.db_url) df = pd.read_csv( Path(os.path.dirname(__file__)).joinpath("../../data/chalutiers_pelagiques.csv"), sep=";", From 9f076a4b5a7d88cbac44760542f55ccc53a766cd Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Mon, 4 Mar 2024 19:09:06 +0100 Subject: [PATCH 18/23] fix: docker compose + contener init --- docker-compose.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index a59ba9af..c9586d13 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -14,7 +14,12 @@ services: build: context: ./bloom/ + # Nominal start: command: streamlit run Trawlwatcher.py + # Debug start: + #command: bash + #tty: true + #stdin_open: true volumes: - ./bloom:/source_code environment: @@ -49,7 +54,7 @@ services: image: d4g/bloom:${VERSION:-latest} # Nominal start: - command: alembic check + command: /bin/bash -c "sleep 20 && alembic upgrade head" # Debug start: #command: bash #tty: true @@ -58,10 +63,7 @@ services: environment: <<: *common-env volumes: - - ./bloom/data:/source_code/data - - ./bloom/alembic:/source_code/alembic - - ./bloom/alembic.ini:/source_code/alembic.ini - #- ./bloom/bloom:/source_code/bloom + - ./bloom:/source_code networks: - bloom_net depends_on: From ae497a09630cbd5344d003be81f55639aacf0d19 Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Mon, 4 Mar 2024 19:22:39 +0100 Subject: [PATCH 19/23] fix: move of README + merge of docker compose procedure --- bloom/README.md => README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) rename bloom/README.md => README.md (92%) diff --git a/bloom/README.md b/README.md similarity index 92% rename from bloom/README.md rename to README.md index d7a30067..810838a5 100644 --- a/bloom/README.md +++ b/README.md @@ -12,6 +12,20 @@ **BLOOM** is a non-profit organization founded in 2005 that works to preserve the marine environment and species from unnecessary destruction and to increase social benefits in the fishing sector. **BLOOM** wages awareness and advocacy campaigns in order to accelerate the adoption of concrete solutions for the ocean, humans and the climate. **BLOOM** carries out scientific research projects, independent studies and evaluations that highlight crucial and unaddressed issues such as the financing mechanisms of the fishing sector. **BLOOM**’s actions are meant for the general public as well as policy-makers and economic stakeholders. +## Installing Trawl Watch with Docker Compose + +* git clone https://github.com/dataforgoodfr/12_bloom.git +* cd 12_bloom +* docker compose build +* docker compose pull +* copy and paste bloom/env.template at the same level than docker-compose.yaml and rename it .env +* docker compose run --service-ports bloom /bin/bash +* streamlit run Trawlwatcher.py +* working mmsi : 261084090 + +2. Ensure [Docker](https://docs.docker.com/get-docker/) is installed. + + ## Installing Trawl Watch with `poetry` ### Prerequisites: From 9cebd50f076fb55b7419c3c847a90b8e4c8b4086 Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Mon, 4 Mar 2024 19:23:09 +0100 Subject: [PATCH 20/23] fix: move of poetry files to root --- bloom/poetry.lock => poetry.lock | 0 bloom/pyproject.toml => pyproject.toml | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename bloom/poetry.lock => poetry.lock (100%) rename bloom/pyproject.toml => pyproject.toml (100%) diff --git a/bloom/poetry.lock b/poetry.lock similarity index 100% rename from bloom/poetry.lock rename to poetry.lock diff --git a/bloom/pyproject.toml b/pyproject.toml similarity index 100% rename from bloom/pyproject.toml rename to pyproject.toml From 8bf59729880243ddcb57d4ceb80cf034e60c97c0 Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Mon, 4 Mar 2024 19:24:32 +0100 Subject: [PATCH 21/23] fix: move of README + merge of docker compose procedure --- README.md | 2 +- README.txt | 17 ----------------- 2 files changed, 1 insertion(+), 18 deletions(-) delete mode 100644 README.txt diff --git a/README.md b/README.md index 810838a5..e67956a6 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ ## Installing Trawl Watch with Docker Compose +* Ensure [Docker](https://docs.docker.com/get-docker/) is installed. * git clone https://github.com/dataforgoodfr/12_bloom.git * cd 12_bloom * docker compose build @@ -23,7 +24,6 @@ * streamlit run Trawlwatcher.py * working mmsi : 261084090 -2. Ensure [Docker](https://docs.docker.com/get-docker/) is installed. ## Installing Trawl Watch with `poetry` diff --git a/README.txt b/README.txt deleted file mode 100644 index a2213cb0..00000000 --- a/README.txt +++ /dev/null @@ -1,17 +0,0 @@ -Yet another readme :D - -Quickstart : - -git clone https://github.com/dataforgoodfr/12_bloom.git - -docker compose build - -docker compose pull - -copy and paste bloom/env.template at the same level than docker-compose.yaml and rename it .env - -docker compose run --service-ports bloom /bin/bash - -streamlit run Trawlwatcher.py - -working mmsi : 261084090 From 39a39b20ea73b96c79fd7dc18d04f0e7c8e10ff6 Mon Sep 17 00:00:00 2001 From: RV Date: Mon, 4 Mar 2024 22:28:39 +0100 Subject: [PATCH 22/23] fix: Trawlwatcher config loading & spire_token config loading --- bloom/Trawlwatcher.py | 13 ++++++++----- bloom/bloom/usecase/GetVesselsFromSpire.py | 3 ++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/bloom/Trawlwatcher.py b/bloom/Trawlwatcher.py index e3b251d5..c61640bf 100644 --- a/bloom/Trawlwatcher.py +++ b/bloom/Trawlwatcher.py @@ -3,6 +3,8 @@ import streamlit as st from dotenv import load_dotenv +from bloom.config import settings + st.set_page_config( page_title="Bloom Trawlwatcher Demo app", @@ -11,17 +13,18 @@ ) # FILL IN YOUR CREDENTIALS .env file HERE !! -env_path = Path('.') / '.env.template' -if not env_path.is_file(): - raise FileNotFoundError(f"Couldn't find .env file at {env_path.absolute()}") -load_dotenv(env_path) + +#env_path = Path('.') / '.env.template' +#if not env_path.is_file(): +# raise FileNotFoundError(f"Couldn't find .env file at {env_path.absolute()}") +#load_dotenv(env_path) def local_css(file_name: str) -> None: with Path.open(file_name) as f: st.markdown(f"", unsafe_allow_html=True) return -local_css(Path("styles.css")) +local_css(Path(__file__).parent.joinpath("./styles.css")) st.write("![](https://upload.wikimedia.org/wikipedia/fr/e/e8/Logo_BLOOM.jpg)") st.write("## Welcome to Bloom Trawlwatcher Demo app! 🐟") diff --git a/bloom/bloom/usecase/GetVesselsFromSpire.py b/bloom/bloom/usecase/GetVesselsFromSpire.py index 8f512d1d..6b69aac0 100644 --- a/bloom/bloom/usecase/GetVesselsFromSpire.py +++ b/bloom/bloom/usecase/GetVesselsFromSpire.py @@ -10,6 +10,7 @@ from bloom.infra.http.spire_api_utils import Paging from bloom.infra.repositories.repository_vessel import RepositoryVessel from bloom.logger import logger +from bloom.config import settings class GetVesselsFromSpire: @@ -19,7 +20,7 @@ def __init__( ) -> None: self.vessel_repository: RepositoryVessel = vessel_repository - spire_token = os.environ.get("SPIRE_TOKEN") + spire_token = settings.spire_token self.transport = RequestsHTTPTransport( url="https://api.spire.com/graphql", From b55ee29f6280efd600e2b3b9d5b09b69cbbf0576 Mon Sep 17 00:00:00 2001 From: "herve.le-bars" Date: Tue, 5 Mar 2024 20:57:11 +0100 Subject: [PATCH 23/23] fix: Trawlwatcher loading data from settings.data_folder --- bloom/bloom/infra/repositories/repository_vessel.py | 2 +- bloom/pages/1_Vessel_Exploration.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bloom/bloom/infra/repositories/repository_vessel.py b/bloom/bloom/infra/repositories/repository_vessel.py index f43006ce..917a4cec 100644 --- a/bloom/bloom/infra/repositories/repository_vessel.py +++ b/bloom/bloom/infra/repositories/repository_vessel.py @@ -23,7 +23,7 @@ def __init__( session_factory: Callable, ) -> Callable[..., AbstractContextManager]: self.session_factory = session_factory - self.vessels_path = Path.joinpath(Path.cwd(), "data/chalutiers_pelagiques.csv") + self.vessels_path = Path(settings.data_folder).joinpath("./chalutiers_pelagiques.csv") def load_vessel_metadata(self) -> list[Vessel]: with self.session_factory() as session: diff --git a/bloom/pages/1_Vessel_Exploration.py b/bloom/pages/1_Vessel_Exploration.py index b4221e97..ebdd15e4 100644 --- a/bloom/pages/1_Vessel_Exploration.py +++ b/bloom/pages/1_Vessel_Exploration.py @@ -2,6 +2,7 @@ import streamlit as st from dotenv import load_dotenv from streamlit_folium import st_folium +from pathlib import Path from bloom.config import settings from bloom.infra.database.database_manager import Database @@ -31,7 +32,7 @@ def load_trajectory(mmsi, mpa): # @st.cache_data # 👈 Add the caching decorator def load_trawlers(): - trawlers = pd.read_csv("data/chalutiers_pelagiques.csv", sep=";") + trawlers = pd.read_csv(Path(settings.data_folder).joinpath("./chalutiers_pelagiques.csv"), sep=";") return trawlers