Feature/SK-545 | Statestore init from api-server lack proper setup

docker-compose.yaml

@@ -85,6 +85,7 @@ services:
       - PROJECT=project
       - FLASK_DEBUG=1
       - STATESTORE_CONFIG=/app/config/settings-reducer.yaml
+      - MODELSTORAGE_CONFIG=/app/config/settings-reducer.yaml
     build:
       context: .
       args:

fedn/cli/run_cmd.py

@@ -190,7 +190,7 @@ def dashboard_cmd(ctx, host, port, secret_key, local_package, name, init):
     statestore_config = fedn_config['statestore']
     if statestore_config['type'] == 'MongoDB':
         statestore = MongoStateStore(
-            network_id, statestore_config['mongo_config'], defaults=config['init'])
+            network_id, statestore_config['mongo_config'], fedn_config['storage'])
     else:
         print("Unsupported statestore type, exiting. ", flush=True)
         exit(-1)

fedn/fedn/common/config.py

@@ -2,18 +2,33 @@
 
 import yaml
 
-STATESTORE_CONFIG = os.environ.get('STATESTORE_CONFIG', '/workspaces/fedn/config/settings-reducer.yaml.template')
-MODELSTORAGE_CONFIG = os.environ.get('MODELSTORAGE_CONFIG', '/workspaces/fedn/config/settings-reducer.yaml.template')
+global STATESTORE_CONFIG
+global MODELSTORAGE_CONFIG
 
 
-def get_statestore_config(file=STATESTORE_CONFIG):
+def get_environment_config():
+    """ Get the configuration from environment variables.
+    """
+    global STATESTORE_CONFIG
+    global MODELSTORAGE_CONFIG
+
+    STATESTORE_CONFIG = os.environ.get('STATESTORE_CONFIG',
+                                       '/workspaces/fedn/config/settings-reducer.yaml.template')
+    MODELSTORAGE_CONFIG = os.environ.get('MODELSTORAGE_CONFIG',
+                                         '/workspaces/fedn/config/settings-reducer.yaml.template')
+
+
+def get_statestore_config(file=None):
     """ Get the statestore configuration from file.
 
-    :param file: The statestore configuration file (yaml) path.
+    :param file: The statestore configuration file (yaml) path (optional).
     :type file: str
     :return: The statestore configuration as a dict.
     :rtype: dict
     """
+    if file is None:
+        get_environment_config()
+        file = STATESTORE_CONFIG
     with open(file, 'r') as config_file:
         try:
             settings = dict(yaml.safe_load(config_file))
@@ -22,14 +37,17 @@ def get_statestore_config(file=STATESTORE_CONFIG):
     return settings["statestore"]
 
 
-def get_modelstorage_config(file=MODELSTORAGE_CONFIG):
+def get_modelstorage_config(file=None):
     """ Get the model storage configuration from file.
 
-    :param file: The model storage configuration file (yaml) path.
+    :param file: The model storage configuration file (yaml) path (optional).
     :type file: str
     :return: The model storage configuration as a dict.
     :rtype: dict
     """
+    if file is None:
+        get_environment_config()
+        file = MODELSTORAGE_CONFIG
     with open(file, 'r') as config_file:
         try:
             settings = dict(yaml.safe_load(config_file))
@@ -38,14 +56,17 @@ def get_modelstorage_config(file=MODELSTORAGE_CONFIG):
     return settings["storage"]
 
 
-def get_network_config(file=STATESTORE_CONFIG):
+def get_network_config(file=None):
     """ Get the network configuration from file.
 
-    :param file: The network configuration file (yaml) path.
+    :param file: The network configuration file (yaml) path (optional).
     :type file: str
     :return: The network id.
     :rtype: str
     """
+    if file is None:
+        get_environment_config()
+        file = STATESTORE_CONFIG
     with open(file, 'r') as config_file:
         try:
             settings = dict(yaml.safe_load(config_file))
@@ -54,14 +75,17 @@ def get_network_config(file=STATESTORE_CONFIG):
     return settings["network_id"]
 
 
-def get_controller_config(file=STATESTORE_CONFIG):
+def get_controller_config(file=None):
     """ Get the controller configuration from file.
 
-    :param file: The controller configuration file (yaml) path.
+    :param file: The controller configuration file (yaml) path (optional).
     :type file: str
     :return: The controller configuration as a dict.
     :rtype: dict
     """
+    if file is None:
+        get_environment_config()
+        file = STATESTORE_CONFIG
     with open(file, 'r') as config_file:
         try:
             settings = dict(yaml.safe_load(config_file))

fedn/fedn/network/api/server.py

@@ -1,16 +1,18 @@
 from flask import Flask, jsonify, request
 
-from fedn.common.config import (get_controller_config, get_network_config,
-                                get_statestore_config)
+from fedn.common.config import (get_controller_config, get_modelstorage_config,
+                                get_network_config, get_statestore_config)
 from fedn.network.api.interface import API
 from fedn.network.controller.control import Control
 from fedn.network.statestore.mongostatestore import MongoStateStore
 
 statestore_config = get_statestore_config()
 network_id = get_network_config()
+modelstorage_config = get_modelstorage_config()
 statestore = MongoStateStore(
     network_id,
-    statestore_config['mongo_config']
+    statestore_config['mongo_config'],
+    modelstorage_config
 )
 control = Control(statestore=statestore)
 api = API(statestore, control)

fedn/fedn/network/controller/controlbase.py

@@ -10,6 +10,9 @@
 from fedn.network.combiner.interfaces import CombinerUnavailableError
 from fedn.network.state import ReducerState
 
+# Maximum number of tries to connect to statestore and retrieve storage configuration
+MAX_TRIES_BACKEND = os.getenv('MAX_TRIES_BACKEND', 10)
+
 
 class UnsupportedStorageBackend(Exception):
     pass
@@ -42,12 +45,18 @@ def __init__(self, statestore):
 
         try:
             not_ready = True
+            tries = 0
             while not_ready:
                 storage_config = self.statestore.get_storage_backend()
                 if storage_config:
                     not_ready = False
                 else:
+                    print(
+                        "REDUCER CONTROL: Storage backend not configured, waiting...", flush=True)
                     sleep(5)
+                    tries += 1
+                    if tries > MAX_TRIES_BACKEND:
+                        raise Exception
         except Exception:
             print(
                 "REDUCER CONTROL: Failed to retrive storage configuration, exiting.", flush=True)

fedn/fedn/network/statestore/mongostatestore.py

@@ -2,7 +2,6 @@
 from datetime import datetime
 
 import pymongo
-import yaml
 
 from fedn.common.storage.db.mongo import connect_to_mongodb
 from fedn.network.state import ReducerStateToString, StringToReducerState
@@ -21,7 +20,7 @@ class MongoStateStore(StateStoreBase):
     :type defaults: dict
     """
 
-    def __init__(self, network_id, config, defaults=None):
+    def __init__(self, network_id, config, model_storage_config):
         """ Constructor."""
         self.__inited = False
         try:
@@ -58,56 +57,9 @@ def __init__(self, network_id, config, defaults=None):
             self.clients = None
             raise
 
-        if defaults:
-            with open(defaults, 'r') as file:
-                try:
-                    settings = dict(yaml.safe_load(file))
-                    print(settings, flush=True)
-
-                    # Control settings
-                    if "control" in settings and settings["control"]:
-                        control = settings['control']
-                        try:
-                            self.transition(str(control['state']))
-                        except KeyError:
-                            self.transition("idle")
-
-                        if "model" in control:
-                            if not self.get_latest():
-                                self.set_latest(str(control['model']))
-                            else:
-                                print(
-                                    "Model trail already initialized - refusing to overwrite from config. Purge model trail if you want to reseed the system.",
-                                    flush=True)
-
-                        if "context" in control:
-                            print("Setting filepath to {}".format(
-                                control['context']), flush=True)
-                            # TODO Fix the ugly latering of indirection due to a bug in secure_filename returning an object with filename as attribute
-                            # TODO fix with unboxing of value before storing and where consuming.
-                            self.control.config.update_one({'key': 'package'},
-                                                           {'$set': {'filename': control['context']}}, True)
-                        if "helper" in control:
-                            # self.set_framework(control['helper'])
-                            pass
-
-                        round_config = {'timeout': 180, 'validate': True}
-                        try:
-                            round_config['timeout'] = control['timeout']
-                        except Exception:
-                            pass
-
-                        try:
-                            round_config['validate'] = control['validate']
-                        except Exception:
-                            pass
-
-                    # Storage settings
-                    self.set_storage_backend(settings['storage'])
-
-                    self.__inited = True
-                except yaml.YAMLError as e:
-                    print(e)
+        # Storage settings
+        self.set_storage_backend(model_storage_config)
+        self.__inited = True
 
     def is_inited(self):
         """ Check if the statestore is intialized.