From 3d92b77795459b315725585aa24636c7bae4a03e Mon Sep 17 00:00:00 2001 From: Fredrik Wrede Date: Wed, 2 Oct 2024 15:46:37 +0000 Subject: [PATCH 01/17] statestore refactor + ModelInference --- fedn/cli/status_cmd.py | 3 +- fedn/network/api/v1/inference_routes.py | 10 +- fedn/network/clients/client.py | 28 ++++- fedn/network/combiner/combiner.py | 79 +++++++++--- fedn/network/combiner/roundhandler.py | 20 +-- fedn/network/combiner/shared.py | 21 ++++ fedn/network/grpc/fedn.proto | 12 ++ fedn/network/grpc/fedn_pb2.py | 114 +++++++++--------- fedn/network/grpc/fedn_pb2_grpc.py | 33 +++++ .../storage/statestore/stores/client_store.py | 21 ++-- .../statestore/stores/combiner_store.py | 41 +++---- .../statestore/stores/inference_store.py | 84 +++++++++++++ .../storage/statestore/stores/status_store.py | 18 +-- .../storage/statestore/stores/store.py | 5 +- .../statestore/stores/validation_store.py | 22 +--- 15 files changed, 356 insertions(+), 155 deletions(-) create mode 100644 fedn/network/storage/statestore/stores/inference_store.py diff --git a/fedn/cli/status_cmd.py b/fedn/cli/status_cmd.py index 078acaf13..c879ca1ef 100644 --- a/fedn/cli/status_cmd.py +++ b/fedn/cli/status_cmd.py @@ -8,8 +8,7 @@ @main.group("status") @click.pass_context def status_cmd(ctx): - """:param ctx: - """ + """:param ctx:""" pass diff --git a/fedn/network/api/v1/inference_routes.py b/fedn/network/api/v1/inference_routes.py index 6da2dc8b4..460dd495f 100644 --- a/fedn/network/api/v1/inference_routes.py +++ b/fedn/network/api/v1/inference_routes.py @@ -13,19 +13,19 @@ @jwt_auth_required(role="admin") def start_session(): """Start a new inference session. - param: session_id: The session id to start. - type: session_id: str + param: inference_id: The session id to start. + type: inference_id: str param: rounds: The number of rounds to run. type: rounds: int """ try: data = request.json if request.headers["Content-Type"] == "application/json" else request.form.to_dict() - session_id: str = data.get("session_id") + inference_id: str = data.get("inference_id") - if not session_id or session_id == "": + if not inference_id or inference_id == "": return jsonify({"message": "Session ID is required"}), 400 - session_config = {"session_id": session_id} + session_config = {"inference_id": inference_id} threading.Thread(target=control.inference_session, kwargs={"config": session_config}).start() diff --git a/fedn/network/clients/client.py b/fedn/network/clients/client.py index fa7000a99..97574e7b2 100644 --- a/fedn/network/clients/client.py +++ b/fedn/network/clients/client.py @@ -176,7 +176,7 @@ def connect(self, combiner_config): logger.debug("Client using metadata: {}.".format(self.metadata)) port = combiner_config["port"] secure = False - if combiner_config["fqdn"] is not None: + if "fqdn" in combiner_config.keys() and combiner_config["fqdn"] is not None: host = combiner_config["fqdn"] # assuming https if fqdn is used port = 443 @@ -721,7 +721,33 @@ def process_request(self): self.state = ClientState.idle continue presigned_url = presigned_url["presigned_url"] + # Obs that session_id in request is the inference_id _ = self._process_inference_request(request.model_id, request.session_id, presigned_url) + inference = fedn.ModelInference() + inference.sender.name = self.name + inference.sender.role = fedn.WORKER + inference.receiver.name = request.sender.name + inference.receiver.name = request.sender.name + inference.receiver.role = request.sender.role + inference.model_id = str(request.model_id) + # TODO: Add inference data + inference.data = "" + inference.timestamp.GetCurrentTime() + inference.correlation_id = request.correlation_id + # Obs that session_id in request is the inference_id + inference.inference_id = request.session_id + + try: + _ = self.combinerStub.SendModelInference(inference, metadata=self.metadata) + status_type = fedn.StatusType.INFERENCE + self.send_status( + "Model inference completed.", log_level=fedn.Status.AUDIT, type=status_type, request=inference, sesssion_id=request.session_id + ) + except grpc.RpcError as e: + status_code = e.code() + logger.error("GRPC error, {}.".format(status_code.name)) + logger.debug(e) + self.state = ClientState.idle except queue.Empty: pass diff --git a/fedn/network/combiner/combiner.py b/fedn/network/combiner/combiner.py index d336932c5..c4abd5290 100644 --- a/fedn/network/combiner/combiner.py +++ b/fedn/network/combiner/combiner.py @@ -9,12 +9,14 @@ from enum import Enum from typing import TypedDict +from google.protobuf.json_format import MessageToDict + import fedn.network.grpc.fedn_pb2 as fedn import fedn.network.grpc.fedn_pb2_grpc as rpc from fedn.common.certificate.certificate import Certificate from fedn.common.log_config import logger, set_log_level_from_string, set_log_stream from fedn.network.combiner.roundhandler import RoundConfig, RoundHandler -from fedn.network.combiner.shared import repository, statestore +from fedn.network.combiner.shared import client_store, combiner_store, inference_store, repository, statestore, status_store, validation_store from fedn.network.grpc.server import Server, ServerConfig VALID_NAME_REGEX = "^[a-zA-Z0-9_-]*$" @@ -107,18 +109,20 @@ def __init__(self, config): "parent": "localhost", "ip": "", } - self.statestore.set_combiner(interface_config) + combiner_store.add(interface_config) # Fetch all clients previously connected to the combiner # If a client and a combiner goes down at the same time, # the client will be stuck listed as "online" in the statestore. # Set the status to offline for previous clients. - previous_clients = self.statestore.clients.find({"combiner": config["name"]}) + previous_clients = client_store.list(limit=0, skip=0, sort_key=None, kwargs={"combiner": config["name"]}) + previous_clients = previous_clients["result"] for client in previous_clients: try: - self.statestore.set_client({"name": client["name"], "status": "offline", "client_id": client["client_id"]}) + client_store.update(client["_id"], {"name": client["name"], "status": "offline", "client_id": client["client_id"]}) except KeyError: - self.statestore.set_client({"name": client["name"], "status": "offline"}) + # Old clients might not have a client_id + client_store.update(client["_id"], {"name": client["name"], "status": "offline"}) # Set up gRPC server configuration if config["secure"]: @@ -191,7 +195,7 @@ def request_model_validation(self, session_id, model_id, clients=[]): else: logger.info("Sent model validation request for model {} to {} clients".format(model_id, len(clients))) - def request_model_inference(self, session_id: str, model_id: str, clients: list = []) -> None: + def request_model_inference(self, inference_id: str, model_id: str, clients: list = []) -> None: """Ask clients to perform inference on the model. :param model_id: the model id to perform inference on @@ -202,7 +206,7 @@ def request_model_inference(self, session_id: str, model_id: str, clients: list :type clients: list """ - clients = self._send_request_type(fedn.StatusType.INFERENCE, session_id, model_id, clients) + clients = self._send_request_type(fedn.StatusType.INFERENCE, inference_id, model_id, clients) if len(clients) < 20: logger.info("Sent model inference request for model {} to clients {}".format(model_id, clients)) @@ -214,6 +218,8 @@ def _send_request_type(self, request_type, session_id, model_id, config=None, cl :param request_type: the type of request :type request_type: :class:`fedn.network.grpc.fedn_pb2.StatusType` + :param session_id: the session id to send in the request. Obs that for inference, this is the inference id. + :type session_id: str :param model_id: the model id to send in the request :type model_id: str :param config: the model configuration to send to clients @@ -354,7 +360,7 @@ def _list_active_clients(self, channel): then = self.clients[client]["last_seen"] if (now - then) < timedelta(seconds=10): clients["active_clients"].append(client) - # If client has changed status, update statestore + # If client has changed status, update client queue if status != "online": self.clients[client]["status"] = "online" clients["update_active_clients"].append(client) @@ -363,9 +369,11 @@ def _list_active_clients(self, channel): clients["update_offline_clients"].append(client) # Update statestore with client status if len(clients["update_active_clients"]) > 0: - self.statestore.update_client_status(clients["update_active_clients"], "online") + for client in clients["update_active_clients"]: + client_store.update(self.clients[client]["_id"], {"status": "online"}) if len(clients["update_offline_clients"]) > 0: - self.statestore.update_client_status(clients["update_offline_clients"], "offline") + for client in clients["update_offline_clients"]: + client_store.update(self.clients[client]["_id"], {"status": "offline"}) return clients["active_clients"] @@ -395,10 +403,11 @@ def _put_request_to_client_queue(self, request, queue_name): def _send_status(self, status): """Report a status to backend db. - :param status: the status to report + :param status: the status message to report :type status: :class:`fedn.network.grpc.fedn_pb2.Status` """ - self.statestore.report_status(status) + data = MessageToDict(status, including_default_value_fields=True) + _ = status_store.add(data) def _flush_model_update_queue(self): """Clear the model update queue (aggregator). @@ -627,6 +636,7 @@ def TaskStream(self, response, context): self.__whoami(status.sender, self) + # Subscribe client, this also adds the client to self.clients self._subscribe_client_to_queue(client, fedn.Queue.TASK_QUEUE) q = self.__get_queue(client, fedn.Queue.TASK_QUEUE) @@ -634,7 +644,21 @@ def TaskStream(self, response, context): # Set client status to online self.clients[client.client_id]["status"] = "online" - self.statestore.set_client({"name": client.name, "status": "online", "client_id": client.client_id, "last_seen": datetime.now()}) + try: + # If the client is already in the client store, update the status + success, result = client_store.update( + self.clients[client.client_id]["_id"], {"name": client.name, "status": "online", "client_id": client.client_id, "last_seen": datetime.now()} + ) + if not success: + logger.error(result) + except KeyError: + # If the client is not in the client store, add the client + success, result = client_store.add({"name": client.name, "status": "online", "client_id": client.client_id, "last_seen": datetime.now()}) + # Get the _id of the client and add it to the clients dict + if success: + self.clients[client.client_id]["_id"] = result["id"] + else: + logger.error(result) # Keep track of the time context has been active start_time = time.time() @@ -673,7 +697,12 @@ def register_model_validation(self, validation): :param validation: the model validation :type validation: :class:`fedn.network.grpc.fedn_pb2.ModelValidation` """ - self.statestore.report_validation(validation) + data = MessageToDict(validation, including_default_value_fields=True) + success, result = validation_store.add(data) + if not success: + logger.error(result) + else: + logger.info("Model validation registered: {}".format(result)) def SendModelValidation(self, request, context): """Send a model validation response. @@ -687,12 +716,32 @@ def SendModelValidation(self, request, context): """ logger.info("Recieved ModelValidation from {}".format(request.sender.name)) - self.register_model_validation(request) + validation = MessageToDict(request, including_default_value_fields=True) + validation_store.add(validation) response = fedn.Response() response.response = "RECEIVED ModelValidation {} from client {}".format(response, response.sender.name) return response + def SendModelInference(self, request, context): + """Send a model inference response. + + :param request: the request + :type request: :class:`fedn.network.grpc.fedn_pb2.ModelInference` + :param context: the context + :type context: :class:`grpc._server._Context` + :return: the response + :rtype: :class:`fedn.network.grpc.fedn_pb2.Response` + """ + logger.info("Recieved ModelInference from {}".format(request.sender.name)) + + result = MessageToDict(request, including_default_value_fields=True) + inference_store.add(result) + + response = fedn.Response() + response.response = "RECEIVED ModelInference {} from client {}".format(response, response.sender.name) + return response + #################################################################################################################### def run(self): diff --git a/fedn/network/combiner/roundhandler.py b/fedn/network/combiner/roundhandler.py index 1f0025303..64d793318 100644 --- a/fedn/network/combiner/roundhandler.py +++ b/fedn/network/combiner/roundhandler.py @@ -42,6 +42,8 @@ class RoundConfig(TypedDict): :type model_metadata: dict :param session_id: The session identifier. Set by (Controller?). :type session_id: str + :param inference_id: The inference identifier. Only used for inference tasks. + :type inference_id: str :param helper_type: The helper type. :type helper_type: str :param aggregator: The aggregator type. @@ -250,7 +252,7 @@ def _validation_round(self, session_id, model_id, clients): """ self.server.request_model_validation(session_id, model_id, clients=clients) - def _inference_round(self, session_id: str, model_id: str, clients: list): + def _inference_round(self, inference_id: str, model_id: str, clients: list): """Send model inference requests to clients. :param config: The round config object (passed to the client). @@ -260,7 +262,7 @@ def _inference_round(self, session_id: str, model_id: str, clients: list): :param model_id: The ID of the model to use for inference :type model_id: str """ - self.server.request_model_inference(session_id, model_id, clients=clients) + self.server.request_model_inference(inference_id, model_id, clients=clients) def stage_model(self, model_id, timeout_retry=3, retry=2): """Download a model from persistent storage and set in modelservice. @@ -348,7 +350,7 @@ def execute_validation_round(self, session_id, model_id): validators = self._assign_round_clients(self.server.max_clients, type="validators") self._validation_round(session_id, model_id, validators) - def execute_inference_round(self, session_id: str, model_id: str) -> None: + def execute_inference_round(self, inference_id: str, model_id: str) -> None: """Coordinate inference rounds as specified in config. :param round_config: The round config object. @@ -358,7 +360,7 @@ def execute_inference_round(self, session_id: str, model_id: str) -> None: self.stage_model(model_id) # TODO: Implement inference client type clients = self._assign_round_clients(self.server.max_clients, type="validators") - self._inference_round(session_id, model_id, clients) + self._inference_round(inference_id, model_id, clients) def execute_training_round(self, config): """Coordinates clients to execute training tasks. @@ -407,8 +409,6 @@ def run(self, polling_interval=1.0): while True: try: round_config = self.round_configs.get(block=False) - session_id = round_config["session_id"] - model_id = round_config["model_id"] # Check that the minimum allowed number of clients are connected ready = self._check_nr_round_clients(round_config) @@ -416,6 +416,8 @@ def run(self, polling_interval=1.0): if ready: if round_config["task"] == "training": + session_id = round_config["session_id"] + model_id = round_config["model_id"] tic = time.time() round_meta = self.execute_training_round(round_config) round_meta["time_exec_training"] = time.time() - tic @@ -423,9 +425,13 @@ def run(self, polling_interval=1.0): round_meta["name"] = self.server.id self.server.statestore.set_round_combiner_data(round_meta) elif round_config["task"] == "validation": + session_id = round_config["session_id"] + model_id = round_config["model_id"] self.execute_validation_round(session_id, model_id) elif round_config["task"] == "inference": - self.execute_inference_round(session_id, model_id) + inference_id = round_config["inference_id"] + model_id = round_config["model_id"] + self.execute_inference_round(inference_id, model_id) else: logger.warning("config contains unkown task type.") else: diff --git a/fedn/network/combiner/shared.py b/fedn/network/combiner/shared.py index e1dd6854f..4ecb34ab4 100644 --- a/fedn/network/combiner/shared.py +++ b/fedn/network/combiner/shared.py @@ -2,12 +2,33 @@ from fedn.network.combiner.modelservice import ModelService from fedn.network.storage.s3.repository import Repository from fedn.network.storage.statestore.mongostatestore import MongoStateStore +from fedn.network.storage.statestore.stores.client_store import ClientStore +from fedn.network.storage.statestore.stores.combiner_store import CombinerStore +from fedn.network.storage.statestore.stores.inference_store import InferenceStore +from fedn.network.storage.statestore.stores.status_store import StatusStore +from fedn.network.storage.statestore.stores.validation_store import ValidationStore statestore_config = get_statestore_config() modelstorage_config = get_modelstorage_config() network_id = get_network_config() statestore = MongoStateStore(network_id, statestore_config["mongo_config"]) + +if statestore_config["type"] == "MongoDB": + # floating import + import pymongo + from pymongo.database import Database + + mc = pymongo.MongoClient(**statestore_config["mongo_config"]) + mc.server_info() + mdb: Database = mc[network_id] + +client_store = ClientStore(mdb, "network.clients") +validation_store = ValidationStore(mdb, "network.validations") +combiner_store = CombinerStore(mdb, "network.combiners") +status_store = StatusStore(mdb, "network.status") +inference_store = InferenceStore(mdb, "network.inferences") + repository = Repository(modelstorage_config["storage_config"], init_buckets=False) modelservice = ModelService() diff --git a/fedn/network/grpc/fedn.proto b/fedn/network/grpc/fedn.proto index 04a1e5175..03009c2d9 100644 --- a/fedn/network/grpc/fedn.proto +++ b/fedn/network/grpc/fedn.proto @@ -80,6 +80,17 @@ message ModelValidation { string session_id = 8; } +message ModelInference { + Client sender = 1; + Client receiver = 2; + string model_id = 3; + string data = 4; + string correlation_id = 5; + google.protobuf.Timestamp timestamp = 6; + string meta = 7; + string inference_id = 8; +} + enum ModelStatus { OK = 0; IN_PROGRESS = 1; @@ -241,6 +252,7 @@ service Combiner { rpc SendModelUpdate (ModelUpdate) returns (Response); rpc SendModelValidation (ModelValidation) returns (Response); + rpc SendModelInference (ModelInference) returns (Response); } diff --git a/fedn/network/grpc/fedn_pb2.py b/fedn/network/grpc/fedn_pb2.py index d47f08792..92233c9b8 100644 --- a/fedn/network/grpc/fedn_pb2.py +++ b/fedn/network/grpc/fedn_pb2.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: network/grpc/fedn.proto +# Protobuf Python Version: 4.25.0 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -14,26 +15,25 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17network/grpc/fedn.proto\x12\x04\x66\x65\x64n\x1a\x1fgoogle/protobuf/timestamp.proto\":\n\x08Response\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08response\x18\x02 \x01(\t\"\xbc\x02\n\x06Status\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06status\x18\x02 \x01(\t\x12(\n\tlog_level\x18\x03 \x01(\x0e\x32\x15.fedn.Status.LogLevel\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1e\n\x04type\x18\x07 \x01(\x0e\x32\x10.fedn.StatusType\x12\r\n\x05\x65xtra\x18\x08 \x01(\t\x12\x12\n\nsession_id\x18\t \x01(\t\"B\n\x08LogLevel\x12\x08\n\x04INFO\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\t\n\x05\x41UDIT\x10\x04\"\xd8\x01\n\x0bTaskRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\x12\x1e\n\x04type\x18\t \x01(\x0e\x32\x10.fedn.StatusType\"\xbf\x01\n\x0bModelUpdate\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x17\n\x0fmodel_update_id\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x08 \x01(\t\"\xd8\x01\n\x0fModelValidation\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\"\x89\x01\n\x0cModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\n\n\x02id\x18\x04 \x01(\t\x12!\n\x06status\x18\x05 \x01(\x0e\x32\x11.fedn.ModelStatus\"]\n\rModelResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\x12!\n\x06status\x18\x03 \x01(\x0e\x32\x11.fedn.ModelStatus\x12\x0f\n\x07message\x18\x04 \x01(\t\"U\n\x15GetGlobalModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\"h\n\x16GetGlobalModelResponse\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\")\n\tHeartbeat\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\"W\n\x16\x43lientAvailableMessage\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\t\"P\n\x12ListClientsRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1c\n\x07\x63hannel\x18\x02 \x01(\x0e\x32\x0b.fedn.Queue\"*\n\nClientList\x12\x1c\n\x06\x63lient\x18\x01 \x03(\x0b\x32\x0c.fedn.Client\"C\n\x06\x43lient\x12\x18\n\x04role\x18\x01 \x01(\x0e\x32\n.fedn.Role\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x11\n\tclient_id\x18\x03 \x01(\t\"m\n\x0fReassignRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06server\x18\x03 \x01(\t\x12\x0c\n\x04port\x18\x04 \x01(\r\"c\n\x10ReconnectRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x11\n\treconnect\x18\x03 \x01(\r\"\'\n\tParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"T\n\x0e\x43ontrolRequest\x12\x1e\n\x07\x63ommand\x18\x01 \x01(\x0e\x32\r.fedn.Command\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"F\n\x0f\x43ontrolResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"\x13\n\x11\x43onnectionRequest\"<\n\x12\x43onnectionResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.fedn.ConnectionStatus*\x84\x01\n\nStatusType\x12\x07\n\x03LOG\x10\x00\x12\x18\n\x14MODEL_UPDATE_REQUEST\x10\x01\x12\x10\n\x0cMODEL_UPDATE\x10\x02\x12\x1c\n\x18MODEL_VALIDATION_REQUEST\x10\x03\x12\x14\n\x10MODEL_VALIDATION\x10\x04\x12\r\n\tINFERENCE\x10\x05*$\n\x05Queue\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x0e\n\nTASK_QUEUE\x10\x01*S\n\x0bModelStatus\x12\x06\n\x02OK\x10\x00\x12\x0f\n\x0bIN_PROGRESS\x10\x01\x12\x12\n\x0eIN_PROGRESS_OK\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\x0b\n\x07UNKNOWN\x10\x04*8\n\x04Role\x12\n\n\x06WORKER\x10\x00\x12\x0c\n\x08\x43OMBINER\x10\x01\x12\x0b\n\x07REDUCER\x10\x02\x12\t\n\x05OTHER\x10\x03*J\n\x07\x43ommand\x12\x08\n\x04IDLE\x10\x00\x12\t\n\x05START\x10\x01\x12\t\n\x05PAUSE\x10\x02\x12\x08\n\x04STOP\x10\x03\x12\t\n\x05RESET\x10\x04\x12\n\n\x06REPORT\x10\x05*I\n\x10\x43onnectionStatus\x12\x11\n\rNOT_ACCEPTING\x10\x00\x12\r\n\tACCEPTING\x10\x01\x12\x13\n\x0fTRY_AGAIN_LATER\x10\x02\x32z\n\x0cModelService\x12\x33\n\x06Upload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse(\x01\x12\x35\n\x08\x44ownload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse0\x01\x32\xf8\x01\n\x07\x43ontrol\x12\x34\n\x05Start\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x33\n\x04Stop\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x44\n\x15\x46lushAggregationQueue\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12<\n\rSetAggregator\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse2V\n\x07Reducer\x12K\n\x0eGetGlobalModel\x12\x1b.fedn.GetGlobalModelRequest\x1a\x1c.fedn.GetGlobalModelResponse2\xab\x03\n\tConnector\x12\x44\n\x14\x41llianceStatusStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x0c.fedn.Status0\x01\x12*\n\nSendStatus\x12\x0c.fedn.Status\x1a\x0e.fedn.Response\x12?\n\x11ListActiveClients\x12\x18.fedn.ListClientsRequest\x1a\x10.fedn.ClientList\x12\x45\n\x10\x41\x63\x63\x65ptingClients\x12\x17.fedn.ConnectionRequest\x1a\x18.fedn.ConnectionResponse\x12\x30\n\rSendHeartbeat\x12\x0f.fedn.Heartbeat\x1a\x0e.fedn.Response\x12\x37\n\x0eReassignClient\x12\x15.fedn.ReassignRequest\x1a\x0e.fedn.Response\x12\x39\n\x0fReconnectClient\x12\x16.fedn.ReconnectRequest\x1a\x0e.fedn.Response2\xbf\x01\n\x08\x43ombiner\x12?\n\nTaskStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x11.fedn.TaskRequest0\x01\x12\x34\n\x0fSendModelUpdate\x12\x11.fedn.ModelUpdate\x1a\x0e.fedn.Response\x12<\n\x13SendModelValidation\x12\x15.fedn.ModelValidation\x1a\x0e.fedn.Responseb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17network/grpc/fedn.proto\x12\x04\x66\x65\x64n\x1a\x1fgoogle/protobuf/timestamp.proto\":\n\x08Response\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08response\x18\x02 \x01(\t\"\xbc\x02\n\x06Status\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06status\x18\x02 \x01(\t\x12(\n\tlog_level\x18\x03 \x01(\x0e\x32\x15.fedn.Status.LogLevel\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1e\n\x04type\x18\x07 \x01(\x0e\x32\x10.fedn.StatusType\x12\r\n\x05\x65xtra\x18\x08 \x01(\t\x12\x12\n\nsession_id\x18\t \x01(\t\"B\n\x08LogLevel\x12\x08\n\x04INFO\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\t\n\x05\x41UDIT\x10\x04\"\xd8\x01\n\x0bTaskRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\x12\x1e\n\x04type\x18\t \x01(\x0e\x32\x10.fedn.StatusType\"\xbf\x01\n\x0bModelUpdate\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x17\n\x0fmodel_update_id\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x08 \x01(\t\"\xd8\x01\n\x0fModelValidation\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\"\xd9\x01\n\x0eModelInference\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x14\n\x0cinference_id\x18\x08 \x01(\t\"\x89\x01\n\x0cModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\n\n\x02id\x18\x04 \x01(\t\x12!\n\x06status\x18\x05 \x01(\x0e\x32\x11.fedn.ModelStatus\"]\n\rModelResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\x12!\n\x06status\x18\x03 \x01(\x0e\x32\x11.fedn.ModelStatus\x12\x0f\n\x07message\x18\x04 \x01(\t\"U\n\x15GetGlobalModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\"h\n\x16GetGlobalModelResponse\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\")\n\tHeartbeat\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\"W\n\x16\x43lientAvailableMessage\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\t\"P\n\x12ListClientsRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1c\n\x07\x63hannel\x18\x02 \x01(\x0e\x32\x0b.fedn.Queue\"*\n\nClientList\x12\x1c\n\x06\x63lient\x18\x01 \x03(\x0b\x32\x0c.fedn.Client\"C\n\x06\x43lient\x12\x18\n\x04role\x18\x01 \x01(\x0e\x32\n.fedn.Role\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x11\n\tclient_id\x18\x03 \x01(\t\"m\n\x0fReassignRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06server\x18\x03 \x01(\t\x12\x0c\n\x04port\x18\x04 \x01(\r\"c\n\x10ReconnectRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x11\n\treconnect\x18\x03 \x01(\r\"\'\n\tParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"T\n\x0e\x43ontrolRequest\x12\x1e\n\x07\x63ommand\x18\x01 \x01(\x0e\x32\r.fedn.Command\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"F\n\x0f\x43ontrolResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"\x13\n\x11\x43onnectionRequest\"<\n\x12\x43onnectionResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.fedn.ConnectionStatus*\x84\x01\n\nStatusType\x12\x07\n\x03LOG\x10\x00\x12\x18\n\x14MODEL_UPDATE_REQUEST\x10\x01\x12\x10\n\x0cMODEL_UPDATE\x10\x02\x12\x1c\n\x18MODEL_VALIDATION_REQUEST\x10\x03\x12\x14\n\x10MODEL_VALIDATION\x10\x04\x12\r\n\tINFERENCE\x10\x05*$\n\x05Queue\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x0e\n\nTASK_QUEUE\x10\x01*S\n\x0bModelStatus\x12\x06\n\x02OK\x10\x00\x12\x0f\n\x0bIN_PROGRESS\x10\x01\x12\x12\n\x0eIN_PROGRESS_OK\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\x0b\n\x07UNKNOWN\x10\x04*8\n\x04Role\x12\n\n\x06WORKER\x10\x00\x12\x0c\n\x08\x43OMBINER\x10\x01\x12\x0b\n\x07REDUCER\x10\x02\x12\t\n\x05OTHER\x10\x03*J\n\x07\x43ommand\x12\x08\n\x04IDLE\x10\x00\x12\t\n\x05START\x10\x01\x12\t\n\x05PAUSE\x10\x02\x12\x08\n\x04STOP\x10\x03\x12\t\n\x05RESET\x10\x04\x12\n\n\x06REPORT\x10\x05*I\n\x10\x43onnectionStatus\x12\x11\n\rNOT_ACCEPTING\x10\x00\x12\r\n\tACCEPTING\x10\x01\x12\x13\n\x0fTRY_AGAIN_LATER\x10\x02\x32z\n\x0cModelService\x12\x33\n\x06Upload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse(\x01\x12\x35\n\x08\x44ownload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse0\x01\x32\xf8\x01\n\x07\x43ontrol\x12\x34\n\x05Start\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x33\n\x04Stop\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x44\n\x15\x46lushAggregationQueue\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12<\n\rSetAggregator\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse2V\n\x07Reducer\x12K\n\x0eGetGlobalModel\x12\x1b.fedn.GetGlobalModelRequest\x1a\x1c.fedn.GetGlobalModelResponse2\xab\x03\n\tConnector\x12\x44\n\x14\x41llianceStatusStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x0c.fedn.Status0\x01\x12*\n\nSendStatus\x12\x0c.fedn.Status\x1a\x0e.fedn.Response\x12?\n\x11ListActiveClients\x12\x18.fedn.ListClientsRequest\x1a\x10.fedn.ClientList\x12\x45\n\x10\x41\x63\x63\x65ptingClients\x12\x17.fedn.ConnectionRequest\x1a\x18.fedn.ConnectionResponse\x12\x30\n\rSendHeartbeat\x12\x0f.fedn.Heartbeat\x1a\x0e.fedn.Response\x12\x37\n\x0eReassignClient\x12\x15.fedn.ReassignRequest\x1a\x0e.fedn.Response\x12\x39\n\x0fReconnectClient\x12\x16.fedn.ReconnectRequest\x1a\x0e.fedn.Response2\xfb\x01\n\x08\x43ombiner\x12?\n\nTaskStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x11.fedn.TaskRequest0\x01\x12\x34\n\x0fSendModelUpdate\x12\x11.fedn.ModelUpdate\x1a\x0e.fedn.Response\x12<\n\x13SendModelValidation\x12\x15.fedn.ModelValidation\x1a\x0e.fedn.Response\x12:\n\x12SendModelInference\x12\x14.fedn.ModelInference\x1a\x0e.fedn.Responseb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'network.grpc.fedn_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None - _globals['_STATUSTYPE']._serialized_start=2327 - _globals['_STATUSTYPE']._serialized_end=2459 - _globals['_QUEUE']._serialized_start=2461 - _globals['_QUEUE']._serialized_end=2497 - _globals['_MODELSTATUS']._serialized_start=2499 - _globals['_MODELSTATUS']._serialized_end=2582 - _globals['_ROLE']._serialized_start=2584 - _globals['_ROLE']._serialized_end=2640 - _globals['_COMMAND']._serialized_start=2642 - _globals['_COMMAND']._serialized_end=2716 - _globals['_CONNECTIONSTATUS']._serialized_start=2718 - _globals['_CONNECTIONSTATUS']._serialized_end=2791 + _globals['_STATUSTYPE']._serialized_start=2547 + _globals['_STATUSTYPE']._serialized_end=2679 + _globals['_QUEUE']._serialized_start=2681 + _globals['_QUEUE']._serialized_end=2717 + _globals['_MODELSTATUS']._serialized_start=2719 + _globals['_MODELSTATUS']._serialized_end=2802 + _globals['_ROLE']._serialized_start=2804 + _globals['_ROLE']._serialized_end=2860 + _globals['_COMMAND']._serialized_start=2862 + _globals['_COMMAND']._serialized_end=2936 + _globals['_CONNECTIONSTATUS']._serialized_start=2938 + _globals['_CONNECTIONSTATUS']._serialized_end=3011 _globals['_RESPONSE']._serialized_start=66 _globals['_RESPONSE']._serialized_end=124 _globals['_STATUS']._serialized_start=127 @@ -46,46 +46,48 @@ _globals['_MODELUPDATE']._serialized_end=856 _globals['_MODELVALIDATION']._serialized_start=859 _globals['_MODELVALIDATION']._serialized_end=1075 - _globals['_MODELREQUEST']._serialized_start=1078 - _globals['_MODELREQUEST']._serialized_end=1215 - _globals['_MODELRESPONSE']._serialized_start=1217 - _globals['_MODELRESPONSE']._serialized_end=1310 - _globals['_GETGLOBALMODELREQUEST']._serialized_start=1312 - _globals['_GETGLOBALMODELREQUEST']._serialized_end=1397 - _globals['_GETGLOBALMODELRESPONSE']._serialized_start=1399 - _globals['_GETGLOBALMODELRESPONSE']._serialized_end=1503 - _globals['_HEARTBEAT']._serialized_start=1505 - _globals['_HEARTBEAT']._serialized_end=1546 - _globals['_CLIENTAVAILABLEMESSAGE']._serialized_start=1548 - _globals['_CLIENTAVAILABLEMESSAGE']._serialized_end=1635 - _globals['_LISTCLIENTSREQUEST']._serialized_start=1637 - _globals['_LISTCLIENTSREQUEST']._serialized_end=1717 - _globals['_CLIENTLIST']._serialized_start=1719 - _globals['_CLIENTLIST']._serialized_end=1761 - _globals['_CLIENT']._serialized_start=1763 - _globals['_CLIENT']._serialized_end=1830 - _globals['_REASSIGNREQUEST']._serialized_start=1832 - _globals['_REASSIGNREQUEST']._serialized_end=1941 - _globals['_RECONNECTREQUEST']._serialized_start=1943 - _globals['_RECONNECTREQUEST']._serialized_end=2042 - _globals['_PARAMETER']._serialized_start=2044 - _globals['_PARAMETER']._serialized_end=2083 - _globals['_CONTROLREQUEST']._serialized_start=2085 - _globals['_CONTROLREQUEST']._serialized_end=2169 - _globals['_CONTROLRESPONSE']._serialized_start=2171 - _globals['_CONTROLRESPONSE']._serialized_end=2241 - _globals['_CONNECTIONREQUEST']._serialized_start=2243 - _globals['_CONNECTIONREQUEST']._serialized_end=2262 - _globals['_CONNECTIONRESPONSE']._serialized_start=2264 - _globals['_CONNECTIONRESPONSE']._serialized_end=2324 - _globals['_MODELSERVICE']._serialized_start=2793 - _globals['_MODELSERVICE']._serialized_end=2915 - _globals['_CONTROL']._serialized_start=2918 - _globals['_CONTROL']._serialized_end=3166 - _globals['_REDUCER']._serialized_start=3168 - _globals['_REDUCER']._serialized_end=3254 - _globals['_CONNECTOR']._serialized_start=3257 - _globals['_CONNECTOR']._serialized_end=3684 - _globals['_COMBINER']._serialized_start=3687 - _globals['_COMBINER']._serialized_end=3878 + _globals['_MODELINFERENCE']._serialized_start=1078 + _globals['_MODELINFERENCE']._serialized_end=1295 + _globals['_MODELREQUEST']._serialized_start=1298 + _globals['_MODELREQUEST']._serialized_end=1435 + _globals['_MODELRESPONSE']._serialized_start=1437 + _globals['_MODELRESPONSE']._serialized_end=1530 + _globals['_GETGLOBALMODELREQUEST']._serialized_start=1532 + _globals['_GETGLOBALMODELREQUEST']._serialized_end=1617 + _globals['_GETGLOBALMODELRESPONSE']._serialized_start=1619 + _globals['_GETGLOBALMODELRESPONSE']._serialized_end=1723 + _globals['_HEARTBEAT']._serialized_start=1725 + _globals['_HEARTBEAT']._serialized_end=1766 + _globals['_CLIENTAVAILABLEMESSAGE']._serialized_start=1768 + _globals['_CLIENTAVAILABLEMESSAGE']._serialized_end=1855 + _globals['_LISTCLIENTSREQUEST']._serialized_start=1857 + _globals['_LISTCLIENTSREQUEST']._serialized_end=1937 + _globals['_CLIENTLIST']._serialized_start=1939 + _globals['_CLIENTLIST']._serialized_end=1981 + _globals['_CLIENT']._serialized_start=1983 + _globals['_CLIENT']._serialized_end=2050 + _globals['_REASSIGNREQUEST']._serialized_start=2052 + _globals['_REASSIGNREQUEST']._serialized_end=2161 + _globals['_RECONNECTREQUEST']._serialized_start=2163 + _globals['_RECONNECTREQUEST']._serialized_end=2262 + _globals['_PARAMETER']._serialized_start=2264 + _globals['_PARAMETER']._serialized_end=2303 + _globals['_CONTROLREQUEST']._serialized_start=2305 + _globals['_CONTROLREQUEST']._serialized_end=2389 + _globals['_CONTROLRESPONSE']._serialized_start=2391 + _globals['_CONTROLRESPONSE']._serialized_end=2461 + _globals['_CONNECTIONREQUEST']._serialized_start=2463 + _globals['_CONNECTIONREQUEST']._serialized_end=2482 + _globals['_CONNECTIONRESPONSE']._serialized_start=2484 + _globals['_CONNECTIONRESPONSE']._serialized_end=2544 + _globals['_MODELSERVICE']._serialized_start=3013 + _globals['_MODELSERVICE']._serialized_end=3135 + _globals['_CONTROL']._serialized_start=3138 + _globals['_CONTROL']._serialized_end=3386 + _globals['_REDUCER']._serialized_start=3388 + _globals['_REDUCER']._serialized_end=3474 + _globals['_CONNECTOR']._serialized_start=3477 + _globals['_CONNECTOR']._serialized_end=3904 + _globals['_COMBINER']._serialized_start=3907 + _globals['_COMBINER']._serialized_end=4158 # @@protoc_insertion_point(module_scope) diff --git a/fedn/network/grpc/fedn_pb2_grpc.py b/fedn/network/grpc/fedn_pb2_grpc.py index 86ebea055..1c533953b 100644 --- a/fedn/network/grpc/fedn_pb2_grpc.py +++ b/fedn/network/grpc/fedn_pb2_grpc.py @@ -608,6 +608,11 @@ def __init__(self, channel): request_serializer=network_dot_grpc_dot_fedn__pb2.ModelValidation.SerializeToString, response_deserializer=network_dot_grpc_dot_fedn__pb2.Response.FromString, ) + self.SendModelInference = channel.unary_unary( + '/fedn.Combiner/SendModelInference', + request_serializer=network_dot_grpc_dot_fedn__pb2.ModelInference.SerializeToString, + response_deserializer=network_dot_grpc_dot_fedn__pb2.Response.FromString, + ) class CombinerServicer(object): @@ -632,6 +637,12 @@ def SendModelValidation(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def SendModelInference(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def add_CombinerServicer_to_server(servicer, server): rpc_method_handlers = { @@ -650,6 +661,11 @@ def add_CombinerServicer_to_server(servicer, server): request_deserializer=network_dot_grpc_dot_fedn__pb2.ModelValidation.FromString, response_serializer=network_dot_grpc_dot_fedn__pb2.Response.SerializeToString, ), + 'SendModelInference': grpc.unary_unary_rpc_method_handler( + servicer.SendModelInference, + request_deserializer=network_dot_grpc_dot_fedn__pb2.ModelInference.FromString, + response_serializer=network_dot_grpc_dot_fedn__pb2.Response.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( 'fedn.Combiner', rpc_method_handlers) @@ -710,3 +726,20 @@ def SendModelValidation(request, network_dot_grpc_dot_fedn__pb2.Response.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def SendModelInference(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/fedn.Combiner/SendModelInference', + network_dot_grpc_dot_fedn__pb2.ModelInference.SerializeToString, + network_dot_grpc_dot_fedn__pb2.Response.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/fedn/network/storage/statestore/stores/client_store.py b/fedn/network/storage/statestore/stores/client_store.py index c3c2e5225..f8693fb34 100644 --- a/fedn/network/storage/statestore/stores/client_store.py +++ b/fedn/network/storage/statestore/stores/client_store.py @@ -30,7 +30,7 @@ def from_dict(data: dict) -> "Client": ip=data["ip"] if "ip" in data else None, status=data["status"] if "status" in data else None, updated_at=data["updated_at"] if "updated_at" in data else None, - last_seen=data["last_seen"] if "last_seen" in data else None + last_seen=data["last_seen"] if "last_seen" in data else None, ) @@ -50,13 +50,13 @@ def get(self, id: str, use_typing: bool = False) -> Client: return Client.from_dict(response) if use_typing else response def update(self, id: str, item: Client) -> bool: - raise NotImplementedError("Update not implemented for ClientStore") + return super().update(id, item) - def add(self, item: Client)-> Tuple[bool, Any]: - raise NotImplementedError("Add not implemented for ClientStore") + def add(self, item: Client) -> Tuple[bool, Any]: + return super().add(item) def delete(self, id: str) -> bool: - kwargs = { "_id": ObjectId(id) } if ObjectId.is_valid(id) else { "client_id": id } + kwargs = {"_id": ObjectId(id)} if ObjectId.is_valid(id) else {"client_id": id} document = self.database[self.collection].find_one(kwargs) @@ -86,10 +86,7 @@ def list(self, limit: int, skip: int, sort_key: str, sort_order=pymongo.DESCENDI result = [Client.from_dict(item) for item in response["result"]] if use_typing else response["result"] - return { - "count": response["count"], - "result": result - } + return {"count": response["count"], "result": result} def count(self, **kwargs) -> int: return super().count(**kwargs) @@ -111,13 +108,13 @@ def connected_client_count(self, combiners): [ {"$match": {"combiner": {"$in": combiners}, "status": "online"}}, {"$group": {"_id": "$combiner", "count": {"$sum": 1}}}, - {"$project": {"id": "$_id", "count": 1, "_id": 0}} + {"$project": {"id": "$_id", "count": 1, "_id": 0}}, ] if len(combiners) > 0 else [ - {"$match": { "status": "online"}}, + {"$match": {"status": "online"}}, {"$group": {"_id": "$combiner", "count": {"$sum": 1}}}, - {"$project": {"id": "$_id", "count": 1, "_id": 0}} + {"$project": {"id": "$_id", "count": 1, "_id": 0}}, ] ) diff --git a/fedn/network/storage/statestore/stores/combiner_store.py b/fedn/network/storage/statestore/stores/combiner_store.py index 5fceea1b7..2ad6437ea 100644 --- a/fedn/network/storage/statestore/stores/combiner_store.py +++ b/fedn/network/storage/statestore/stores/combiner_store.py @@ -11,19 +11,19 @@ class Combiner: def __init__( - self, - id: str, - name: str, - address: str, - certificate: str, - config: dict, - fqdn: str, - ip: str, - key: str, - parent: dict, - port: int, - status: str, - updated_at: str + self, + id: str, + name: str, + address: str, + certificate: str, + config: dict, + fqdn: str, + ip: str, + key: str, + parent: dict, + port: int, + status: str, + updated_at: str, ): self.id = id self.name = name @@ -51,7 +51,7 @@ def from_dict(data: dict) -> "Combiner": parent=data["parent"] if "parent" in data else None, port=data["port"] if "port" in data else None, status=data["status"] if "status" in data else None, - updated_at=data["updated_at"] if "updated_at" in data else None + updated_at=data["updated_at"] if "updated_at" in data else None, ) @@ -82,12 +82,12 @@ def get(self, id: str, use_typing: bool = False) -> Combiner: def update(self, id: str, item: Combiner) -> bool: raise NotImplementedError("Update not implemented for CombinerStore") - def add(self, item: Combiner)-> Tuple[bool, Any]: - raise NotImplementedError("Add not implemented for CombinerStore") + def add(self, item: Combiner) -> Tuple[bool, Any]: + return super().add(item) def delete(self, id: str) -> bool: - if(ObjectId.is_valid(id)): - kwargs = { "_id": ObjectId(id)} + if ObjectId.is_valid(id): + kwargs = {"_id": ObjectId(id)} else: return False @@ -119,10 +119,7 @@ def list(self, limit: int, skip: int, sort_key: str, sort_order=pymongo.DESCENDI result = [Combiner.from_dict(item) for item in response["result"]] if use_typing else response["result"] - return { - "count": response["count"], - "result": result - } + return {"count": response["count"], "result": result} def count(self, **kwargs) -> int: return super().count(**kwargs) diff --git a/fedn/network/storage/statestore/stores/inference_store.py b/fedn/network/storage/statestore/stores/inference_store.py new file mode 100644 index 000000000..cff1a9d89 --- /dev/null +++ b/fedn/network/storage/statestore/stores/inference_store.py @@ -0,0 +1,84 @@ +from typing import Any, Dict, List, Tuple + +import pymongo +from pymongo.database import Database + +from fedn.network.storage.statestore.stores.store import Store + + +class Inference: + def __init__( + self, id: str, model_id: str, data: str, correlation_id: str, timestamp: str, inference_id: str, meta: str, sender: dict = None, receiver: dict = None + ): + self.id = id + self.model_id = model_id + self.data = data + self.correlation_id = correlation_id + self.timestamp = timestamp + self.inference_id = inference_id + self.meta = meta + self.sender = sender + self.receiver = receiver + + def from_dict(data: dict) -> "Inference": + return Inference( + id=str(data["_id"]), + model_id=data["modelId"] if "modelId" in data else None, + data=data["data"] if "data" in data else None, + correlation_id=data["correlationId"] if "correlationId" in data else None, + timestamp=data["timestamp"] if "timestamp" in data else None, + session_id=data["sessionId"] if "sessionId" in data else None, + meta=data["meta"] if "meta" in data else None, + sender=data["sender"] if "sender" in data else None, + receiver=data["receiver"] if "receiver" in data else None, + ) + + +class InferenceStore(Store[Inference]): + def __init__(self, database: Database, collection: str): + super().__init__(database, collection) + + def get(self, id: str, use_typing: bool = False) -> Inference: + """Get an entity by id + param id: The id of the entity + type: str + description: The id of the entity, can be either the id or the Inference (property) + param use_typing: Whether to return the entity as a typed object or as a dict + type: bool + return: The entity + """ + response = super().get(id, use_typing=use_typing) + return Inference.from_dict(response) if use_typing else response + + def update(self, id: str, item: Inference) -> bool: + raise NotImplementedError("Update not implemented for InferenceStore") + + def add(self, item: Inference) -> Tuple[bool, Any]: + return super().add(item) + + def delete(self, id: str) -> bool: + raise NotImplementedError("Delete not implemented for InferenceStore") + + def list(self, limit: int, skip: int, sort_key: str, sort_order=pymongo.DESCENDING, use_typing: bool = False, **kwargs) -> Dict[int, List[Inference]]: + """List entities + param limit: The maximum number of entities to return + type: int + description: The maximum number of entities to return + param skip: The number of entities to skip + type: int + description: The number of entities to skip + param sort_key: The key to sort by + type: str + description: The key to sort by + param sort_order: The order to sort by + type: pymongo.DESCENDING + description: The order to sort by + param use_typing: Whether to return the entities as typed objects or as dicts + type: bool + description: Whether to return the entities as typed objects or as dicts + return: A dictionary with the count and a list of entities + """ + response = super().list(limit, skip, sort_key or "timestamp", sort_order, use_typing=use_typing, **kwargs) + + result = [Inference.from_dict(item) for item in response["result"]] if use_typing else response["result"] + return {"count": response["count"], "result": result} diff --git a/fedn/network/storage/statestore/stores/status_store.py b/fedn/network/storage/statestore/stores/status_store.py index 3c79d2b8d..9233d0b23 100644 --- a/fedn/network/storage/statestore/stores/status_store.py +++ b/fedn/network/storage/statestore/stores/status_store.py @@ -8,17 +8,7 @@ class Status: def __init__( - self, - id: str, - status: str, - timestamp: str, - log_level: str, - data: str, - correlation_id: str, - type: str, - extra: str, - session_id: str, - sender: dict = None + self, id: str, status: str, timestamp: str, log_level: str, data: str, correlation_id: str, type: str, extra: str, session_id: str, sender: dict = None ): self.id = id self.status = status @@ -42,7 +32,7 @@ def from_dict(data: dict) -> "Status": type=data["type"] if "type" in data else None, extra=data["extra"] if "extra" in data else None, session_id=data["sessionId"] if "sessionId" in data else None, - sender=data["sender"] if "sender" in data else None + sender=data["sender"] if "sender" in data else None, ) @@ -65,8 +55,8 @@ def get(self, id: str, use_typing: bool = False) -> Status: def update(self, id: str, item: Status) -> bool: raise NotImplementedError("Update not implemented for StatusStore") - def add(self, item: Status)-> Tuple[bool, Any]: - raise NotImplementedError("Add not implemented for StatusStore") + def add(self, item: Status) -> Tuple[bool, Any]: + return super().add(item) def delete(self, id: str) -> bool: raise NotImplementedError("Delete not implemented for StatusStore") diff --git a/fedn/network/storage/statestore/stores/store.py b/fedn/network/storage/statestore/stores/store.py index eb9d8b1bb..f6a8f67e0 100644 --- a/fedn/network/storage/statestore/stores/store.py +++ b/fedn/network/storage/statestore/stores/store.py @@ -77,10 +77,7 @@ def list(self, limit: int, skip: int, sort_key: str, sort_order=pymongo.DESCENDI result = [document for document in cursor] if use_typing else [from_document(document) for document in cursor] - return { - "count": count, - "result": result - } + return {"count": count, "result": result} def count(self, **kwargs) -> int: """Count entities diff --git a/fedn/network/storage/statestore/stores/validation_store.py b/fedn/network/storage/statestore/stores/validation_store.py index 7e8548135..59b5a0730 100644 --- a/fedn/network/storage/statestore/stores/validation_store.py +++ b/fedn/network/storage/statestore/stores/validation_store.py @@ -8,16 +8,7 @@ class Validation: def __init__( - self, - id: str, - model_id: str, - data: str, - correlation_id: str, - timestamp: str, - session_id: str, - meta: str, - sender: dict = None, - receiver: dict = None + self, id: str, model_id: str, data: str, correlation_id: str, timestamp: str, session_id: str, meta: str, sender: dict = None, receiver: dict = None ): self.id = id self.model_id = model_id @@ -39,7 +30,7 @@ def from_dict(data: dict) -> "Validation": session_id=data["sessionId"] if "sessionId" in data else None, meta=data["meta"] if "meta" in data else None, sender=data["sender"] if "sender" in data else None, - receiver=data["receiver"] if "receiver" in data else None + receiver=data["receiver"] if "receiver" in data else None, ) @@ -62,8 +53,8 @@ def get(self, id: str, use_typing: bool = False) -> Validation: def update(self, id: str, item: Validation) -> bool: raise NotImplementedError("Update not implemented for ValidationStore") - def add(self, item: Validation)-> Tuple[bool, Any]: - raise NotImplementedError("Add not implemented for ValidationStore") + def add(self, item: Validation) -> Tuple[bool, Any]: + return super().add(item) def delete(self, id: str) -> bool: raise NotImplementedError("Delete not implemented for ValidationStore") @@ -90,7 +81,4 @@ def list(self, limit: int, skip: int, sort_key: str, sort_order=pymongo.DESCENDI response = super().list(limit, skip, sort_key or "timestamp", sort_order, use_typing=use_typing, **kwargs) result = [Validation.from_dict(item) for item in response["result"]] if use_typing else response["result"] - return { - "count": response["count"], - "result": result - } + return {"count": response["count"], "result": result} From 659fe8b830cdd570421806e20296e2b3f5501518 Mon Sep 17 00:00:00 2001 From: Fredrik Wrede Date: Thu, 3 Oct 2024 08:47:58 +0000 Subject: [PATCH 02/17] fix missing updated_at key --- fedn/network/combiner/combiner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fedn/network/combiner/combiner.py b/fedn/network/combiner/combiner.py index c4abd5290..bec431188 100644 --- a/fedn/network/combiner/combiner.py +++ b/fedn/network/combiner/combiner.py @@ -108,6 +108,7 @@ def __init__(self, config): "address": config["host"], "parent": "localhost", "ip": "", + "updated_at": str(datetime.now()), } combiner_store.add(interface_config) From eb749478c0daa3374fe5d5d6694b131974c7c14b Mon Sep 17 00:00:00 2001 From: Fredrik Wrede Date: Thu, 3 Oct 2024 08:49:37 +0000 Subject: [PATCH 03/17] fix floating imports --- fedn/network/combiner/shared.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fedn/network/combiner/shared.py b/fedn/network/combiner/shared.py index 4ecb34ab4..30c5ed469 100644 --- a/fedn/network/combiner/shared.py +++ b/fedn/network/combiner/shared.py @@ -1,3 +1,6 @@ +import pymongo +from pymongo.database import Database + from fedn.common.config import get_modelstorage_config, get_network_config, get_statestore_config from fedn.network.combiner.modelservice import ModelService from fedn.network.storage.s3.repository import Repository @@ -15,10 +18,6 @@ statestore = MongoStateStore(network_id, statestore_config["mongo_config"]) if statestore_config["type"] == "MongoDB": - # floating import - import pymongo - from pymongo.database import Database - mc = pymongo.MongoClient(**statestore_config["mongo_config"]) mc.server_info() mdb: Database = mc[network_id] From 4fb372e9adfee8ae81d5f2e6bdcab50886bee93d Mon Sep 17 00:00:00 2001 From: Fredrik Wrede Date: Thu, 3 Oct 2024 09:23:32 +0000 Subject: [PATCH 04/17] fix client_store objects missing combiner name --- fedn/network/combiner/combiner.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fedn/network/combiner/combiner.py b/fedn/network/combiner/combiner.py index bec431188..fcb0873c3 100644 --- a/fedn/network/combiner/combiner.py +++ b/fedn/network/combiner/combiner.py @@ -648,13 +648,16 @@ def TaskStream(self, response, context): try: # If the client is already in the client store, update the status success, result = client_store.update( - self.clients[client.client_id]["_id"], {"name": client.name, "status": "online", "client_id": client.client_id, "last_seen": datetime.now()} + self.clients[client.client_id]["_id"], + {"name": client.name, "status": "online", "client_id": client.client_id, "last_seen": datetime.now(), "combiner": self.id}, ) if not success: logger.error(result) except KeyError: # If the client is not in the client store, add the client - success, result = client_store.add({"name": client.name, "status": "online", "client_id": client.client_id, "last_seen": datetime.now()}) + success, result = client_store.add( + {"name": client.name, "status": "online", "client_id": client.client_id, "last_seen": datetime.now(), "combiner": self.id} + ) # Get the _id of the client and add it to the clients dict if success: self.clients[client.client_id]["_id"] = result["id"] From 92b07bd806577e50cba4e92a5a6e5e0c7c70ed0b Mon Sep 17 00:00:00 2001 From: Fredrik Wrede Date: Thu, 3 Oct 2024 12:46:20 +0000 Subject: [PATCH 05/17] fix --- fedn/network/combiner/combiner.py | 56 ++++++++++++------- .../storage/statestore/stores/client_store.py | 26 ++++++++- 2 files changed, 60 insertions(+), 22 deletions(-) diff --git a/fedn/network/combiner/combiner.py b/fedn/network/combiner/combiner.py index fcb0873c3..8203592a5 100644 --- a/fedn/network/combiner/combiner.py +++ b/fedn/network/combiner/combiner.py @@ -84,6 +84,12 @@ def __init__(self, config): set_log_stream(config.get("logfile", None)) # Client queues + # Each client in the dict is stored with its client_id as key, and the value is a dict with keys: + # name: str + # status: str + # last_seen: str + # fedn.Queue.TASK_QUEUE: queue.Queue + # Obs that fedn.Queue.TASK_QUEUE is just str(1) self.clients = {} # Validate combiner name @@ -116,14 +122,20 @@ def __init__(self, config): # If a client and a combiner goes down at the same time, # the client will be stuck listed as "online" in the statestore. # Set the status to offline for previous clients. - previous_clients = client_store.list(limit=0, skip=0, sort_key=None, kwargs={"combiner": config["name"]}) + previous_clients = client_store.list(limit=0, skip=0, sort_key=None, kwargs={"combiner": self.id}) + logger.info(f"Found {previous_clients["count"]} previous clients") + logger.info("Updating previous clients status to offline") previous_clients = previous_clients["result"] for client in previous_clients: try: - client_store.update(client["_id"], {"name": client["name"], "status": "offline", "client_id": client["client_id"]}) - except KeyError: - # Old clients might not have a client_id - client_store.update(client["_id"], {"name": client["name"], "status": "offline"}) + if "client_id" in client.keys(): + client_store.update("client_id", client["client_id"], {"name": client["name"], "status": "offline"}) + else: + # Old clients might not have a client_id + client_store.update("name", client["name"], {"name": client["name"], "status": "offline"}) + + except Exception as e: + logger.error("Failed to update previous client status: {}".format(str(e))) # Set up gRPC server configuration if config["secure"]: @@ -371,10 +383,10 @@ def _list_active_clients(self, channel): # Update statestore with client status if len(clients["update_active_clients"]) > 0: for client in clients["update_active_clients"]: - client_store.update(self.clients[client]["_id"], {"status": "online"}) + client_store.update("client_id", client, {"status": "online"}) if len(clients["update_offline_clients"]) > 0: for client in clients["update_offline_clients"]: - client_store.update(self.clients[client]["_id"], {"status": "offline"}) + client_store.update("client_id", client, {"status": "offline"}) return clients["active_clients"] @@ -648,21 +660,27 @@ def TaskStream(self, response, context): try: # If the client is already in the client store, update the status success, result = client_store.update( - self.clients[client.client_id]["_id"], + "client_id", + client.client_id, {"name": client.name, "status": "online", "client_id": client.client_id, "last_seen": datetime.now(), "combiner": self.id}, ) - if not success: - logger.error(result) - except KeyError: - # If the client is not in the client store, add the client - success, result = client_store.add( - {"name": client.name, "status": "online", "client_id": client.client_id, "last_seen": datetime.now(), "combiner": self.id} - ) - # Get the _id of the client and add it to the clients dict - if success: - self.clients[client.client_id]["_id"] = result["id"] - else: + if not success and result == "Entity not found": + # If the client is not in the client store, add the client + success, result = client_store.add( + { + "name": client.name, + "status": "online", + "client_id": client.client_id, + "last_seen": datetime.now(), + "combiner": self.id, + "combiner_preferred": self.id, + "updated_at": datetime.now(), + } + ) + elif not success: logger.error(result) + except Exception as e: + logger.error(f"Failed to update client status: {str(e)}") # Keep track of the time context has been active start_time = time.time() diff --git a/fedn/network/storage/statestore/stores/client_store.py b/fedn/network/storage/statestore/stores/client_store.py index f8693fb34..dd521860f 100644 --- a/fedn/network/storage/statestore/stores/client_store.py +++ b/fedn/network/storage/statestore/stores/client_store.py @@ -7,7 +7,7 @@ from fedn.network.storage.statestore.stores.store import Store -from .shared import EntityNotFound +from .shared import EntityNotFound, from_document class Client: @@ -49,8 +49,28 @@ def get(self, id: str, use_typing: bool = False) -> Client: response = super().get(id, use_typing=use_typing) return Client.from_dict(response) if use_typing else response - def update(self, id: str, item: Client) -> bool: - return super().update(id, item) + def _get_client_by_client_id(self, client_id: str) -> Dict: + document = self.database[self.collection].find_one({"client_id": client_id}) + if document is None: + raise EntityNotFound(f"Entity with client_id {client_id} not found") + return document + + def _get_client_by_name(self, name: str) -> Dict: + document = self.database[self.collection].find_one({"name": name}) + if document is None: + raise EntityNotFound(f"Entity with name {name} not found") + return document + + def update(self, by_key: str, value: str, item: Client) -> bool: + try: + result = self.database[self.collection].update_one({by_key: value}, {"$set": item}) + if result.modified_count == 1: + document = self.database[self.collection].find_one({by_key: value}) + return True, from_document(document) + else: + return False, "Entity not found" + except Exception as e: + return False, str(e) def add(self, item: Client) -> Tuple[bool, Any]: return super().add(item) From 3450880ae2e4d02840cce42345fb0a279d406920 Mon Sep 17 00:00:00 2001 From: Fredrik Wrede Date: Thu, 3 Oct 2024 18:51:26 +0000 Subject: [PATCH 06/17] fix --- fedn/network/combiner/combiner.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fedn/network/combiner/combiner.py b/fedn/network/combiner/combiner.py index 8203592a5..a848fa68d 100644 --- a/fedn/network/combiner/combiner.py +++ b/fedn/network/combiner/combiner.py @@ -123,10 +123,11 @@ def __init__(self, config): # the client will be stuck listed as "online" in the statestore. # Set the status to offline for previous clients. previous_clients = client_store.list(limit=0, skip=0, sort_key=None, kwargs={"combiner": self.id}) - logger.info(f"Found {previous_clients["count"]} previous clients") + count = previous_clients["count"] + result = previous_clients["result"] + logger.info(f"Found {count} previous clients") logger.info("Updating previous clients status to offline") - previous_clients = previous_clients["result"] - for client in previous_clients: + for client in result: try: if "client_id" in client.keys(): client_store.update("client_id", client["client_id"], {"name": client["name"], "status": "offline"}) From db425d6b9da2244af80c3eb835abeba34174dee3 Mon Sep 17 00:00:00 2001 From: Fredrik Wrede Date: Fri, 4 Oct 2024 15:21:10 +0000 Subject: [PATCH 07/17] fix inconsistant naming of collections --- fedn/network/api/v1/inference_routes.py | 5 ++++- fedn/network/api/v1/status_routes.py | 4 ++++ fedn/network/combiner/shared.py | 6 +++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/fedn/network/api/v1/inference_routes.py b/fedn/network/api/v1/inference_routes.py index 460dd495f..5a6b0dced 100644 --- a/fedn/network/api/v1/inference_routes.py +++ b/fedn/network/api/v1/inference_routes.py @@ -4,10 +4,13 @@ from fedn.network.api.auth import jwt_auth_required from fedn.network.api.shared import control -from fedn.network.api.v1.shared import api_version +from fedn.network.api.v1.shared import api_version, mdb +from fedn.network.storage.statestore.stores.inference_store import InferenceStore bp = Blueprint("inference", __name__, url_prefix=f"/api/{api_version}/infer") +inference_store = InferenceStore(mdb, "control.inferences") + @bp.route("/start", methods=["POST"]) @jwt_auth_required(role="admin") diff --git a/fedn/network/api/v1/status_routes.py b/fedn/network/api/v1/status_routes.py index cf3907bea..0716f965b 100644 --- a/fedn/network/api/v1/status_routes.py +++ b/fedn/network/api/v1/status_routes.py @@ -124,8 +124,12 @@ def get_statuses(): limit, skip, sort_key, sort_order, use_typing = get_typed_list_headers(request.headers) kwargs = request.args.to_dict() + # print all the typed headers + print(f"limit: {limit}, skip: {skip}, sort_key: {sort_key}, sort_order: {sort_order}, use_typing: {use_typing}") + print(f"kwargs: {kwargs}") statuses = status_store.list(limit, skip, sort_key, sort_order, use_typing=use_typing, **kwargs) + print(f"statuses: {statuses}") result = [status.__dict__ for status in statuses["result"]] if use_typing else statuses["result"] response = {"count": statuses["count"], "result": result} diff --git a/fedn/network/combiner/shared.py b/fedn/network/combiner/shared.py index 30c5ed469..11caa1e98 100644 --- a/fedn/network/combiner/shared.py +++ b/fedn/network/combiner/shared.py @@ -23,10 +23,10 @@ mdb: Database = mc[network_id] client_store = ClientStore(mdb, "network.clients") -validation_store = ValidationStore(mdb, "network.validations") +validation_store = ValidationStore(mdb, "control.validations") combiner_store = CombinerStore(mdb, "network.combiners") -status_store = StatusStore(mdb, "network.status") -inference_store = InferenceStore(mdb, "network.inferences") +status_store = StatusStore(mdb, "control.status") +inference_store = InferenceStore(mdb, "control.inferences") repository = Repository(modelstorage_config["storage_config"], init_buckets=False) From 399f85e9385fbbad504d47b4937258cff295f9ae Mon Sep 17 00:00:00 2001 From: benjaminastrand Date: Wed, 9 Oct 2024 11:56:59 +0200 Subject: [PATCH 08/17] Replaced 'inference' with 'prediction' --- fedn/network/api/v1/inference_routes.py | 10 +++--- fedn/network/clients/client.py | 34 +++++++++---------- fedn/network/combiner/combiner.py | 12 +++---- fedn/network/combiner/roundhandler.py | 16 ++++----- fedn/network/grpc/fedn.proto | 6 ++-- fedn/network/grpc/fedn_pb2.py | 2 +- fedn/network/grpc/fedn_pb2_grpc.py | 20 +++++------ .../statestore/stores/inference_store.py | 4 +-- 8 files changed, 52 insertions(+), 52 deletions(-) diff --git a/fedn/network/api/v1/inference_routes.py b/fedn/network/api/v1/inference_routes.py index 5a6b0dced..60ab86e5b 100644 --- a/fedn/network/api/v1/inference_routes.py +++ b/fedn/network/api/v1/inference_routes.py @@ -16,19 +16,19 @@ @jwt_auth_required(role="admin") def start_session(): """Start a new inference session. - param: inference_id: The session id to start. - type: inference_id: str + param: prediction_id: The session id to start. + type: prediction_id: str param: rounds: The number of rounds to run. type: rounds: int """ try: data = request.json if request.headers["Content-Type"] == "application/json" else request.form.to_dict() - inference_id: str = data.get("inference_id") + prediction_id: str = data.get("prediction_id") - if not inference_id or inference_id == "": + if not prediction_id or prediction_id == "": return jsonify({"message": "Session ID is required"}), 400 - session_config = {"inference_id": inference_id} + session_config = {"prediction_id": prediction_id} threading.Thread(target=control.inference_session, kwargs={"config": session_config}).start() diff --git a/fedn/network/clients/client.py b/fedn/network/clients/client.py index 97574e7b2..f2b0b1138 100644 --- a/fedn/network/clients/client.py +++ b/fedn/network/clients/client.py @@ -712,7 +712,7 @@ def process_request(self): try: presigned_url = json.loads(request.data) except json.JSONDecodeError as e: - logger.error(f"Failed to decode inference request data: {e}") + logger.error(f"Failed to decode prediction request data: {e}") self.state = ClientState.idle continue @@ -721,27 +721,27 @@ def process_request(self): self.state = ClientState.idle continue presigned_url = presigned_url["presigned_url"] - # Obs that session_id in request is the inference_id + # Obs that session_id in request is the prediction_id _ = self._process_inference_request(request.model_id, request.session_id, presigned_url) - inference = fedn.ModelInference() - inference.sender.name = self.name - inference.sender.role = fedn.WORKER - inference.receiver.name = request.sender.name - inference.receiver.name = request.sender.name - inference.receiver.role = request.sender.role - inference.model_id = str(request.model_id) - # TODO: Add inference data - inference.data = "" - inference.timestamp.GetCurrentTime() - inference.correlation_id = request.correlation_id - # Obs that session_id in request is the inference_id - inference.inference_id = request.session_id + prediction = fedn.ModelPrediction() + prediction.sender.name = self.name + prediction.sender.role = fedn.WORKER + prediction.receiver.name = request.sender.name + prediction.receiver.name = request.sender.name + prediction.receiver.role = request.sender.role + prediction.model_id = str(request.model_id) + # TODO: Add prediction data + prediction.data = "" + prediction.timestamp.GetCurrentTime() + prediction.correlation_id = request.correlation_id + # Obs that session_id in request is the prediction_id + prediction.prediction_id = request.session_id try: - _ = self.combinerStub.SendModelInference(inference, metadata=self.metadata) + _ = self.combinerStub.SendModelPrediction(prediction, metadata=self.metadata) status_type = fedn.StatusType.INFERENCE self.send_status( - "Model inference completed.", log_level=fedn.Status.AUDIT, type=status_type, request=inference, sesssion_id=request.session_id + "Model prediction completed.", log_level=fedn.Status.AUDIT, type=status_type, request=prediction, sesssion_id=request.session_id ) except grpc.RpcError as e: status_code = e.code() diff --git a/fedn/network/combiner/combiner.py b/fedn/network/combiner/combiner.py index a848fa68d..a643cb759 100644 --- a/fedn/network/combiner/combiner.py +++ b/fedn/network/combiner/combiner.py @@ -209,7 +209,7 @@ def request_model_validation(self, session_id, model_id, clients=[]): else: logger.info("Sent model validation request for model {} to {} clients".format(model_id, len(clients))) - def request_model_inference(self, inference_id: str, model_id: str, clients: list = []) -> None: + def request_model_inference(self, prediction_id: str, model_id: str, clients: list = []) -> None: """Ask clients to perform inference on the model. :param model_id: the model id to perform inference on @@ -220,7 +220,7 @@ def request_model_inference(self, inference_id: str, model_id: str, clients: lis :type clients: list """ - clients = self._send_request_type(fedn.StatusType.INFERENCE, inference_id, model_id, clients) + clients = self._send_request_type(fedn.StatusType.INFERENCE, prediction_id, model_id, clients) if len(clients) < 20: logger.info("Sent model inference request for model {} to clients {}".format(model_id, clients)) @@ -746,23 +746,23 @@ def SendModelValidation(self, request, context): response.response = "RECEIVED ModelValidation {} from client {}".format(response, response.sender.name) return response - def SendModelInference(self, request, context): + def SendModelPrediction(self, request, context): """Send a model inference response. :param request: the request - :type request: :class:`fedn.network.grpc.fedn_pb2.ModelInference` + :type request: :class:`fedn.network.grpc.fedn_pb2.ModelPrediction` :param context: the context :type context: :class:`grpc._server._Context` :return: the response :rtype: :class:`fedn.network.grpc.fedn_pb2.Response` """ - logger.info("Recieved ModelInference from {}".format(request.sender.name)) + logger.info("Recieved ModelPrediction from {}".format(request.sender.name)) result = MessageToDict(request, including_default_value_fields=True) inference_store.add(result) response = fedn.Response() - response.response = "RECEIVED ModelInference {} from client {}".format(response, response.sender.name) + response.response = "RECEIVED ModelPrediction {} from client {}".format(response, response.sender.name) return response #################################################################################################################### diff --git a/fedn/network/combiner/roundhandler.py b/fedn/network/combiner/roundhandler.py index 64d793318..44b756941 100644 --- a/fedn/network/combiner/roundhandler.py +++ b/fedn/network/combiner/roundhandler.py @@ -42,8 +42,8 @@ class RoundConfig(TypedDict): :type model_metadata: dict :param session_id: The session identifier. Set by (Controller?). :type session_id: str - :param inference_id: The inference identifier. Only used for inference tasks. - :type inference_id: str + :param prediction_id: The inference identifier. Only used for inference tasks. + :type prediction_id: str :param helper_type: The helper type. :type helper_type: str :param aggregator: The aggregator type. @@ -252,7 +252,7 @@ def _validation_round(self, session_id, model_id, clients): """ self.server.request_model_validation(session_id, model_id, clients=clients) - def _inference_round(self, inference_id: str, model_id: str, clients: list): + def _inference_round(self, prediction_id: str, model_id: str, clients: list): """Send model inference requests to clients. :param config: The round config object (passed to the client). @@ -262,7 +262,7 @@ def _inference_round(self, inference_id: str, model_id: str, clients: list): :param model_id: The ID of the model to use for inference :type model_id: str """ - self.server.request_model_inference(inference_id, model_id, clients=clients) + self.server.request_model_inference(prediction_id, model_id, clients=clients) def stage_model(self, model_id, timeout_retry=3, retry=2): """Download a model from persistent storage and set in modelservice. @@ -350,7 +350,7 @@ def execute_validation_round(self, session_id, model_id): validators = self._assign_round_clients(self.server.max_clients, type="validators") self._validation_round(session_id, model_id, validators) - def execute_inference_round(self, inference_id: str, model_id: str) -> None: + def execute_inference_round(self, prediction_id: str, model_id: str) -> None: """Coordinate inference rounds as specified in config. :param round_config: The round config object. @@ -360,7 +360,7 @@ def execute_inference_round(self, inference_id: str, model_id: str) -> None: self.stage_model(model_id) # TODO: Implement inference client type clients = self._assign_round_clients(self.server.max_clients, type="validators") - self._inference_round(inference_id, model_id, clients) + self._inference_round(prediction_id, model_id, clients) def execute_training_round(self, config): """Coordinates clients to execute training tasks. @@ -429,9 +429,9 @@ def run(self, polling_interval=1.0): model_id = round_config["model_id"] self.execute_validation_round(session_id, model_id) elif round_config["task"] == "inference": - inference_id = round_config["inference_id"] + prediction_id = round_config["prediction_id"] model_id = round_config["model_id"] - self.execute_inference_round(inference_id, model_id) + self.execute_inference_round(prediction_id, model_id) else: logger.warning("config contains unkown task type.") else: diff --git a/fedn/network/grpc/fedn.proto b/fedn/network/grpc/fedn.proto index 03009c2d9..c3846444c 100644 --- a/fedn/network/grpc/fedn.proto +++ b/fedn/network/grpc/fedn.proto @@ -80,7 +80,7 @@ message ModelValidation { string session_id = 8; } -message ModelInference { +message ModelPrediction { Client sender = 1; Client receiver = 2; string model_id = 3; @@ -88,7 +88,7 @@ message ModelInference { string correlation_id = 5; google.protobuf.Timestamp timestamp = 6; string meta = 7; - string inference_id = 8; + string prediction_id = 8; } enum ModelStatus { @@ -252,7 +252,7 @@ service Combiner { rpc SendModelUpdate (ModelUpdate) returns (Response); rpc SendModelValidation (ModelValidation) returns (Response); - rpc SendModelInference (ModelInference) returns (Response); + rpc SendModelPrediction (ModelPrediction) returns (Response); } diff --git a/fedn/network/grpc/fedn_pb2.py b/fedn/network/grpc/fedn_pb2.py index 92233c9b8..c7a721215 100644 --- a/fedn/network/grpc/fedn_pb2.py +++ b/fedn/network/grpc/fedn_pb2.py @@ -15,7 +15,7 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17network/grpc/fedn.proto\x12\x04\x66\x65\x64n\x1a\x1fgoogle/protobuf/timestamp.proto\":\n\x08Response\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08response\x18\x02 \x01(\t\"\xbc\x02\n\x06Status\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06status\x18\x02 \x01(\t\x12(\n\tlog_level\x18\x03 \x01(\x0e\x32\x15.fedn.Status.LogLevel\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1e\n\x04type\x18\x07 \x01(\x0e\x32\x10.fedn.StatusType\x12\r\n\x05\x65xtra\x18\x08 \x01(\t\x12\x12\n\nsession_id\x18\t \x01(\t\"B\n\x08LogLevel\x12\x08\n\x04INFO\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\t\n\x05\x41UDIT\x10\x04\"\xd8\x01\n\x0bTaskRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\x12\x1e\n\x04type\x18\t \x01(\x0e\x32\x10.fedn.StatusType\"\xbf\x01\n\x0bModelUpdate\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x17\n\x0fmodel_update_id\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x08 \x01(\t\"\xd8\x01\n\x0fModelValidation\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\"\xd9\x01\n\x0eModelInference\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x14\n\x0cinference_id\x18\x08 \x01(\t\"\x89\x01\n\x0cModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\n\n\x02id\x18\x04 \x01(\t\x12!\n\x06status\x18\x05 \x01(\x0e\x32\x11.fedn.ModelStatus\"]\n\rModelResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\x12!\n\x06status\x18\x03 \x01(\x0e\x32\x11.fedn.ModelStatus\x12\x0f\n\x07message\x18\x04 \x01(\t\"U\n\x15GetGlobalModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\"h\n\x16GetGlobalModelResponse\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\")\n\tHeartbeat\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\"W\n\x16\x43lientAvailableMessage\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\t\"P\n\x12ListClientsRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1c\n\x07\x63hannel\x18\x02 \x01(\x0e\x32\x0b.fedn.Queue\"*\n\nClientList\x12\x1c\n\x06\x63lient\x18\x01 \x03(\x0b\x32\x0c.fedn.Client\"C\n\x06\x43lient\x12\x18\n\x04role\x18\x01 \x01(\x0e\x32\n.fedn.Role\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x11\n\tclient_id\x18\x03 \x01(\t\"m\n\x0fReassignRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06server\x18\x03 \x01(\t\x12\x0c\n\x04port\x18\x04 \x01(\r\"c\n\x10ReconnectRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x11\n\treconnect\x18\x03 \x01(\r\"\'\n\tParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"T\n\x0e\x43ontrolRequest\x12\x1e\n\x07\x63ommand\x18\x01 \x01(\x0e\x32\r.fedn.Command\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"F\n\x0f\x43ontrolResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"\x13\n\x11\x43onnectionRequest\"<\n\x12\x43onnectionResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.fedn.ConnectionStatus*\x84\x01\n\nStatusType\x12\x07\n\x03LOG\x10\x00\x12\x18\n\x14MODEL_UPDATE_REQUEST\x10\x01\x12\x10\n\x0cMODEL_UPDATE\x10\x02\x12\x1c\n\x18MODEL_VALIDATION_REQUEST\x10\x03\x12\x14\n\x10MODEL_VALIDATION\x10\x04\x12\r\n\tINFERENCE\x10\x05*$\n\x05Queue\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x0e\n\nTASK_QUEUE\x10\x01*S\n\x0bModelStatus\x12\x06\n\x02OK\x10\x00\x12\x0f\n\x0bIN_PROGRESS\x10\x01\x12\x12\n\x0eIN_PROGRESS_OK\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\x0b\n\x07UNKNOWN\x10\x04*8\n\x04Role\x12\n\n\x06WORKER\x10\x00\x12\x0c\n\x08\x43OMBINER\x10\x01\x12\x0b\n\x07REDUCER\x10\x02\x12\t\n\x05OTHER\x10\x03*J\n\x07\x43ommand\x12\x08\n\x04IDLE\x10\x00\x12\t\n\x05START\x10\x01\x12\t\n\x05PAUSE\x10\x02\x12\x08\n\x04STOP\x10\x03\x12\t\n\x05RESET\x10\x04\x12\n\n\x06REPORT\x10\x05*I\n\x10\x43onnectionStatus\x12\x11\n\rNOT_ACCEPTING\x10\x00\x12\r\n\tACCEPTING\x10\x01\x12\x13\n\x0fTRY_AGAIN_LATER\x10\x02\x32z\n\x0cModelService\x12\x33\n\x06Upload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse(\x01\x12\x35\n\x08\x44ownload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse0\x01\x32\xf8\x01\n\x07\x43ontrol\x12\x34\n\x05Start\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x33\n\x04Stop\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x44\n\x15\x46lushAggregationQueue\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12<\n\rSetAggregator\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse2V\n\x07Reducer\x12K\n\x0eGetGlobalModel\x12\x1b.fedn.GetGlobalModelRequest\x1a\x1c.fedn.GetGlobalModelResponse2\xab\x03\n\tConnector\x12\x44\n\x14\x41llianceStatusStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x0c.fedn.Status0\x01\x12*\n\nSendStatus\x12\x0c.fedn.Status\x1a\x0e.fedn.Response\x12?\n\x11ListActiveClients\x12\x18.fedn.ListClientsRequest\x1a\x10.fedn.ClientList\x12\x45\n\x10\x41\x63\x63\x65ptingClients\x12\x17.fedn.ConnectionRequest\x1a\x18.fedn.ConnectionResponse\x12\x30\n\rSendHeartbeat\x12\x0f.fedn.Heartbeat\x1a\x0e.fedn.Response\x12\x37\n\x0eReassignClient\x12\x15.fedn.ReassignRequest\x1a\x0e.fedn.Response\x12\x39\n\x0fReconnectClient\x12\x16.fedn.ReconnectRequest\x1a\x0e.fedn.Response2\xfb\x01\n\x08\x43ombiner\x12?\n\nTaskStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x11.fedn.TaskRequest0\x01\x12\x34\n\x0fSendModelUpdate\x12\x11.fedn.ModelUpdate\x1a\x0e.fedn.Response\x12<\n\x13SendModelValidation\x12\x15.fedn.ModelValidation\x1a\x0e.fedn.Response\x12:\n\x12SendModelInference\x12\x14.fedn.ModelInference\x1a\x0e.fedn.Responseb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17network/grpc/fedn.proto\x12\x04\x66\x65\x64n\x1a\x1fgoogle/protobuf/timestamp.proto\":\n\x08Response\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08response\x18\x02 \x01(\t\"\xbc\x02\n\x06Status\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06status\x18\x02 \x01(\t\x12(\n\tlog_level\x18\x03 \x01(\x0e\x32\x15.fedn.Status.LogLevel\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1e\n\x04type\x18\x07 \x01(\x0e\x32\x10.fedn.StatusType\x12\r\n\x05\x65xtra\x18\x08 \x01(\t\x12\x12\n\nsession_id\x18\t \x01(\t\"B\n\x08LogLevel\x12\x08\n\x04INFO\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\t\n\x05\x41UDIT\x10\x04\"\xd8\x01\n\x0bTaskRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\x12\x1e\n\x04type\x18\t \x01(\x0e\x32\x10.fedn.StatusType\"\xbf\x01\n\x0bModelUpdate\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x17\n\x0fmodel_update_id\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x08 \x01(\t\"\xd8\x01\n\x0fModelValidation\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\"\xd9\x01\n\x0eModelPrediction\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x14\n\x0cprediction_id\x18\x08 \x01(\t\"\x89\x01\n\x0cModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\n\n\x02id\x18\x04 \x01(\t\x12!\n\x06status\x18\x05 \x01(\x0e\x32\x11.fedn.ModelStatus\"]\n\rModelResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\x12!\n\x06status\x18\x03 \x01(\x0e\x32\x11.fedn.ModelStatus\x12\x0f\n\x07message\x18\x04 \x01(\t\"U\n\x15GetGlobalModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\"h\n\x16GetGlobalModelResponse\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\")\n\tHeartbeat\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\"W\n\x16\x43lientAvailableMessage\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\t\"P\n\x12ListClientsRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1c\n\x07\x63hannel\x18\x02 \x01(\x0e\x32\x0b.fedn.Queue\"*\n\nClientList\x12\x1c\n\x06\x63lient\x18\x01 \x03(\x0b\x32\x0c.fedn.Client\"C\n\x06\x43lient\x12\x18\n\x04role\x18\x01 \x01(\x0e\x32\n.fedn.Role\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x11\n\tclient_id\x18\x03 \x01(\t\"m\n\x0fReassignRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06server\x18\x03 \x01(\t\x12\x0c\n\x04port\x18\x04 \x01(\r\"c\n\x10ReconnectRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x11\n\treconnect\x18\x03 \x01(\r\"\'\n\tParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"T\n\x0e\x43ontrolRequest\x12\x1e\n\x07\x63ommand\x18\x01 \x01(\x0e\x32\r.fedn.Command\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"F\n\x0f\x43ontrolResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"\x13\n\x11\x43onnectionRequest\"<\n\x12\x43onnectionResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.fedn.ConnectionStatus*\x84\x01\n\nStatusType\x12\x07\n\x03LOG\x10\x00\x12\x18\n\x14MODEL_UPDATE_REQUEST\x10\x01\x12\x10\n\x0cMODEL_UPDATE\x10\x02\x12\x1c\n\x18MODEL_VALIDATION_REQUEST\x10\x03\x12\x14\n\x10MODEL_VALIDATION\x10\x04\x12\r\n\tINFERENCE\x10\x05*$\n\x05Queue\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x0e\n\nTASK_QUEUE\x10\x01*S\n\x0bModelStatus\x12\x06\n\x02OK\x10\x00\x12\x0f\n\x0bIN_PROGRESS\x10\x01\x12\x12\n\x0eIN_PROGRESS_OK\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\x0b\n\x07UNKNOWN\x10\x04*8\n\x04Role\x12\n\n\x06WORKER\x10\x00\x12\x0c\n\x08\x43OMBINER\x10\x01\x12\x0b\n\x07REDUCER\x10\x02\x12\t\n\x05OTHER\x10\x03*J\n\x07\x43ommand\x12\x08\n\x04IDLE\x10\x00\x12\t\n\x05START\x10\x01\x12\t\n\x05PAUSE\x10\x02\x12\x08\n\x04STOP\x10\x03\x12\t\n\x05RESET\x10\x04\x12\n\n\x06REPORT\x10\x05*I\n\x10\x43onnectionStatus\x12\x11\n\rNOT_ACCEPTING\x10\x00\x12\r\n\tACCEPTING\x10\x01\x12\x13\n\x0fTRY_AGAIN_LATER\x10\x02\x32z\n\x0cModelService\x12\x33\n\x06Upload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse(\x01\x12\x35\n\x08\x44ownload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse0\x01\x32\xf8\x01\n\x07\x43ontrol\x12\x34\n\x05Start\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x33\n\x04Stop\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x44\n\x15\x46lushAggregationQueue\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12<\n\rSetAggregator\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse2V\n\x07Reducer\x12K\n\x0eGetGlobalModel\x12\x1b.fedn.GetGlobalModelRequest\x1a\x1c.fedn.GetGlobalModelResponse2\xab\x03\n\tConnector\x12\x44\n\x14\x41llianceStatusStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x0c.fedn.Status0\x01\x12*\n\nSendStatus\x12\x0c.fedn.Status\x1a\x0e.fedn.Response\x12?\n\x11ListActiveClients\x12\x18.fedn.ListClientsRequest\x1a\x10.fedn.ClientList\x12\x45\n\x10\x41\x63\x63\x65ptingClients\x12\x17.fedn.ConnectionRequest\x1a\x18.fedn.ConnectionResponse\x12\x30\n\rSendHeartbeat\x12\x0f.fedn.Heartbeat\x1a\x0e.fedn.Response\x12\x37\n\x0eReassignClient\x12\x15.fedn.ReassignRequest\x1a\x0e.fedn.Response\x12\x39\n\x0fReconnectClient\x12\x16.fedn.ReconnectRequest\x1a\x0e.fedn.Response2\xfb\x01\n\x08\x43ombiner\x12?\n\nTaskStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x11.fedn.TaskRequest0\x01\x12\x34\n\x0fSendModelUpdate\x12\x11.fedn.ModelUpdate\x1a\x0e.fedn.Response\x12<\n\x13SendModelValidation\x12\x15.fedn.ModelValidation\x1a\x0e.fedn.Response\x12:\n\x12SendModelPrediction\x12\x14.fedn.ModelPrediction\x1a\x0e.fedn.Responseb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) diff --git a/fedn/network/grpc/fedn_pb2_grpc.py b/fedn/network/grpc/fedn_pb2_grpc.py index 1c533953b..32ac134d7 100644 --- a/fedn/network/grpc/fedn_pb2_grpc.py +++ b/fedn/network/grpc/fedn_pb2_grpc.py @@ -608,9 +608,9 @@ def __init__(self, channel): request_serializer=network_dot_grpc_dot_fedn__pb2.ModelValidation.SerializeToString, response_deserializer=network_dot_grpc_dot_fedn__pb2.Response.FromString, ) - self.SendModelInference = channel.unary_unary( - '/fedn.Combiner/SendModelInference', - request_serializer=network_dot_grpc_dot_fedn__pb2.ModelInference.SerializeToString, + self.SendModelPrediction = channel.unary_unary( + '/fedn.Combiner/SendModelPrediction', + request_serializer=network_dot_grpc_dot_fedn__pb2.ModelPrediction.SerializeToString, response_deserializer=network_dot_grpc_dot_fedn__pb2.Response.FromString, ) @@ -637,7 +637,7 @@ def SendModelValidation(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') - def SendModelInference(self, request, context): + def SendModelPrediction(self, request, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -661,9 +661,9 @@ def add_CombinerServicer_to_server(servicer, server): request_deserializer=network_dot_grpc_dot_fedn__pb2.ModelValidation.FromString, response_serializer=network_dot_grpc_dot_fedn__pb2.Response.SerializeToString, ), - 'SendModelInference': grpc.unary_unary_rpc_method_handler( - servicer.SendModelInference, - request_deserializer=network_dot_grpc_dot_fedn__pb2.ModelInference.FromString, + 'SendModelPrediction': grpc.unary_unary_rpc_method_handler( + servicer.SendModelPrediction, + request_deserializer=network_dot_grpc_dot_fedn__pb2.ModelPrediction.FromString, response_serializer=network_dot_grpc_dot_fedn__pb2.Response.SerializeToString, ), } @@ -728,7 +728,7 @@ def SendModelValidation(request, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) @staticmethod - def SendModelInference(request, + def SendModelPrediction(request, target, options=(), channel_credentials=None, @@ -738,8 +738,8 @@ def SendModelInference(request, wait_for_ready=None, timeout=None, metadata=None): - return grpc.experimental.unary_unary(request, target, '/fedn.Combiner/SendModelInference', - network_dot_grpc_dot_fedn__pb2.ModelInference.SerializeToString, + return grpc.experimental.unary_unary(request, target, '/fedn.Combiner/SendModelPrediction', + network_dot_grpc_dot_fedn__pb2.ModelPrediction.SerializeToString, network_dot_grpc_dot_fedn__pb2.Response.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/fedn/network/storage/statestore/stores/inference_store.py b/fedn/network/storage/statestore/stores/inference_store.py index cff1a9d89..69a17f170 100644 --- a/fedn/network/storage/statestore/stores/inference_store.py +++ b/fedn/network/storage/statestore/stores/inference_store.py @@ -8,14 +8,14 @@ class Inference: def __init__( - self, id: str, model_id: str, data: str, correlation_id: str, timestamp: str, inference_id: str, meta: str, sender: dict = None, receiver: dict = None + self, id: str, model_id: str, data: str, correlation_id: str, timestamp: str, prediction_id: str, meta: str, sender: dict = None, receiver: dict = None ): self.id = id self.model_id = model_id self.data = data self.correlation_id = correlation_id self.timestamp = timestamp - self.inference_id = inference_id + self.prediction_id = prediction_id self.meta = meta self.sender = sender self.receiver = receiver From eac749a0413157be6dd50394f47e4360489647d9 Mon Sep 17 00:00:00 2001 From: benjaminastrand Date: Thu, 10 Oct 2024 14:17:51 +0200 Subject: [PATCH 09/17] Replace "inference" with "prediction" across entire repo (except StatusType in protos) --- .../{inference_test.py => prediction_test.py} | 2 +- .../{run_inference.sh => run_prediction.sh} | 8 ++-- .github/workflows/integration-tests.yaml | 4 +- examples/mnist-keras/client/predict.py | 4 +- fedn/network/api/v1/__init__.py | 4 +- ...ference_routes.py => prediction_routes.py} | 14 +++--- fedn/network/clients/client.py | 48 ++++++++----------- fedn/network/clients/state.py | 2 +- fedn/network/combiner/combiner.py | 24 +++++----- fedn/network/combiner/roundhandler.py | 28 +++++------ fedn/network/combiner/shared.py | 4 +- fedn/network/controller/control.py | 38 +++++++-------- fedn/network/storage/s3/repository.py | 6 +-- ...inference_store.py => prediction_store.py} | 26 +++++----- 14 files changed, 102 insertions(+), 110 deletions(-) rename .ci/tests/examples/{inference_test.py => prediction_test.py} (92%) rename .ci/tests/examples/{run_inference.sh => run_prediction.sh} (57%) rename fedn/network/api/v1/{inference_routes.py => prediction_routes.py} (62%) rename fedn/network/storage/statestore/stores/{inference_store.py => prediction_store.py} (80%) diff --git a/.ci/tests/examples/inference_test.py b/.ci/tests/examples/prediction_test.py similarity index 92% rename from .ci/tests/examples/inference_test.py rename to .ci/tests/examples/prediction_test.py index 6e27d2499..eed62f784 100644 --- a/.ci/tests/examples/inference_test.py +++ b/.ci/tests/examples/prediction_test.py @@ -32,4 +32,4 @@ def _wait_n_rounds(collection): # Wait for successful rounds succeded = _wait_n_rounds(client['fedn-test-network']['control']['status']) assert(succeded == N_CLIENTS) # check that all rounds succeeded - _eprint(f'Succeded inference clients: {succeded}. Test passed.') + _eprint(f'Succeded prediction clients: {succeded}. Test passed.') diff --git a/.ci/tests/examples/run_inference.sh b/.ci/tests/examples/run_prediction.sh similarity index 57% rename from .ci/tests/examples/run_inference.sh rename to .ci/tests/examples/run_prediction.sh index d78771d70..eb676fce3 100755 --- a/.ci/tests/examples/run_inference.sh +++ b/.ci/tests/examples/run_prediction.sh @@ -8,12 +8,12 @@ if [ "$#" -lt 1 ]; then fi example="$1" ->&2 echo "Run inference" +>&2 echo "Run prediction" pushd "examples/$example" -curl -k -X POST https://localhost:8090/infer +curl -k -X POST https://localhost:8090/predict ->&2 echo "Checking inference success" -".$example/bin/python" ../../.ci/tests/examples/inference_test.py +>&2 echo "Checking prediction success" +".$example/bin/python" ../../.ci/tests/examples/prediction_test.py >&2 echo "Test completed successfully" popd \ No newline at end of file diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index ed354a78b..b89df6ca6 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -38,8 +38,8 @@ jobs: - name: run ${{ matrix.to_test }} run: .ci/tests/examples/run.sh ${{ matrix.to_test }} - # - name: run ${{ matrix.to_test }} inference - # run: .ci/tests/examples/run_inference.sh ${{ matrix.to_test }} + # - name: run ${{ matrix.to_test }} prediction + # run: .ci/tests/examples/run_prediction.sh ${{ matrix.to_test }} # if: ${{ matrix.os != 'macos-11' && matrix.to_test == 'mnist-keras keras' }} # example available for Keras - name: print logs diff --git a/examples/mnist-keras/client/predict.py b/examples/mnist-keras/client/predict.py index 9d502ed75..412bb74ae 100644 --- a/examples/mnist-keras/client/predict.py +++ b/examples/mnist-keras/client/predict.py @@ -11,13 +11,13 @@ def predict(in_model_path, out_json_path, data_path=None): - # Using test data for inference but another dataset could be loaded + # Using test data for prediction but another dataset could be loaded x_test, _ = load_data(data_path, is_train=False) # Load model model = load_parameters(in_model_path) - # Infer + # Predict y_pred = model.predict(x_test) y_pred = np.argmax(y_pred, axis=1) diff --git a/fedn/network/api/v1/__init__.py b/fedn/network/api/v1/__init__.py index 0e05dd249..d2030eaab 100644 --- a/fedn/network/api/v1/__init__.py +++ b/fedn/network/api/v1/__init__.py @@ -1,6 +1,6 @@ from fedn.network.api.v1.client_routes import bp as client_bp from fedn.network.api.v1.combiner_routes import bp as combiner_bp -from fedn.network.api.v1.inference_routes import bp as inference_bp +from fedn.network.api.v1.prediction_routes import bp as prediction_bp from fedn.network.api.v1.model_routes import bp as model_bp from fedn.network.api.v1.package_routes import bp as package_bp from fedn.network.api.v1.round_routes import bp as round_bp @@ -8,4 +8,4 @@ from fedn.network.api.v1.status_routes import bp as status_bp from fedn.network.api.v1.validation_routes import bp as validation_bp -_routes = [client_bp, combiner_bp, model_bp, package_bp, round_bp, session_bp, status_bp, validation_bp, inference_bp] +_routes = [client_bp, combiner_bp, model_bp, package_bp, round_bp, session_bp, status_bp, validation_bp, prediction_bp] diff --git a/fedn/network/api/v1/inference_routes.py b/fedn/network/api/v1/prediction_routes.py similarity index 62% rename from fedn/network/api/v1/inference_routes.py rename to fedn/network/api/v1/prediction_routes.py index 60ab86e5b..7cbcc0c29 100644 --- a/fedn/network/api/v1/inference_routes.py +++ b/fedn/network/api/v1/prediction_routes.py @@ -5,17 +5,17 @@ from fedn.network.api.auth import jwt_auth_required from fedn.network.api.shared import control from fedn.network.api.v1.shared import api_version, mdb -from fedn.network.storage.statestore.stores.inference_store import InferenceStore +from fedn.network.storage.statestore.stores.prediction_store import PredictionStore -bp = Blueprint("inference", __name__, url_prefix=f"/api/{api_version}/infer") +bp = Blueprint("prediction", __name__, url_prefix=f"/api/{api_version}/predict") -inference_store = InferenceStore(mdb, "control.inferences") +prediction_store = PredictionStore(mdb, "control.predictions") @bp.route("/start", methods=["POST"]) @jwt_auth_required(role="admin") def start_session(): - """Start a new inference session. + """Start a new prediction session. param: prediction_id: The session id to start. type: prediction_id: str param: rounds: The number of rounds to run. @@ -30,8 +30,8 @@ def start_session(): session_config = {"prediction_id": prediction_id} - threading.Thread(target=control.inference_session, kwargs={"config": session_config}).start() + threading.Thread(target=control.prediction_session, kwargs={"config": session_config}).start() - return jsonify({"message": "Inference session started"}), 200 + return jsonify({"message": "Prediction session started"}), 200 except Exception: - return jsonify({"message": "Failed to start inference session"}), 500 + return jsonify({"message": "Failed to start prediction session"}), 500 diff --git a/fedn/network/clients/client.py b/fedn/network/clients/client.py index f2b0b1138..184ec5615 100644 --- a/fedn/network/clients/client.py +++ b/fedn/network/clients/client.py @@ -418,11 +418,11 @@ def _listen_to_task_stream(self): elif request.type == fedn.StatusType.MODEL_VALIDATION and self.config["validator"]: self.inbox.put(("validate", request)) elif request.type == fedn.StatusType.INFERENCE and self.config["validator"]: - logger.info("Received inference request for model_id {}".format(request.model_id)) + logger.info("Received prediction request for model_id {}".format(request.model_id)) presigned_url = json.loads(request.data) presigned_url = presigned_url["presigned_url"] - logger.info("Inference presigned URL: {}".format(presigned_url)) - self.inbox.put(("infer", request)) + logger.info("Prediction presigned URL: {}".format(presigned_url)) + self.inbox.put(("predict", request)) else: logger.error("Unknown request type: {}".format(request.type)) @@ -519,25 +519,17 @@ def _process_training_request(self, model_id: str, session_id: str = None): return updated_model_id, meta - def _process_validation_request(self, model_id: str, is_inference: bool, session_id: str = None): + def _process_validation_request(self, model_id: str, session_id: str = None): """Process a validation request. :param model_id: The model id of the model to be validated. :type model_id: str - :param is_inference: True if the validation is an inference request, False if it is a validation request. - :type is_inference: bool :param session_id: The id of the current session. :type session_id: str :return: The validation metrics, or None if validation failed. :rtype: dict """ - # Figure out cmd - if is_inference: - cmd = "infer" - else: - cmd = "validate" - - self.send_status(f"Processing {cmd} request for model_id {model_id}", sesssion_id=session_id) + self.send_status(f"Processing validation request for model_id {model_id}", sesssion_id=session_id) self.state = ClientState.validating try: model = self.get_model_from_combiner(str(model_id)) @@ -550,7 +542,7 @@ def _process_validation_request(self, model_id: str, is_inference: bool, session fh.write(model.getbuffer()) outpath = get_tmp_path() - self.dispatcher.run_cmd(f"{cmd} {inpath} {outpath}") + self.dispatcher.run_cmd(f"validate {inpath} {outpath}") with open(outpath, "r") as fh: validation = json.loads(fh.read()) @@ -566,22 +558,22 @@ def _process_validation_request(self, model_id: str, is_inference: bool, session self.state = ClientState.idle return validation - def _process_inference_request(self, model_id: str, session_id: str, presigned_url: str): - """Process an inference request. + def _process_prediction_request(self, model_id: str, session_id: str, presigned_url: str): + """Process an prediction request. - :param model_id: The model id of the model to be used for inference. + :param model_id: The model id of the model to be used for prediction. :type model_id: str :param session_id: The id of the current session. :type session_id: str - :param presigned_url: The presigned URL for the data to be used for inference. + :param presigned_url: The presigned URL for the data to be used for prediction. :type presigned_url: str :return: None """ - self.send_status(f"Processing inference request for model_id {model_id}", sesssion_id=session_id) + self.send_status(f"Processing prediction request for model_id {model_id}", sesssion_id=session_id) try: model = self.get_model_from_combiner(str(model_id)) if model is None: - logger.error("Could not retrieve model from combiner. Aborting inference request.") + logger.error("Could not retrieve model from combiner. Aborting prediction request.") return inpath = self.helper.get_tmp_path() @@ -591,7 +583,7 @@ def _process_inference_request(self, model_id: str, session_id: str, presigned_u outpath = get_tmp_path() self.dispatcher.run_cmd(f"predict {inpath} {outpath}") - # Upload the inference result to the presigned URL + # Upload the prediction result to the presigned URL with open(outpath, "rb") as fh: response = requests.put(presigned_url, data=fh.read()) @@ -599,12 +591,12 @@ def _process_inference_request(self, model_id: str, session_id: str, presigned_u os.unlink(outpath) if response.status_code != 200: - logger.warning("Inference upload failed with status code {}".format(response.status_code)) + logger.warning("Prediction upload failed with status code {}".format(response.status_code)) self.state = ClientState.idle return except Exception as e: - logger.warning("Inference failed with exception {}".format(e)) + logger.warning("Prediction failed with exception {}".format(e)) self.state = ClientState.idle return @@ -668,7 +660,7 @@ def process_request(self): elif task_type == "validate": self.state = ClientState.validating - metrics = self._process_validation_request(request.model_id, False, request.session_id) + metrics = self._process_validation_request(request.model_id, request.session_id) if metrics is not None: # Send validation @@ -707,8 +699,8 @@ def process_request(self): self.state = ClientState.idle self.inbox.task_done() - elif task_type == "infer": - self.state = ClientState.inferencing + elif task_type == "predict": + self.state = ClientState.predicting try: presigned_url = json.loads(request.data) except json.JSONDecodeError as e: @@ -717,12 +709,12 @@ def process_request(self): continue if "presigned_url" not in presigned_url: - logger.error("Inference request missing presigned_url.") + logger.error("Prediction request missing presigned_url.") self.state = ClientState.idle continue presigned_url = presigned_url["presigned_url"] # Obs that session_id in request is the prediction_id - _ = self._process_inference_request(request.model_id, request.session_id, presigned_url) + _ = self._process_prediction_request(request.model_id, request.session_id, presigned_url) prediction = fedn.ModelPrediction() prediction.sender.name = self.name prediction.sender.role = fedn.WORKER diff --git a/fedn/network/clients/state.py b/fedn/network/clients/state.py index d7f82a769..80b182fc9 100644 --- a/fedn/network/clients/state.py +++ b/fedn/network/clients/state.py @@ -7,7 +7,7 @@ class ClientState(Enum): idle = 1 training = 2 validating = 3 - inferencing = 4 + predicting = 4 def ClientStateToString(state): diff --git a/fedn/network/combiner/combiner.py b/fedn/network/combiner/combiner.py index a643cb759..ee6cde974 100644 --- a/fedn/network/combiner/combiner.py +++ b/fedn/network/combiner/combiner.py @@ -16,7 +16,7 @@ from fedn.common.certificate.certificate import Certificate from fedn.common.log_config import logger, set_log_level_from_string, set_log_stream from fedn.network.combiner.roundhandler import RoundConfig, RoundHandler -from fedn.network.combiner.shared import client_store, combiner_store, inference_store, repository, statestore, status_store, validation_store +from fedn.network.combiner.shared import client_store, combiner_store, prediction_store, repository, statestore, status_store, validation_store from fedn.network.grpc.server import Server, ServerConfig VALID_NAME_REGEX = "^[a-zA-Z0-9_-]*$" @@ -209,10 +209,10 @@ def request_model_validation(self, session_id, model_id, clients=[]): else: logger.info("Sent model validation request for model {} to {} clients".format(model_id, len(clients))) - def request_model_inference(self, prediction_id: str, model_id: str, clients: list = []) -> None: - """Ask clients to perform inference on the model. + def request_model_prediction(self, prediction_id: str, model_id: str, clients: list = []) -> None: + """Ask clients to perform prediction on the model. - :param model_id: the model id to perform inference on + :param model_id: the model id to perform prediction on :type model_id: str :param config: the model configuration to send to clients :type config: dict @@ -223,16 +223,16 @@ def request_model_inference(self, prediction_id: str, model_id: str, clients: li clients = self._send_request_type(fedn.StatusType.INFERENCE, prediction_id, model_id, clients) if len(clients) < 20: - logger.info("Sent model inference request for model {} to clients {}".format(model_id, clients)) + logger.info("Sent model prediction request for model {} to clients {}".format(model_id, clients)) else: - logger.info("Sent model inference request for model {} to {} clients".format(model_id, len(clients))) + logger.info("Sent model prediction request for model {} to {} clients".format(model_id, len(clients))) def _send_request_type(self, request_type, session_id, model_id, config=None, clients=[]): """Send a request of a specific type to clients. :param request_type: the type of request :type request_type: :class:`fedn.network.grpc.fedn_pb2.StatusType` - :param session_id: the session id to send in the request. Obs that for inference, this is the inference id. + :param session_id: the session id to send in the request. Obs that for prediction, this is the prediction id. :type session_id: str :param model_id: the model id to send in the request :type model_id: str @@ -249,7 +249,7 @@ def _send_request_type(self, request_type, session_id, model_id, config=None, cl elif request_type == fedn.StatusType.MODEL_VALIDATION: clients = self.get_active_validators() elif request_type == fedn.StatusType.INFERENCE: - # TODO: add inference clients type + # TODO: add prediction clients type clients = self.get_active_validators() for client in clients: request = fedn.TaskRequest() @@ -265,8 +265,8 @@ def _send_request_type(self, request_type, session_id, model_id, config=None, cl request.receiver.role = fedn.WORKER # Set the request data, not used in validation if request_type == fedn.StatusType.INFERENCE: - presigned_url = self.repository.presigned_put_url(self.repository.inference_bucket, f"{client}/{session_id}") - # TODO: in inference, request.data should also contain user-defined data/parameters + presigned_url = self.repository.presigned_put_url(self.repository.prediction_bucket, f"{client}/{session_id}") + # TODO: in prediction, request.data should also contain user-defined data/parameters request.data = json.dumps({"presigned_url": presigned_url}) elif request_type == fedn.StatusType.MODEL_UPDATE: request.data = json.dumps(config) @@ -747,7 +747,7 @@ def SendModelValidation(self, request, context): return response def SendModelPrediction(self, request, context): - """Send a model inference response. + """Send a model prediction response. :param request: the request :type request: :class:`fedn.network.grpc.fedn_pb2.ModelPrediction` @@ -759,7 +759,7 @@ def SendModelPrediction(self, request, context): logger.info("Recieved ModelPrediction from {}".format(request.sender.name)) result = MessageToDict(request, including_default_value_fields=True) - inference_store.add(result) + prediction_store.add(result) response = fedn.Response() response.response = "RECEIVED ModelPrediction {} from client {}".format(response, response.sender.name) diff --git a/fedn/network/combiner/roundhandler.py b/fedn/network/combiner/roundhandler.py index 44b756941..a4808f7a5 100644 --- a/fedn/network/combiner/roundhandler.py +++ b/fedn/network/combiner/roundhandler.py @@ -21,7 +21,7 @@ class RoundConfig(TypedDict): :type _job_id: str :param committed_at: The time the round was committed. Set by Controller. :type committed_at: str - :param task: The task to perform in the round. Set by Controller. Supported tasks are "training", "validation", and "inference". + :param task: The task to perform in the round. Set by Controller. Supported tasks are "training", "validation", and "prediction". :type task: str :param round_id: The round identifier as str(int) :type round_id: str @@ -42,7 +42,7 @@ class RoundConfig(TypedDict): :type model_metadata: dict :param session_id: The session identifier. Set by (Controller?). :type session_id: str - :param prediction_id: The inference identifier. Only used for inference tasks. + :param prediction_id: The prediction identifier. Only used for prediction tasks. :type prediction_id: str :param helper_type: The helper type. :type helper_type: str @@ -252,17 +252,17 @@ def _validation_round(self, session_id, model_id, clients): """ self.server.request_model_validation(session_id, model_id, clients=clients) - def _inference_round(self, prediction_id: str, model_id: str, clients: list): - """Send model inference requests to clients. + def _prediction_round(self, prediction_id: str, model_id: str, clients: list): + """Send model prediction requests to clients. :param config: The round config object (passed to the client). :type config: dict - :param clients: clients to send inference requests to + :param clients: clients to send prediction requests to :type clients: list - :param model_id: The ID of the model to use for inference + :param model_id: The ID of the model to use for prediction :type model_id: str """ - self.server.request_model_inference(prediction_id, model_id, clients=clients) + self.server.request_model_prediction(prediction_id, model_id, clients=clients) def stage_model(self, model_id, timeout_retry=3, retry=2): """Download a model from persistent storage and set in modelservice. @@ -350,17 +350,17 @@ def execute_validation_round(self, session_id, model_id): validators = self._assign_round_clients(self.server.max_clients, type="validators") self._validation_round(session_id, model_id, validators) - def execute_inference_round(self, prediction_id: str, model_id: str) -> None: - """Coordinate inference rounds as specified in config. + def execute_prediction_round(self, prediction_id: str, model_id: str) -> None: + """Coordinate prediction rounds as specified in config. :param round_config: The round config object. :type round_config: dict """ - logger.info("COMBINER orchestrating inference using model {}".format(model_id)) + logger.info("COMBINER orchestrating prediction using model {}".format(model_id)) self.stage_model(model_id) - # TODO: Implement inference client type + # TODO: Implement prediction client type clients = self._assign_round_clients(self.server.max_clients, type="validators") - self._inference_round(prediction_id, model_id, clients) + self._prediction_round(prediction_id, model_id, clients) def execute_training_round(self, config): """Coordinates clients to execute training tasks. @@ -428,10 +428,10 @@ def run(self, polling_interval=1.0): session_id = round_config["session_id"] model_id = round_config["model_id"] self.execute_validation_round(session_id, model_id) - elif round_config["task"] == "inference": + elif round_config["task"] == "prediction": prediction_id = round_config["prediction_id"] model_id = round_config["model_id"] - self.execute_inference_round(prediction_id, model_id) + self.execute_prediction_round(prediction_id, model_id) else: logger.warning("config contains unkown task type.") else: diff --git a/fedn/network/combiner/shared.py b/fedn/network/combiner/shared.py index 11caa1e98..bf9a63032 100644 --- a/fedn/network/combiner/shared.py +++ b/fedn/network/combiner/shared.py @@ -7,7 +7,7 @@ from fedn.network.storage.statestore.mongostatestore import MongoStateStore from fedn.network.storage.statestore.stores.client_store import ClientStore from fedn.network.storage.statestore.stores.combiner_store import CombinerStore -from fedn.network.storage.statestore.stores.inference_store import InferenceStore +from fedn.network.storage.statestore.stores.prediction_store import PredictionStore from fedn.network.storage.statestore.stores.status_store import StatusStore from fedn.network.storage.statestore.stores.validation_store import ValidationStore @@ -26,7 +26,7 @@ validation_store = ValidationStore(mdb, "control.validations") combiner_store = CombinerStore(mdb, "network.combiners") status_store = StatusStore(mdb, "control.status") -inference_store = InferenceStore(mdb, "control.inferences") +prediction_store = PredictionStore(mdb, "control.predictions") repository = Repository(modelstorage_config["storage_config"], init_buckets=False) diff --git a/fedn/network/controller/control.py b/fedn/network/controller/control.py index 38e775d0e..a283b7389 100644 --- a/fedn/network/controller/control.py +++ b/fedn/network/controller/control.py @@ -82,7 +82,7 @@ def __init__(self, message): class Control(ControlBase): - """Controller, implementing the overall global training, validation and inference logic. + """Controller, implementing the overall global training, validation and prediction logic. :param statestore: A StateStorage instance. :type statestore: class: `fedn.network.statestorebase.StateStorageBase` @@ -211,11 +211,11 @@ def session(self, config: RoundConfig) -> None: self.set_session_status(config["session_id"], "Finished") self._state = ReducerState.idle - def inference_session(self, config: RoundConfig) -> None: - """Execute a new inference session. + def prediction_session(self, config: RoundConfig) -> None: + """Execute a new prediction session. :param config: The round config. - :type config: InferenceConfig + :type config: PredictionConfig :return: None """ if self._state == ReducerState.instructing: @@ -223,14 +223,14 @@ def inference_session(self, config: RoundConfig) -> None: return if len(self.network.get_combiners()) < 1: - logger.warning("Inference round cannot start, no combiners connected!") + logger.warning("Prediction round cannot start, no combiners connected!") return if "model_id" not in config.keys(): config["model_id"] = self.statestore.get_latest_model() config["committed_at"] = datetime.datetime.now() - config["task"] = "inference" + config["task"] = "prediction" config["rounds"] = str(1) config["clients_required"] = 1 @@ -240,10 +240,10 @@ def inference_session(self, config: RoundConfig) -> None: round_start = self.evaluate_round_start_policy(participating_combiners) if round_start: - logger.info("Inference round start policy met, {} participating combiners.".format(len(participating_combiners))) + logger.info("Prediction round start policy met, {} participating combiners.".format(len(participating_combiners))) for combiner, _ in participating_combiners: combiner.submit(config) - logger.info("Inference round submitted to combiner {}".format(combiner)) + logger.info("Prediction round submitted to combiner {}".format(combiner)) def round(self, session_config: RoundConfig, round_id: str): """Execute one global round. @@ -437,10 +437,10 @@ def reduce(self, combiners): return model, meta - def infer_instruct(self, config): - """Main entrypoint for executing the inference compute plan. + def predict_instruct(self, config): + """Main entrypoint for executing the prediction compute plan. - : param config: configuration for the inference round + : param config: configuration for the prediction round """ # Check/set instucting state if self.__state == ReducerState.instructing: @@ -455,19 +455,19 @@ def infer_instruct(self, config): # Set reducer in monitoring state self.__state = ReducerState.monitoring - # Start inference round + # Start prediction round try: - self.inference_round(config) + self.prediction_round(config) except TypeError: logger.error("Round failed.") # Set reducer in idle state self.__state = ReducerState.idle - def inference_round(self, config): - """Execute an inference round. + def prediction_round(self, config): + """Execute an prediction round. - : param config: configuration for the inference round + : param config: configuration for the prediction round """ # Init meta round_data = {} @@ -480,7 +480,7 @@ def inference_round(self, config): # Setup combiner configuration combiner_config = copy.deepcopy(config) combiner_config["model_id"] = self.statestore.get_latest_model() - combiner_config["task"] = "inference" + combiner_config["task"] = "prediction" combiner_config["helper_type"] = self.statestore.get_framework() # Select combiners @@ -494,12 +494,12 @@ def inference_round(self, config): logger.warning("Round start policy not met, skipping round!") return None - # Synch combiners with latest model and trigger inference + # Synch combiners with latest model and trigger prediction for combiner, combiner_config in validating_combiners: try: combiner.submit(combiner_config) except CombinerUnavailableError: - # It is OK if inference fails for a combiner + # It is OK if prediction fails for a combiner self._handle_unavailable_combiner(combiner) pass diff --git a/fedn/network/storage/s3/repository.py b/fedn/network/storage/s3/repository.py index 0b08fe7b9..4ad11bc0b 100644 --- a/fedn/network/storage/s3/repository.py +++ b/fedn/network/storage/s3/repository.py @@ -12,9 +12,9 @@ def __init__(self, config, init_buckets=True): self.model_bucket = config["storage_bucket"] self.context_bucket = config["context_bucket"] try: - self.inference_bucket = config["inference_bucket"] + self.prediction_bucket = config["prediction_bucket"] except KeyError: - self.inference_bucket = "fedn-inference" + self.prediction_bucket = "fedn-prediction" # TODO: Make a plug-in solution self.client = MINIORepository(config) @@ -22,7 +22,7 @@ def __init__(self, config, init_buckets=True): if init_buckets: self.client.create_bucket(self.context_bucket) self.client.create_bucket(self.model_bucket) - self.client.create_bucket(self.inference_bucket) + self.client.create_bucket(self.prediction_bucket) def get_model(self, model_id): """Retrieve a model with id model_id. diff --git a/fedn/network/storage/statestore/stores/inference_store.py b/fedn/network/storage/statestore/stores/prediction_store.py similarity index 80% rename from fedn/network/storage/statestore/stores/inference_store.py rename to fedn/network/storage/statestore/stores/prediction_store.py index 69a17f170..3a14ec8b9 100644 --- a/fedn/network/storage/statestore/stores/inference_store.py +++ b/fedn/network/storage/statestore/stores/prediction_store.py @@ -6,7 +6,7 @@ from fedn.network.storage.statestore.stores.store import Store -class Inference: +class Prediction: def __init__( self, id: str, model_id: str, data: str, correlation_id: str, timestamp: str, prediction_id: str, meta: str, sender: dict = None, receiver: dict = None ): @@ -20,8 +20,8 @@ def __init__( self.sender = sender self.receiver = receiver - def from_dict(data: dict) -> "Inference": - return Inference( + def from_dict(data: dict) -> "Prediction": + return Prediction( id=str(data["_id"]), model_id=data["modelId"] if "modelId" in data else None, data=data["data"] if "data" in data else None, @@ -34,32 +34,32 @@ def from_dict(data: dict) -> "Inference": ) -class InferenceStore(Store[Inference]): +class PredictionStore(Store[Prediction]): def __init__(self, database: Database, collection: str): super().__init__(database, collection) - def get(self, id: str, use_typing: bool = False) -> Inference: + def get(self, id: str, use_typing: bool = False) -> Prediction: """Get an entity by id param id: The id of the entity type: str - description: The id of the entity, can be either the id or the Inference (property) + description: The id of the entity, can be either the id or the Prediction (property) param use_typing: Whether to return the entity as a typed object or as a dict type: bool return: The entity """ response = super().get(id, use_typing=use_typing) - return Inference.from_dict(response) if use_typing else response + return Prediction.from_dict(response) if use_typing else response - def update(self, id: str, item: Inference) -> bool: - raise NotImplementedError("Update not implemented for InferenceStore") + def update(self, id: str, item: Prediction) -> bool: + raise NotImplementedError("Update not implemented for PredictionStore") - def add(self, item: Inference) -> Tuple[bool, Any]: + def add(self, item: Prediction) -> Tuple[bool, Any]: return super().add(item) def delete(self, id: str) -> bool: - raise NotImplementedError("Delete not implemented for InferenceStore") + raise NotImplementedError("Delete not implemented for PredictionStore") - def list(self, limit: int, skip: int, sort_key: str, sort_order=pymongo.DESCENDING, use_typing: bool = False, **kwargs) -> Dict[int, List[Inference]]: + def list(self, limit: int, skip: int, sort_key: str, sort_order=pymongo.DESCENDING, use_typing: bool = False, **kwargs) -> Dict[int, List[Prediction]]: """List entities param limit: The maximum number of entities to return type: int @@ -80,5 +80,5 @@ def list(self, limit: int, skip: int, sort_key: str, sort_order=pymongo.DESCENDI """ response = super().list(limit, skip, sort_key or "timestamp", sort_order, use_typing=use_typing, **kwargs) - result = [Inference.from_dict(item) for item in response["result"]] if use_typing else response["result"] + result = [Prediction.from_dict(item) for item in response["result"]] if use_typing else response["result"] return {"count": response["count"], "result": result} From a699720cf896c019962f514ab67edbf6ebe9c9a1 Mon Sep 17 00:00:00 2001 From: benjaminastrand Date: Thu, 10 Oct 2024 14:41:21 +0200 Subject: [PATCH 10/17] Regenerated protos --- fedn/network/grpc/fedn_pb2.py | 114 +++++++++++++++++----------------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/fedn/network/grpc/fedn_pb2.py b/fedn/network/grpc/fedn_pb2.py index c7a721215..e43facd13 100644 --- a/fedn/network/grpc/fedn_pb2.py +++ b/fedn/network/grpc/fedn_pb2.py @@ -15,25 +15,25 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17network/grpc/fedn.proto\x12\x04\x66\x65\x64n\x1a\x1fgoogle/protobuf/timestamp.proto\":\n\x08Response\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08response\x18\x02 \x01(\t\"\xbc\x02\n\x06Status\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06status\x18\x02 \x01(\t\x12(\n\tlog_level\x18\x03 \x01(\x0e\x32\x15.fedn.Status.LogLevel\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1e\n\x04type\x18\x07 \x01(\x0e\x32\x10.fedn.StatusType\x12\r\n\x05\x65xtra\x18\x08 \x01(\t\x12\x12\n\nsession_id\x18\t \x01(\t\"B\n\x08LogLevel\x12\x08\n\x04INFO\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\t\n\x05\x41UDIT\x10\x04\"\xd8\x01\n\x0bTaskRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\x12\x1e\n\x04type\x18\t \x01(\x0e\x32\x10.fedn.StatusType\"\xbf\x01\n\x0bModelUpdate\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x17\n\x0fmodel_update_id\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x08 \x01(\t\"\xd8\x01\n\x0fModelValidation\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\"\xd9\x01\n\x0eModelPrediction\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x14\n\x0cprediction_id\x18\x08 \x01(\t\"\x89\x01\n\x0cModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\n\n\x02id\x18\x04 \x01(\t\x12!\n\x06status\x18\x05 \x01(\x0e\x32\x11.fedn.ModelStatus\"]\n\rModelResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\x12!\n\x06status\x18\x03 \x01(\x0e\x32\x11.fedn.ModelStatus\x12\x0f\n\x07message\x18\x04 \x01(\t\"U\n\x15GetGlobalModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\"h\n\x16GetGlobalModelResponse\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\")\n\tHeartbeat\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\"W\n\x16\x43lientAvailableMessage\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\t\"P\n\x12ListClientsRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1c\n\x07\x63hannel\x18\x02 \x01(\x0e\x32\x0b.fedn.Queue\"*\n\nClientList\x12\x1c\n\x06\x63lient\x18\x01 \x03(\x0b\x32\x0c.fedn.Client\"C\n\x06\x43lient\x12\x18\n\x04role\x18\x01 \x01(\x0e\x32\n.fedn.Role\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x11\n\tclient_id\x18\x03 \x01(\t\"m\n\x0fReassignRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06server\x18\x03 \x01(\t\x12\x0c\n\x04port\x18\x04 \x01(\r\"c\n\x10ReconnectRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x11\n\treconnect\x18\x03 \x01(\r\"\'\n\tParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"T\n\x0e\x43ontrolRequest\x12\x1e\n\x07\x63ommand\x18\x01 \x01(\x0e\x32\r.fedn.Command\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"F\n\x0f\x43ontrolResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"\x13\n\x11\x43onnectionRequest\"<\n\x12\x43onnectionResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.fedn.ConnectionStatus*\x84\x01\n\nStatusType\x12\x07\n\x03LOG\x10\x00\x12\x18\n\x14MODEL_UPDATE_REQUEST\x10\x01\x12\x10\n\x0cMODEL_UPDATE\x10\x02\x12\x1c\n\x18MODEL_VALIDATION_REQUEST\x10\x03\x12\x14\n\x10MODEL_VALIDATION\x10\x04\x12\r\n\tINFERENCE\x10\x05*$\n\x05Queue\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x0e\n\nTASK_QUEUE\x10\x01*S\n\x0bModelStatus\x12\x06\n\x02OK\x10\x00\x12\x0f\n\x0bIN_PROGRESS\x10\x01\x12\x12\n\x0eIN_PROGRESS_OK\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\x0b\n\x07UNKNOWN\x10\x04*8\n\x04Role\x12\n\n\x06WORKER\x10\x00\x12\x0c\n\x08\x43OMBINER\x10\x01\x12\x0b\n\x07REDUCER\x10\x02\x12\t\n\x05OTHER\x10\x03*J\n\x07\x43ommand\x12\x08\n\x04IDLE\x10\x00\x12\t\n\x05START\x10\x01\x12\t\n\x05PAUSE\x10\x02\x12\x08\n\x04STOP\x10\x03\x12\t\n\x05RESET\x10\x04\x12\n\n\x06REPORT\x10\x05*I\n\x10\x43onnectionStatus\x12\x11\n\rNOT_ACCEPTING\x10\x00\x12\r\n\tACCEPTING\x10\x01\x12\x13\n\x0fTRY_AGAIN_LATER\x10\x02\x32z\n\x0cModelService\x12\x33\n\x06Upload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse(\x01\x12\x35\n\x08\x44ownload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse0\x01\x32\xf8\x01\n\x07\x43ontrol\x12\x34\n\x05Start\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x33\n\x04Stop\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x44\n\x15\x46lushAggregationQueue\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12<\n\rSetAggregator\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse2V\n\x07Reducer\x12K\n\x0eGetGlobalModel\x12\x1b.fedn.GetGlobalModelRequest\x1a\x1c.fedn.GetGlobalModelResponse2\xab\x03\n\tConnector\x12\x44\n\x14\x41llianceStatusStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x0c.fedn.Status0\x01\x12*\n\nSendStatus\x12\x0c.fedn.Status\x1a\x0e.fedn.Response\x12?\n\x11ListActiveClients\x12\x18.fedn.ListClientsRequest\x1a\x10.fedn.ClientList\x12\x45\n\x10\x41\x63\x63\x65ptingClients\x12\x17.fedn.ConnectionRequest\x1a\x18.fedn.ConnectionResponse\x12\x30\n\rSendHeartbeat\x12\x0f.fedn.Heartbeat\x1a\x0e.fedn.Response\x12\x37\n\x0eReassignClient\x12\x15.fedn.ReassignRequest\x1a\x0e.fedn.Response\x12\x39\n\x0fReconnectClient\x12\x16.fedn.ReconnectRequest\x1a\x0e.fedn.Response2\xfb\x01\n\x08\x43ombiner\x12?\n\nTaskStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x11.fedn.TaskRequest0\x01\x12\x34\n\x0fSendModelUpdate\x12\x11.fedn.ModelUpdate\x1a\x0e.fedn.Response\x12<\n\x13SendModelValidation\x12\x15.fedn.ModelValidation\x1a\x0e.fedn.Response\x12:\n\x12SendModelPrediction\x12\x14.fedn.ModelPrediction\x1a\x0e.fedn.Responseb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17network/grpc/fedn.proto\x12\x04\x66\x65\x64n\x1a\x1fgoogle/protobuf/timestamp.proto\":\n\x08Response\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08response\x18\x02 \x01(\t\"\xbc\x02\n\x06Status\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06status\x18\x02 \x01(\t\x12(\n\tlog_level\x18\x03 \x01(\x0e\x32\x15.fedn.Status.LogLevel\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1e\n\x04type\x18\x07 \x01(\x0e\x32\x10.fedn.StatusType\x12\r\n\x05\x65xtra\x18\x08 \x01(\t\x12\x12\n\nsession_id\x18\t \x01(\t\"B\n\x08LogLevel\x12\x08\n\x04INFO\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\t\n\x05\x41UDIT\x10\x04\"\xd8\x01\n\x0bTaskRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\x12\x1e\n\x04type\x18\t \x01(\x0e\x32\x10.fedn.StatusType\"\xbf\x01\n\x0bModelUpdate\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x17\n\x0fmodel_update_id\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x08 \x01(\t\"\xd8\x01\n\x0fModelValidation\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\"\xdb\x01\n\x0fModelPrediction\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x15\n\rprediction_id\x18\x08 \x01(\t\"\x89\x01\n\x0cModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\n\n\x02id\x18\x04 \x01(\t\x12!\n\x06status\x18\x05 \x01(\x0e\x32\x11.fedn.ModelStatus\"]\n\rModelResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\x12!\n\x06status\x18\x03 \x01(\x0e\x32\x11.fedn.ModelStatus\x12\x0f\n\x07message\x18\x04 \x01(\t\"U\n\x15GetGlobalModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\"h\n\x16GetGlobalModelResponse\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\")\n\tHeartbeat\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\"W\n\x16\x43lientAvailableMessage\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\t\"P\n\x12ListClientsRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1c\n\x07\x63hannel\x18\x02 \x01(\x0e\x32\x0b.fedn.Queue\"*\n\nClientList\x12\x1c\n\x06\x63lient\x18\x01 \x03(\x0b\x32\x0c.fedn.Client\"C\n\x06\x43lient\x12\x18\n\x04role\x18\x01 \x01(\x0e\x32\n.fedn.Role\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x11\n\tclient_id\x18\x03 \x01(\t\"m\n\x0fReassignRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06server\x18\x03 \x01(\t\x12\x0c\n\x04port\x18\x04 \x01(\r\"c\n\x10ReconnectRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x11\n\treconnect\x18\x03 \x01(\r\"\'\n\tParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"T\n\x0e\x43ontrolRequest\x12\x1e\n\x07\x63ommand\x18\x01 \x01(\x0e\x32\r.fedn.Command\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"F\n\x0f\x43ontrolResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"\x13\n\x11\x43onnectionRequest\"<\n\x12\x43onnectionResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.fedn.ConnectionStatus*\x84\x01\n\nStatusType\x12\x07\n\x03LOG\x10\x00\x12\x18\n\x14MODEL_UPDATE_REQUEST\x10\x01\x12\x10\n\x0cMODEL_UPDATE\x10\x02\x12\x1c\n\x18MODEL_VALIDATION_REQUEST\x10\x03\x12\x14\n\x10MODEL_VALIDATION\x10\x04\x12\r\n\tINFERENCE\x10\x05*$\n\x05Queue\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x0e\n\nTASK_QUEUE\x10\x01*S\n\x0bModelStatus\x12\x06\n\x02OK\x10\x00\x12\x0f\n\x0bIN_PROGRESS\x10\x01\x12\x12\n\x0eIN_PROGRESS_OK\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\x0b\n\x07UNKNOWN\x10\x04*8\n\x04Role\x12\n\n\x06WORKER\x10\x00\x12\x0c\n\x08\x43OMBINER\x10\x01\x12\x0b\n\x07REDUCER\x10\x02\x12\t\n\x05OTHER\x10\x03*J\n\x07\x43ommand\x12\x08\n\x04IDLE\x10\x00\x12\t\n\x05START\x10\x01\x12\t\n\x05PAUSE\x10\x02\x12\x08\n\x04STOP\x10\x03\x12\t\n\x05RESET\x10\x04\x12\n\n\x06REPORT\x10\x05*I\n\x10\x43onnectionStatus\x12\x11\n\rNOT_ACCEPTING\x10\x00\x12\r\n\tACCEPTING\x10\x01\x12\x13\n\x0fTRY_AGAIN_LATER\x10\x02\x32z\n\x0cModelService\x12\x33\n\x06Upload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse(\x01\x12\x35\n\x08\x44ownload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse0\x01\x32\xf8\x01\n\x07\x43ontrol\x12\x34\n\x05Start\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x33\n\x04Stop\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x44\n\x15\x46lushAggregationQueue\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12<\n\rSetAggregator\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse2V\n\x07Reducer\x12K\n\x0eGetGlobalModel\x12\x1b.fedn.GetGlobalModelRequest\x1a\x1c.fedn.GetGlobalModelResponse2\xab\x03\n\tConnector\x12\x44\n\x14\x41llianceStatusStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x0c.fedn.Status0\x01\x12*\n\nSendStatus\x12\x0c.fedn.Status\x1a\x0e.fedn.Response\x12?\n\x11ListActiveClients\x12\x18.fedn.ListClientsRequest\x1a\x10.fedn.ClientList\x12\x45\n\x10\x41\x63\x63\x65ptingClients\x12\x17.fedn.ConnectionRequest\x1a\x18.fedn.ConnectionResponse\x12\x30\n\rSendHeartbeat\x12\x0f.fedn.Heartbeat\x1a\x0e.fedn.Response\x12\x37\n\x0eReassignClient\x12\x15.fedn.ReassignRequest\x1a\x0e.fedn.Response\x12\x39\n\x0fReconnectClient\x12\x16.fedn.ReconnectRequest\x1a\x0e.fedn.Response2\xfd\x01\n\x08\x43ombiner\x12?\n\nTaskStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x11.fedn.TaskRequest0\x01\x12\x34\n\x0fSendModelUpdate\x12\x11.fedn.ModelUpdate\x1a\x0e.fedn.Response\x12<\n\x13SendModelValidation\x12\x15.fedn.ModelValidation\x1a\x0e.fedn.Response\x12<\n\x13SendModelPrediction\x12\x15.fedn.ModelPrediction\x1a\x0e.fedn.Responseb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'network.grpc.fedn_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None - _globals['_STATUSTYPE']._serialized_start=2547 - _globals['_STATUSTYPE']._serialized_end=2679 - _globals['_QUEUE']._serialized_start=2681 - _globals['_QUEUE']._serialized_end=2717 - _globals['_MODELSTATUS']._serialized_start=2719 - _globals['_MODELSTATUS']._serialized_end=2802 - _globals['_ROLE']._serialized_start=2804 - _globals['_ROLE']._serialized_end=2860 - _globals['_COMMAND']._serialized_start=2862 - _globals['_COMMAND']._serialized_end=2936 - _globals['_CONNECTIONSTATUS']._serialized_start=2938 - _globals['_CONNECTIONSTATUS']._serialized_end=3011 + _globals['_STATUSTYPE']._serialized_start=2549 + _globals['_STATUSTYPE']._serialized_end=2681 + _globals['_QUEUE']._serialized_start=2683 + _globals['_QUEUE']._serialized_end=2719 + _globals['_MODELSTATUS']._serialized_start=2721 + _globals['_MODELSTATUS']._serialized_end=2804 + _globals['_ROLE']._serialized_start=2806 + _globals['_ROLE']._serialized_end=2862 + _globals['_COMMAND']._serialized_start=2864 + _globals['_COMMAND']._serialized_end=2938 + _globals['_CONNECTIONSTATUS']._serialized_start=2940 + _globals['_CONNECTIONSTATUS']._serialized_end=3013 _globals['_RESPONSE']._serialized_start=66 _globals['_RESPONSE']._serialized_end=124 _globals['_STATUS']._serialized_start=127 @@ -46,48 +46,48 @@ _globals['_MODELUPDATE']._serialized_end=856 _globals['_MODELVALIDATION']._serialized_start=859 _globals['_MODELVALIDATION']._serialized_end=1075 - _globals['_MODELINFERENCE']._serialized_start=1078 - _globals['_MODELINFERENCE']._serialized_end=1295 - _globals['_MODELREQUEST']._serialized_start=1298 - _globals['_MODELREQUEST']._serialized_end=1435 - _globals['_MODELRESPONSE']._serialized_start=1437 - _globals['_MODELRESPONSE']._serialized_end=1530 - _globals['_GETGLOBALMODELREQUEST']._serialized_start=1532 - _globals['_GETGLOBALMODELREQUEST']._serialized_end=1617 - _globals['_GETGLOBALMODELRESPONSE']._serialized_start=1619 - _globals['_GETGLOBALMODELRESPONSE']._serialized_end=1723 - _globals['_HEARTBEAT']._serialized_start=1725 - _globals['_HEARTBEAT']._serialized_end=1766 - _globals['_CLIENTAVAILABLEMESSAGE']._serialized_start=1768 - _globals['_CLIENTAVAILABLEMESSAGE']._serialized_end=1855 - _globals['_LISTCLIENTSREQUEST']._serialized_start=1857 - _globals['_LISTCLIENTSREQUEST']._serialized_end=1937 - _globals['_CLIENTLIST']._serialized_start=1939 - _globals['_CLIENTLIST']._serialized_end=1981 - _globals['_CLIENT']._serialized_start=1983 - _globals['_CLIENT']._serialized_end=2050 - _globals['_REASSIGNREQUEST']._serialized_start=2052 - _globals['_REASSIGNREQUEST']._serialized_end=2161 - _globals['_RECONNECTREQUEST']._serialized_start=2163 - _globals['_RECONNECTREQUEST']._serialized_end=2262 - _globals['_PARAMETER']._serialized_start=2264 - _globals['_PARAMETER']._serialized_end=2303 - _globals['_CONTROLREQUEST']._serialized_start=2305 - _globals['_CONTROLREQUEST']._serialized_end=2389 - _globals['_CONTROLRESPONSE']._serialized_start=2391 - _globals['_CONTROLRESPONSE']._serialized_end=2461 - _globals['_CONNECTIONREQUEST']._serialized_start=2463 - _globals['_CONNECTIONREQUEST']._serialized_end=2482 - _globals['_CONNECTIONRESPONSE']._serialized_start=2484 - _globals['_CONNECTIONRESPONSE']._serialized_end=2544 - _globals['_MODELSERVICE']._serialized_start=3013 - _globals['_MODELSERVICE']._serialized_end=3135 - _globals['_CONTROL']._serialized_start=3138 - _globals['_CONTROL']._serialized_end=3386 - _globals['_REDUCER']._serialized_start=3388 - _globals['_REDUCER']._serialized_end=3474 - _globals['_CONNECTOR']._serialized_start=3477 - _globals['_CONNECTOR']._serialized_end=3904 - _globals['_COMBINER']._serialized_start=3907 - _globals['_COMBINER']._serialized_end=4158 + _globals['_MODELPREDICTION']._serialized_start=1078 + _globals['_MODELPREDICTION']._serialized_end=1297 + _globals['_MODELREQUEST']._serialized_start=1300 + _globals['_MODELREQUEST']._serialized_end=1437 + _globals['_MODELRESPONSE']._serialized_start=1439 + _globals['_MODELRESPONSE']._serialized_end=1532 + _globals['_GETGLOBALMODELREQUEST']._serialized_start=1534 + _globals['_GETGLOBALMODELREQUEST']._serialized_end=1619 + _globals['_GETGLOBALMODELRESPONSE']._serialized_start=1621 + _globals['_GETGLOBALMODELRESPONSE']._serialized_end=1725 + _globals['_HEARTBEAT']._serialized_start=1727 + _globals['_HEARTBEAT']._serialized_end=1768 + _globals['_CLIENTAVAILABLEMESSAGE']._serialized_start=1770 + _globals['_CLIENTAVAILABLEMESSAGE']._serialized_end=1857 + _globals['_LISTCLIENTSREQUEST']._serialized_start=1859 + _globals['_LISTCLIENTSREQUEST']._serialized_end=1939 + _globals['_CLIENTLIST']._serialized_start=1941 + _globals['_CLIENTLIST']._serialized_end=1983 + _globals['_CLIENT']._serialized_start=1985 + _globals['_CLIENT']._serialized_end=2052 + _globals['_REASSIGNREQUEST']._serialized_start=2054 + _globals['_REASSIGNREQUEST']._serialized_end=2163 + _globals['_RECONNECTREQUEST']._serialized_start=2165 + _globals['_RECONNECTREQUEST']._serialized_end=2264 + _globals['_PARAMETER']._serialized_start=2266 + _globals['_PARAMETER']._serialized_end=2305 + _globals['_CONTROLREQUEST']._serialized_start=2307 + _globals['_CONTROLREQUEST']._serialized_end=2391 + _globals['_CONTROLRESPONSE']._serialized_start=2393 + _globals['_CONTROLRESPONSE']._serialized_end=2463 + _globals['_CONNECTIONREQUEST']._serialized_start=2465 + _globals['_CONNECTIONREQUEST']._serialized_end=2484 + _globals['_CONNECTIONRESPONSE']._serialized_start=2486 + _globals['_CONNECTIONRESPONSE']._serialized_end=2546 + _globals['_MODELSERVICE']._serialized_start=3015 + _globals['_MODELSERVICE']._serialized_end=3137 + _globals['_CONTROL']._serialized_start=3140 + _globals['_CONTROL']._serialized_end=3388 + _globals['_REDUCER']._serialized_start=3390 + _globals['_REDUCER']._serialized_end=3476 + _globals['_CONNECTOR']._serialized_start=3479 + _globals['_CONNECTOR']._serialized_end=3906 + _globals['_COMBINER']._serialized_start=3909 + _globals['_COMBINER']._serialized_end=4162 # @@protoc_insertion_point(module_scope) From 07eb5add0a95a2be45eb7ebdd81ffc67d0199205 Mon Sep 17 00:00:00 2001 From: Fredrik Wrede Date: Fri, 11 Oct 2024 12:26:08 +0000 Subject: [PATCH 11/17] fix --- fedn/network/api/v1/prediction_routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedn/network/api/v1/prediction_routes.py b/fedn/network/api/v1/prediction_routes.py index 7cbcc0c29..7fb2ac935 100644 --- a/fedn/network/api/v1/prediction_routes.py +++ b/fedn/network/api/v1/prediction_routes.py @@ -26,7 +26,7 @@ def start_session(): prediction_id: str = data.get("prediction_id") if not prediction_id or prediction_id == "": - return jsonify({"message": "Session ID is required"}), 400 + return jsonify({"message": "prediction_id is required"}), 400 session_config = {"prediction_id": prediction_id} From 185dafe69c57ae4359d8a6a2fa14c2358a40d218 Mon Sep 17 00:00:00 2001 From: Fredrik Wrede Date: Mon, 14 Oct 2024 11:51:03 +0000 Subject: [PATCH 12/17] fix if no models have been created --- fedn/network/api/v1/prediction_routes.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/fedn/network/api/v1/prediction_routes.py b/fedn/network/api/v1/prediction_routes.py index 7fb2ac935..798846d88 100644 --- a/fedn/network/api/v1/prediction_routes.py +++ b/fedn/network/api/v1/prediction_routes.py @@ -5,11 +5,13 @@ from fedn.network.api.auth import jwt_auth_required from fedn.network.api.shared import control from fedn.network.api.v1.shared import api_version, mdb +from fedn.network.storage.statestore.stores.model_store import ModelStore from fedn.network.storage.statestore.stores.prediction_store import PredictionStore bp = Blueprint("prediction", __name__, url_prefix=f"/api/{api_version}/predict") prediction_store = PredictionStore(mdb, "control.predictions") +model_store = ModelStore(mdb, "control.model") @bp.route("/start", methods=["POST"]) @@ -28,6 +30,16 @@ def start_session(): if not prediction_id or prediction_id == "": return jsonify({"message": "prediction_id is required"}), 400 + if data.get("model_id") is None: + count = model_store.count() + if count == 0: + return jsonify({"message": "No models available"}), 400 + else: + model_id = data.get("model_id") + model = model_store.get(model_id) + if model is None: + return jsonify({"message": "Model not found"}), 404 + session_config = {"prediction_id": prediction_id} threading.Thread(target=control.prediction_session, kwargs={"config": session_config}).start() From ca95237843089588d25afad6a3d0d2aa8a212de1 Mon Sep 17 00:00:00 2001 From: Fredrik Wrede Date: Mon, 14 Oct 2024 14:31:50 +0000 Subject: [PATCH 13/17] fix multiple comibiners in statestore --- fedn/network/api/v1/prediction_routes.py | 10 ++++++---- fedn/network/combiner/combiner.py | 7 ++++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/fedn/network/api/v1/prediction_routes.py b/fedn/network/api/v1/prediction_routes.py index 798846d88..d5dd804cc 100644 --- a/fedn/network/api/v1/prediction_routes.py +++ b/fedn/network/api/v1/prediction_routes.py @@ -7,6 +7,7 @@ from fedn.network.api.v1.shared import api_version, mdb from fedn.network.storage.statestore.stores.model_store import ModelStore from fedn.network.storage.statestore.stores.prediction_store import PredictionStore +from fedn.network.storage.statestore.stores.shared import EntityNotFound bp = Blueprint("prediction", __name__, url_prefix=f"/api/{api_version}/predict") @@ -35,10 +36,11 @@ def start_session(): if count == 0: return jsonify({"message": "No models available"}), 400 else: - model_id = data.get("model_id") - model = model_store.get(model_id) - if model is None: - return jsonify({"message": "Model not found"}), 404 + try: + model_id = data.get("model_id") + _ = model_store.get(model_id) + except EntityNotFound: + return jsonify({"message": f"Model {model_id} not found"}), 404 session_config = {"prediction_id": prediction_id} diff --git a/fedn/network/combiner/combiner.py b/fedn/network/combiner/combiner.py index ee6cde974..0d6ad244e 100644 --- a/fedn/network/combiner/combiner.py +++ b/fedn/network/combiner/combiner.py @@ -18,6 +18,7 @@ from fedn.network.combiner.roundhandler import RoundConfig, RoundHandler from fedn.network.combiner.shared import client_store, combiner_store, prediction_store, repository, statestore, status_store, validation_store from fedn.network.grpc.server import Server, ServerConfig +from fedn.network.storage.statestore.stores.shared import EntityNotFound VALID_NAME_REGEX = "^[a-zA-Z0-9_-]*$" @@ -116,7 +117,11 @@ def __init__(self, config): "ip": "", "updated_at": str(datetime.now()), } - combiner_store.add(interface_config) + # Check if combiner already exists in statestore + try: + _ = combiner_store.get(config["name"]) + except EntityNotFound: + combiner_store.add(interface_config) # Fetch all clients previously connected to the combiner # If a client and a combiner goes down at the same time, From df9d5f717f848f64baad6bfdbf6bd08e2bebdc6e Mon Sep 17 00:00:00 2001 From: benjaminastrand Date: Mon, 14 Oct 2024 17:09:04 +0200 Subject: [PATCH 14/17] Updates StatusType and ClientStateToString --- .ci/tests/examples/prediction_test.py | 2 +- fedn/network/clients/client.py | 6 ++-- fedn/network/clients/state.py | 2 ++ fedn/network/combiner/combiner.py | 6 ++-- fedn/network/controller/control.py | 2 +- fedn/network/grpc/fedn.proto | 2 +- fedn/network/grpc/fedn_pb2.py | 44 +++++++++++++-------------- 7 files changed, 33 insertions(+), 31 deletions(-) diff --git a/.ci/tests/examples/prediction_test.py b/.ci/tests/examples/prediction_test.py index eed62f784..cd5076e4f 100644 --- a/.ci/tests/examples/prediction_test.py +++ b/.ci/tests/examples/prediction_test.py @@ -15,7 +15,7 @@ def _eprint(*args, **kwargs): def _wait_n_rounds(collection): n = 0 for _ in range(RETRIES): - query = {'type': 'INFERENCE'} + query = {'type': 'MODEL_PREDICTION'} n = collection.count_documents(query) if n == N_CLIENTS: return n diff --git a/fedn/network/clients/client.py b/fedn/network/clients/client.py index 184ec5615..12f76e2a4 100644 --- a/fedn/network/clients/client.py +++ b/fedn/network/clients/client.py @@ -417,7 +417,7 @@ def _listen_to_task_stream(self): self.inbox.put(("train", request)) elif request.type == fedn.StatusType.MODEL_VALIDATION and self.config["validator"]: self.inbox.put(("validate", request)) - elif request.type == fedn.StatusType.INFERENCE and self.config["validator"]: + elif request.type == fedn.StatusType.MODEL_PREDICTION and self.config["validator"]: logger.info("Received prediction request for model_id {}".format(request.model_id)) presigned_url = json.loads(request.data) presigned_url = presigned_url["presigned_url"] @@ -559,7 +559,7 @@ def _process_validation_request(self, model_id: str, session_id: str = None): return validation def _process_prediction_request(self, model_id: str, session_id: str, presigned_url: str): - """Process an prediction request. + """Process a prediction request. :param model_id: The model id of the model to be used for prediction. :type model_id: str @@ -731,7 +731,7 @@ def process_request(self): try: _ = self.combinerStub.SendModelPrediction(prediction, metadata=self.metadata) - status_type = fedn.StatusType.INFERENCE + status_type = fedn.StatusType.MODEL_PREDICTION self.send_status( "Model prediction completed.", log_level=fedn.Status.AUDIT, type=status_type, request=prediction, sesssion_id=request.session_id ) diff --git a/fedn/network/clients/state.py b/fedn/network/clients/state.py index 80b182fc9..678bb578c 100644 --- a/fedn/network/clients/state.py +++ b/fedn/network/clients/state.py @@ -24,5 +24,7 @@ def ClientStateToString(state): return "TRAINING" if state == ClientState.validating: return "VALIDATING" + if state == ClientState.predicting: + return "PREDICTING" return "UNKNOWN" diff --git a/fedn/network/combiner/combiner.py b/fedn/network/combiner/combiner.py index 0d6ad244e..ab90ed905 100644 --- a/fedn/network/combiner/combiner.py +++ b/fedn/network/combiner/combiner.py @@ -225,7 +225,7 @@ def request_model_prediction(self, prediction_id: str, model_id: str, clients: l :type clients: list """ - clients = self._send_request_type(fedn.StatusType.INFERENCE, prediction_id, model_id, clients) + clients = self._send_request_type(fedn.StatusType.MODEL_PREDICTION, prediction_id, model_id, clients) if len(clients) < 20: logger.info("Sent model prediction request for model {} to clients {}".format(model_id, clients)) @@ -253,7 +253,7 @@ def _send_request_type(self, request_type, session_id, model_id, config=None, cl clients = self.get_active_trainers() elif request_type == fedn.StatusType.MODEL_VALIDATION: clients = self.get_active_validators() - elif request_type == fedn.StatusType.INFERENCE: + elif request_type == fedn.StatusType.MODEL_PREDICTION: # TODO: add prediction clients type clients = self.get_active_validators() for client in clients: @@ -269,7 +269,7 @@ def _send_request_type(self, request_type, session_id, model_id, config=None, cl request.receiver.client_id = client request.receiver.role = fedn.WORKER # Set the request data, not used in validation - if request_type == fedn.StatusType.INFERENCE: + if request_type == fedn.StatusType.MODEL_PREDICTION: presigned_url = self.repository.presigned_put_url(self.repository.prediction_bucket, f"{client}/{session_id}") # TODO: in prediction, request.data should also contain user-defined data/parameters request.data = json.dumps({"presigned_url": presigned_url}) diff --git a/fedn/network/controller/control.py b/fedn/network/controller/control.py index a283b7389..129dd1af0 100644 --- a/fedn/network/controller/control.py +++ b/fedn/network/controller/control.py @@ -465,7 +465,7 @@ def predict_instruct(self, config): self.__state = ReducerState.idle def prediction_round(self, config): - """Execute an prediction round. + """Execute a prediction round. : param config: configuration for the prediction round """ diff --git a/fedn/network/grpc/fedn.proto b/fedn/network/grpc/fedn.proto index c3846444c..558b7e67d 100644 --- a/fedn/network/grpc/fedn.proto +++ b/fedn/network/grpc/fedn.proto @@ -15,7 +15,7 @@ enum StatusType { MODEL_UPDATE = 2; MODEL_VALIDATION_REQUEST = 3; MODEL_VALIDATION = 4; - INFERENCE = 5; + MODEL_PREDICTION = 5; } message Status { diff --git a/fedn/network/grpc/fedn_pb2.py b/fedn/network/grpc/fedn_pb2.py index e43facd13..cb637baba 100644 --- a/fedn/network/grpc/fedn_pb2.py +++ b/fedn/network/grpc/fedn_pb2.py @@ -15,7 +15,7 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17network/grpc/fedn.proto\x12\x04\x66\x65\x64n\x1a\x1fgoogle/protobuf/timestamp.proto\":\n\x08Response\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08response\x18\x02 \x01(\t\"\xbc\x02\n\x06Status\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06status\x18\x02 \x01(\t\x12(\n\tlog_level\x18\x03 \x01(\x0e\x32\x15.fedn.Status.LogLevel\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1e\n\x04type\x18\x07 \x01(\x0e\x32\x10.fedn.StatusType\x12\r\n\x05\x65xtra\x18\x08 \x01(\t\x12\x12\n\nsession_id\x18\t \x01(\t\"B\n\x08LogLevel\x12\x08\n\x04INFO\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\t\n\x05\x41UDIT\x10\x04\"\xd8\x01\n\x0bTaskRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\x12\x1e\n\x04type\x18\t \x01(\x0e\x32\x10.fedn.StatusType\"\xbf\x01\n\x0bModelUpdate\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x17\n\x0fmodel_update_id\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x08 \x01(\t\"\xd8\x01\n\x0fModelValidation\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\"\xdb\x01\n\x0fModelPrediction\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x15\n\rprediction_id\x18\x08 \x01(\t\"\x89\x01\n\x0cModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\n\n\x02id\x18\x04 \x01(\t\x12!\n\x06status\x18\x05 \x01(\x0e\x32\x11.fedn.ModelStatus\"]\n\rModelResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\x12!\n\x06status\x18\x03 \x01(\x0e\x32\x11.fedn.ModelStatus\x12\x0f\n\x07message\x18\x04 \x01(\t\"U\n\x15GetGlobalModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\"h\n\x16GetGlobalModelResponse\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\")\n\tHeartbeat\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\"W\n\x16\x43lientAvailableMessage\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\t\"P\n\x12ListClientsRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1c\n\x07\x63hannel\x18\x02 \x01(\x0e\x32\x0b.fedn.Queue\"*\n\nClientList\x12\x1c\n\x06\x63lient\x18\x01 \x03(\x0b\x32\x0c.fedn.Client\"C\n\x06\x43lient\x12\x18\n\x04role\x18\x01 \x01(\x0e\x32\n.fedn.Role\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x11\n\tclient_id\x18\x03 \x01(\t\"m\n\x0fReassignRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06server\x18\x03 \x01(\t\x12\x0c\n\x04port\x18\x04 \x01(\r\"c\n\x10ReconnectRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x11\n\treconnect\x18\x03 \x01(\r\"\'\n\tParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"T\n\x0e\x43ontrolRequest\x12\x1e\n\x07\x63ommand\x18\x01 \x01(\x0e\x32\r.fedn.Command\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"F\n\x0f\x43ontrolResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"\x13\n\x11\x43onnectionRequest\"<\n\x12\x43onnectionResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.fedn.ConnectionStatus*\x84\x01\n\nStatusType\x12\x07\n\x03LOG\x10\x00\x12\x18\n\x14MODEL_UPDATE_REQUEST\x10\x01\x12\x10\n\x0cMODEL_UPDATE\x10\x02\x12\x1c\n\x18MODEL_VALIDATION_REQUEST\x10\x03\x12\x14\n\x10MODEL_VALIDATION\x10\x04\x12\r\n\tINFERENCE\x10\x05*$\n\x05Queue\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x0e\n\nTASK_QUEUE\x10\x01*S\n\x0bModelStatus\x12\x06\n\x02OK\x10\x00\x12\x0f\n\x0bIN_PROGRESS\x10\x01\x12\x12\n\x0eIN_PROGRESS_OK\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\x0b\n\x07UNKNOWN\x10\x04*8\n\x04Role\x12\n\n\x06WORKER\x10\x00\x12\x0c\n\x08\x43OMBINER\x10\x01\x12\x0b\n\x07REDUCER\x10\x02\x12\t\n\x05OTHER\x10\x03*J\n\x07\x43ommand\x12\x08\n\x04IDLE\x10\x00\x12\t\n\x05START\x10\x01\x12\t\n\x05PAUSE\x10\x02\x12\x08\n\x04STOP\x10\x03\x12\t\n\x05RESET\x10\x04\x12\n\n\x06REPORT\x10\x05*I\n\x10\x43onnectionStatus\x12\x11\n\rNOT_ACCEPTING\x10\x00\x12\r\n\tACCEPTING\x10\x01\x12\x13\n\x0fTRY_AGAIN_LATER\x10\x02\x32z\n\x0cModelService\x12\x33\n\x06Upload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse(\x01\x12\x35\n\x08\x44ownload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse0\x01\x32\xf8\x01\n\x07\x43ontrol\x12\x34\n\x05Start\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x33\n\x04Stop\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x44\n\x15\x46lushAggregationQueue\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12<\n\rSetAggregator\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse2V\n\x07Reducer\x12K\n\x0eGetGlobalModel\x12\x1b.fedn.GetGlobalModelRequest\x1a\x1c.fedn.GetGlobalModelResponse2\xab\x03\n\tConnector\x12\x44\n\x14\x41llianceStatusStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x0c.fedn.Status0\x01\x12*\n\nSendStatus\x12\x0c.fedn.Status\x1a\x0e.fedn.Response\x12?\n\x11ListActiveClients\x12\x18.fedn.ListClientsRequest\x1a\x10.fedn.ClientList\x12\x45\n\x10\x41\x63\x63\x65ptingClients\x12\x17.fedn.ConnectionRequest\x1a\x18.fedn.ConnectionResponse\x12\x30\n\rSendHeartbeat\x12\x0f.fedn.Heartbeat\x1a\x0e.fedn.Response\x12\x37\n\x0eReassignClient\x12\x15.fedn.ReassignRequest\x1a\x0e.fedn.Response\x12\x39\n\x0fReconnectClient\x12\x16.fedn.ReconnectRequest\x1a\x0e.fedn.Response2\xfd\x01\n\x08\x43ombiner\x12?\n\nTaskStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x11.fedn.TaskRequest0\x01\x12\x34\n\x0fSendModelUpdate\x12\x11.fedn.ModelUpdate\x1a\x0e.fedn.Response\x12<\n\x13SendModelValidation\x12\x15.fedn.ModelValidation\x1a\x0e.fedn.Response\x12<\n\x13SendModelPrediction\x12\x15.fedn.ModelPrediction\x1a\x0e.fedn.Responseb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17network/grpc/fedn.proto\x12\x04\x66\x65\x64n\x1a\x1fgoogle/protobuf/timestamp.proto\":\n\x08Response\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08response\x18\x02 \x01(\t\"\xbc\x02\n\x06Status\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06status\x18\x02 \x01(\t\x12(\n\tlog_level\x18\x03 \x01(\x0e\x32\x15.fedn.Status.LogLevel\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x1e\n\x04type\x18\x07 \x01(\x0e\x32\x10.fedn.StatusType\x12\r\n\x05\x65xtra\x18\x08 \x01(\t\x12\x12\n\nsession_id\x18\t \x01(\t\"B\n\x08LogLevel\x12\x08\n\x04INFO\x10\x00\x12\t\n\x05\x44\x45\x42UG\x10\x01\x12\x0b\n\x07WARNING\x10\x02\x12\t\n\x05\x45RROR\x10\x03\x12\t\n\x05\x41UDIT\x10\x04\"\xd8\x01\n\x0bTaskRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\x12\x1e\n\x04type\x18\t \x01(\x0e\x32\x10.fedn.StatusType\"\xbf\x01\n\x0bModelUpdate\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x17\n\x0fmodel_update_id\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12\x11\n\ttimestamp\x18\x06 \x01(\t\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x08 \x01(\t\"\xd8\x01\n\x0fModelValidation\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x12\n\nsession_id\x18\x08 \x01(\t\"\xdb\x01\n\x0fModelPrediction\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\t\x12\x16\n\x0e\x63orrelation_id\x18\x05 \x01(\t\x12-\n\ttimestamp\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0c\n\x04meta\x18\x07 \x01(\t\x12\x15\n\rprediction_id\x18\x08 \x01(\t\"\x89\x01\n\x0cModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\n\n\x02id\x18\x04 \x01(\t\x12!\n\x06status\x18\x05 \x01(\x0e\x32\x11.fedn.ModelStatus\"]\n\rModelResponse\x12\x0c\n\x04\x64\x61ta\x18\x01 \x01(\x0c\x12\n\n\x02id\x18\x02 \x01(\t\x12!\n\x06status\x18\x03 \x01(\x0e\x32\x11.fedn.ModelStatus\x12\x0f\n\x07message\x18\x04 \x01(\t\"U\n\x15GetGlobalModelRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\"h\n\x16GetGlobalModelResponse\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x10\n\x08model_id\x18\x03 \x01(\t\")\n\tHeartbeat\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\"W\n\x16\x43lientAvailableMessage\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x0c\n\x04\x64\x61ta\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\t\"P\n\x12ListClientsRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1c\n\x07\x63hannel\x18\x02 \x01(\x0e\x32\x0b.fedn.Queue\"*\n\nClientList\x12\x1c\n\x06\x63lient\x18\x01 \x03(\x0b\x32\x0c.fedn.Client\"C\n\x06\x43lient\x12\x18\n\x04role\x18\x01 \x01(\x0e\x32\n.fedn.Role\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x11\n\tclient_id\x18\x03 \x01(\t\"m\n\x0fReassignRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x0e\n\x06server\x18\x03 \x01(\t\x12\x0c\n\x04port\x18\x04 \x01(\r\"c\n\x10ReconnectRequest\x12\x1c\n\x06sender\x18\x01 \x01(\x0b\x32\x0c.fedn.Client\x12\x1e\n\x08receiver\x18\x02 \x01(\x0b\x32\x0c.fedn.Client\x12\x11\n\treconnect\x18\x03 \x01(\r\"\'\n\tParameter\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"T\n\x0e\x43ontrolRequest\x12\x1e\n\x07\x63ommand\x18\x01 \x01(\x0e\x32\r.fedn.Command\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"F\n\x0f\x43ontrolResponse\x12\x0f\n\x07message\x18\x01 \x01(\t\x12\"\n\tparameter\x18\x02 \x03(\x0b\x32\x0f.fedn.Parameter\"\x13\n\x11\x43onnectionRequest\"<\n\x12\x43onnectionResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.fedn.ConnectionStatus*\x8b\x01\n\nStatusType\x12\x07\n\x03LOG\x10\x00\x12\x18\n\x14MODEL_UPDATE_REQUEST\x10\x01\x12\x10\n\x0cMODEL_UPDATE\x10\x02\x12\x1c\n\x18MODEL_VALIDATION_REQUEST\x10\x03\x12\x14\n\x10MODEL_VALIDATION\x10\x04\x12\x14\n\x10MODEL_PREDICTION\x10\x05*$\n\x05Queue\x12\x0b\n\x07\x44\x45\x46\x41ULT\x10\x00\x12\x0e\n\nTASK_QUEUE\x10\x01*S\n\x0bModelStatus\x12\x06\n\x02OK\x10\x00\x12\x0f\n\x0bIN_PROGRESS\x10\x01\x12\x12\n\x0eIN_PROGRESS_OK\x10\x02\x12\n\n\x06\x46\x41ILED\x10\x03\x12\x0b\n\x07UNKNOWN\x10\x04*8\n\x04Role\x12\n\n\x06WORKER\x10\x00\x12\x0c\n\x08\x43OMBINER\x10\x01\x12\x0b\n\x07REDUCER\x10\x02\x12\t\n\x05OTHER\x10\x03*J\n\x07\x43ommand\x12\x08\n\x04IDLE\x10\x00\x12\t\n\x05START\x10\x01\x12\t\n\x05PAUSE\x10\x02\x12\x08\n\x04STOP\x10\x03\x12\t\n\x05RESET\x10\x04\x12\n\n\x06REPORT\x10\x05*I\n\x10\x43onnectionStatus\x12\x11\n\rNOT_ACCEPTING\x10\x00\x12\r\n\tACCEPTING\x10\x01\x12\x13\n\x0fTRY_AGAIN_LATER\x10\x02\x32z\n\x0cModelService\x12\x33\n\x06Upload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse(\x01\x12\x35\n\x08\x44ownload\x12\x12.fedn.ModelRequest\x1a\x13.fedn.ModelResponse0\x01\x32\xf8\x01\n\x07\x43ontrol\x12\x34\n\x05Start\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x33\n\x04Stop\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12\x44\n\x15\x46lushAggregationQueue\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse\x12<\n\rSetAggregator\x12\x14.fedn.ControlRequest\x1a\x15.fedn.ControlResponse2V\n\x07Reducer\x12K\n\x0eGetGlobalModel\x12\x1b.fedn.GetGlobalModelRequest\x1a\x1c.fedn.GetGlobalModelResponse2\xab\x03\n\tConnector\x12\x44\n\x14\x41llianceStatusStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x0c.fedn.Status0\x01\x12*\n\nSendStatus\x12\x0c.fedn.Status\x1a\x0e.fedn.Response\x12?\n\x11ListActiveClients\x12\x18.fedn.ListClientsRequest\x1a\x10.fedn.ClientList\x12\x45\n\x10\x41\x63\x63\x65ptingClients\x12\x17.fedn.ConnectionRequest\x1a\x18.fedn.ConnectionResponse\x12\x30\n\rSendHeartbeat\x12\x0f.fedn.Heartbeat\x1a\x0e.fedn.Response\x12\x37\n\x0eReassignClient\x12\x15.fedn.ReassignRequest\x1a\x0e.fedn.Response\x12\x39\n\x0fReconnectClient\x12\x16.fedn.ReconnectRequest\x1a\x0e.fedn.Response2\xfd\x01\n\x08\x43ombiner\x12?\n\nTaskStream\x12\x1c.fedn.ClientAvailableMessage\x1a\x11.fedn.TaskRequest0\x01\x12\x34\n\x0fSendModelUpdate\x12\x11.fedn.ModelUpdate\x1a\x0e.fedn.Response\x12<\n\x13SendModelValidation\x12\x15.fedn.ModelValidation\x1a\x0e.fedn.Response\x12<\n\x13SendModelPrediction\x12\x15.fedn.ModelPrediction\x1a\x0e.fedn.Responseb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -23,17 +23,17 @@ if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None _globals['_STATUSTYPE']._serialized_start=2549 - _globals['_STATUSTYPE']._serialized_end=2681 - _globals['_QUEUE']._serialized_start=2683 - _globals['_QUEUE']._serialized_end=2719 - _globals['_MODELSTATUS']._serialized_start=2721 - _globals['_MODELSTATUS']._serialized_end=2804 - _globals['_ROLE']._serialized_start=2806 - _globals['_ROLE']._serialized_end=2862 - _globals['_COMMAND']._serialized_start=2864 - _globals['_COMMAND']._serialized_end=2938 - _globals['_CONNECTIONSTATUS']._serialized_start=2940 - _globals['_CONNECTIONSTATUS']._serialized_end=3013 + _globals['_STATUSTYPE']._serialized_end=2688 + _globals['_QUEUE']._serialized_start=2690 + _globals['_QUEUE']._serialized_end=2726 + _globals['_MODELSTATUS']._serialized_start=2728 + _globals['_MODELSTATUS']._serialized_end=2811 + _globals['_ROLE']._serialized_start=2813 + _globals['_ROLE']._serialized_end=2869 + _globals['_COMMAND']._serialized_start=2871 + _globals['_COMMAND']._serialized_end=2945 + _globals['_CONNECTIONSTATUS']._serialized_start=2947 + _globals['_CONNECTIONSTATUS']._serialized_end=3020 _globals['_RESPONSE']._serialized_start=66 _globals['_RESPONSE']._serialized_end=124 _globals['_STATUS']._serialized_start=127 @@ -80,14 +80,14 @@ _globals['_CONNECTIONREQUEST']._serialized_end=2484 _globals['_CONNECTIONRESPONSE']._serialized_start=2486 _globals['_CONNECTIONRESPONSE']._serialized_end=2546 - _globals['_MODELSERVICE']._serialized_start=3015 - _globals['_MODELSERVICE']._serialized_end=3137 - _globals['_CONTROL']._serialized_start=3140 - _globals['_CONTROL']._serialized_end=3388 - _globals['_REDUCER']._serialized_start=3390 - _globals['_REDUCER']._serialized_end=3476 - _globals['_CONNECTOR']._serialized_start=3479 - _globals['_CONNECTOR']._serialized_end=3906 - _globals['_COMBINER']._serialized_start=3909 - _globals['_COMBINER']._serialized_end=4162 + _globals['_MODELSERVICE']._serialized_start=3022 + _globals['_MODELSERVICE']._serialized_end=3144 + _globals['_CONTROL']._serialized_start=3147 + _globals['_CONTROL']._serialized_end=3395 + _globals['_REDUCER']._serialized_start=3397 + _globals['_REDUCER']._serialized_end=3483 + _globals['_CONNECTOR']._serialized_start=3486 + _globals['_CONNECTOR']._serialized_end=3913 + _globals['_COMBINER']._serialized_start=3916 + _globals['_COMBINER']._serialized_end=4169 # @@protoc_insertion_point(module_scope) From 197dd96683eb80f6ed21d67405c53d44dc91d53a Mon Sep 17 00:00:00 2001 From: Fredrik Wrede Date: Wed, 16 Oct 2024 12:09:29 +0000 Subject: [PATCH 15/17] add logging for when client disconnect from TaskStream --- fedn/network/combiner/combiner.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fedn/network/combiner/combiner.py b/fedn/network/combiner/combiner.py index ab90ed905..3f732ecd4 100644 --- a/fedn/network/combiner/combiner.py +++ b/fedn/network/combiner/combiner.py @@ -650,6 +650,7 @@ def TaskStream(self, response, context): logger.info("grpc.Combiner.TaskStream: Client connected: {}\n".format(metadata["client"])) status = fedn.Status(status="Client {} connecting to TaskStream.".format(client.name)) + logger.info("Client {} connecting to TaskStream.".format(client.name)) status.log_level = fedn.Status.INFO status.timestamp.GetCurrentTime() @@ -702,6 +703,11 @@ def TaskStream(self, response, context): pass except Exception as e: logger.error("Error in ModelUpdateRequestStream: {}".format(e)) + logger.warning("Client {} disconnected from TaskStream".format(client.name)) + status = fedn.Status(status="Client {} disconnected from TaskStream.".format(client.name)) + status.log_level = fedn.Status.INFO + status.timestamp.GetCurrentTime() + self._send_status(status) def SendModelUpdate(self, request, context): """Send a model update response. From c45477b2655297c56ce3e0101e2f5158d3e50982 Mon Sep 17 00:00:00 2001 From: Fredrik Wrede Date: Thu, 17 Oct 2024 13:46:04 +0000 Subject: [PATCH 16/17] test server keepalive settings --- fedn/network/grpc/server.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/fedn/network/grpc/server.py b/fedn/network/grpc/server.py index edd2fd6d5..8523f4a64 100644 --- a/fedn/network/grpc/server.py +++ b/fedn/network/grpc/server.py @@ -26,7 +26,20 @@ def __init__(self, servicer, config: ServerConfig): set_log_level_from_string(config.get("verbosity", "INFO")) set_log_stream(config.get("logfile", None)) - self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=350), interceptors=[JWTInterceptor()]) + # Keepalive settings: these detect if the client is alive + KEEPALIVE_TIME_MS = 60 * 1000 # send keepalive ping every 60 seconds + KEEPALIVE_TIMEOUT_MS = 20 * 1000 # wait 20 seconds for keepalive ping ack before considering connection dead + MAX_CONNECTION_IDLE_MS = 5 * 60 * 1000 # max idle time before server terminates the connection (5 minutes) + + self.server = grpc.server( + futures.ThreadPoolExecutor(max_workers=350), + interceptors=[JWTInterceptor()], + options=[ + ("grpc.keepalive_time_ms", KEEPALIVE_TIME_MS), + ("grpc.keepalive_timeout_ms", KEEPALIVE_TIMEOUT_MS), + ("grpc.max_connection_idle_ms", MAX_CONNECTION_IDLE_MS), + ], + ) self.certificate = None self.health_servicer = health.HealthServicer() From 2bd2cacca46542c5e5c76186f2409574319c53fa Mon Sep 17 00:00:00 2001 From: Fredrik Wrede Date: Fri, 18 Oct 2024 08:25:37 +0000 Subject: [PATCH 17/17] add keepalive to client --- fedn/network/clients/grpc_handler.py | 58 ++++++++++++---------------- 1 file changed, 25 insertions(+), 33 deletions(-) diff --git a/fedn/network/clients/grpc_handler.py b/fedn/network/clients/grpc_handler.py index 9b8550344..4d2b5d569 100644 --- a/fedn/network/clients/grpc_handler.py +++ b/fedn/network/clients/grpc_handler.py @@ -25,6 +25,7 @@ def __init__(self, key): def __call__(self, context, callback): callback((("authorization", f"{FEDN_AUTH_SCHEME} {self._key}"),), None) + def _get_ssl_certificate(domain, port=443): context = SSL.Context(SSL.TLSv1_2_METHOD) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -39,6 +40,7 @@ def _get_ssl_certificate(domain, port=443): cert = cert.to_cryptography().public_bytes(Encoding.PEM).decode() return cert + class GrpcHandler: def __init__(self, host: str, port: int, name: str, token: str, combiner_name: str): self.metadata = [ @@ -59,6 +61,11 @@ def _init_secure_channel(self, host: str, port: int, token: str): url = f"{host}:{port}" logger.info(f"Connecting (GRPC) to {url}") + # Keepalive settings: these help keep the connection open for long-lived clients + KEEPALIVE_TIME_MS = 60 * 1000 # send keepalive ping every 60 seconds + KEEPALIVE_TIMEOUT_MS = 20 * 1000 # wait 20 seconds for keepalive ping ack before considering connection dead + KEEPALIVE_PERMIT_WITHOUT_CALLS = True # allow keepalive pings even when there are no RPCs + if os.getenv("FEDN_GRPC_ROOT_CERT_PATH"): logger.info("Using root certificate from environment variable for GRPC channel.") with open(os.environ["FEDN_GRPC_ROOT_CERT_PATH"], "rb") as f: @@ -70,7 +77,16 @@ def _init_secure_channel(self, host: str, port: int, token: str): cert = _get_ssl_certificate(host, port) credentials = grpc.ssl_channel_credentials(cert.encode("utf-8")) auth_creds = grpc.metadata_call_credentials(GrpcAuth(token)) - self.channel = grpc.secure_channel("{}:{}".format(host, str(port)), grpc.composite_channel_credentials(credentials, auth_creds)) + self.channel = grpc.secure_channel( + "{}:{}".format(host, str(port)), + grpc.composite_channel_credentials(credentials, auth_creds), + options=[ + ("grpc.keepalive_time_ms", KEEPALIVE_TIME_MS), + ("grpc.keepalive_timeout_ms", KEEPALIVE_TIMEOUT_MS), + ("grpc.keepalive_permit_without_calls", KEEPALIVE_PERMIT_WITHOUT_CALLS), + ("grpc.http2.max_pings_without_data", 0), # unlimited pings without data + ], + ) def _init_insecure_channel(self, host: str, port: int): url = f"{host}:{port}" @@ -115,7 +131,7 @@ def listen_to_task_stream(self, client_name: str, client_id: str, callback: Call type=fedn.StatusType.MODEL_UPDATE_REQUEST, request=request, sesssion_id=request.session_id, - sender_name=client_name + sender_name=client_name, ) logger.info(f"Received task request of type {request.type} for model_id {request.model_id}") @@ -234,7 +250,8 @@ def send_model_to_combiner(self, model: BytesIO, id: str): return result - def send_model_update(self, + def send_model_update( + self, sender_name: str, sender_role: fedn.Role, client_id: str, @@ -242,7 +259,7 @@ def send_model_update(self, model_update_id: str, receiver_name: str, receiver_role: fedn.Role, - meta: dict + meta: dict, ): update = fedn.ModelUpdate() update.sender.name = sender_name @@ -260,17 +277,7 @@ def send_model_update(self, _ = self.combinerStub.SendModelUpdate(update, metadata=self.metadata) except grpc.RpcError as e: return self._handle_grpc_error( - e, - "SendModelUpdate", - lambda: self.send_model_update( - sender_name, - sender_role, - model_id, - model_update_id, - receiver_name, - receiver_role, - meta - ) + e, "SendModelUpdate", lambda: self.send_model_update(sender_name, sender_role, model_id, model_update_id, receiver_name, receiver_role, meta) ) except Exception as e: logger.error(f"GRPC (SendModelUpdate): An error occurred: {e}") @@ -278,14 +285,8 @@ def send_model_update(self, return True - def send_model_validation(self, - sender_name: str, - receiver_name: str, - receiver_role: fedn.Role, - model_id: str, - metrics: str, - correlation_id: str, - session_id: str + def send_model_validation( + self, sender_name: str, receiver_name: str, receiver_role: fedn.Role, model_id: str, metrics: str, correlation_id: str, session_id: str ) -> bool: validation = fedn.ModelValidation() validation.sender.name = sender_name @@ -298,7 +299,6 @@ def send_model_validation(self, validation.correlation_id = correlation_id validation.session_id = session_id - try: logger.info("Sending model validation to combiner.") _ = self.combinerStub.SendModelValidation(validation, metadata=self.metadata) @@ -306,15 +306,7 @@ def send_model_validation(self, return self._handle_grpc_error( e, "SendModelValidation", - lambda: self.send_model_validation( - sender_name, - receiver_name, - receiver_role, - model_id, - metrics, - correlation_id, - session_id - ) + lambda: self.send_model_validation(sender_name, receiver_name, receiver_role, model_id, metrics, correlation_id, session_id), ) except Exception as e: logger.error(f"GRPC (SendModelValidation): An error occurred: {e}")