Initial commit for logging all metaflow exceptions

Netflix · Oct 17, 2024 · 758579b · 758579b
1 parent b23f135
commit 758579b
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 0 deletions.
diff --git a/metaflow/cli.py b/metaflow/cli.py
@@ -974,6 +974,13 @@ def start(
     ctx.obj.monitor.start()
     _system_monitor.init_system_monitor(ctx.obj.flow.name, ctx.obj.monitor)
 
+    current._update_env(
+        {
+            "system_logger": _system_logger,
+            "system_monitor": _system_monitor,
+        }
+    )
+
     ctx.obj.metadata = [m for m in METADATA_PROVIDERS if m.TYPE == metadata][0](
         ctx.obj.environment, ctx.obj.flow, ctx.obj.event_logger, ctx.obj.monitor
     )

diff --git a/metaflow/exception.py b/metaflow/exception.py
@@ -46,6 +46,22 @@ class MetaflowException(Exception):
     def __init__(self, msg="", lineno=None):
         self.message = msg
         self.line_no = lineno
+
+        # We can do this check only if a config like METAFLOW_LOG_EXCEPTIONS is set
+        from metaflow import current
+
+        if hasattr(current, "system_logger") and hasattr(current, "system_monitor"):
+            # Wait for the system logger and monitor to be initialized
+            from metaflow.system import _system_logger, _system_monitor
+
+            with _system_monitor.count("metaflow.exception"):
+                _system_logger.log_event(
+                    level="error",
+                    module="metaflow.exception",
+                    name=self.__class__.__name__,
+                    payload={"msg": msg},
+                )
+
         super(MetaflowException, self).__init__()
 
     def __str__(self):

diff --git a/metaflow/system/system_logger.py b/metaflow/system/system_logger.py
@@ -38,11 +38,18 @@ def _init_logger_outside_flow(self):
         from .system_utils import init_environment_outside_flow
         from metaflow.plugins import LOGGING_SIDECARS
         from metaflow.metaflow_config import DEFAULT_EVENT_LOGGER
+        from metaflow.metaflow_current import current
 
         self._flow_name = "not_a_real_flow"
         _flow = DummyFlow(self._flow_name)
         _environment = init_environment_outside_flow(_flow)
         _logger = LOGGING_SIDECARS[DEFAULT_EVENT_LOGGER](_flow, _environment)
+
+        current._update_env(
+            {
+                "system_logger": self,
+            }
+        )
         return _logger
 
     @property

diff --git a/metaflow/system/system_monitor.py b/metaflow/system/system_monitor.py
@@ -49,11 +49,17 @@ def _init_system_monitor_outside_flow(self):
         from .system_utils import init_environment_outside_flow
         from metaflow.plugins import MONITOR_SIDECARS
         from metaflow.metaflow_config import DEFAULT_MONITOR
+        from metaflow.metaflow_current import current
 
         self._flow_name = "not_a_real_flow"
         _flow = DummyFlow(self._flow_name)
         _environment = init_environment_outside_flow(_flow)
         _monitor = MONITOR_SIDECARS[DEFAULT_MONITOR](_flow, _environment)
+        current._update_env(
+            {
+                "system_monitor": self,
+            }
+        )
         return _monitor
 
     @property