From 730671a29c26884bb57bc7f3643ca3e8e30dead4 Mon Sep 17 00:00:00 2001 From: Romain Cledat Date: Thu, 28 Sep 2023 12:45:55 -0700 Subject: [PATCH] Fix some docstrings and add Add To Current --- metaflow/client/core.py | 7 +++- metaflow/current.py | 6 +-- metaflow/events.py | 23 ++++++----- metaflow/includefile.py | 4 +- metaflow/plugins/cards/card_decorator.py | 17 +++++++- metaflow/plugins/cards/card_modules/card.py | 2 +- metaflow/plugins/events_decorator.py | 32 +++++++++++---- metaflow/plugins/project_decorator.py | 45 ++++++++++++++++++++- 8 files changed, 109 insertions(+), 27 deletions(-) diff --git a/metaflow/client/core.py b/metaflow/client/core.py index b1d9ae693d8..f97e5dae9d9 100644 --- a/metaflow/client/core.py +++ b/metaflow/client/core.py @@ -1438,7 +1438,10 @@ def _get_logsize(self, stream): return self._log_size(stream, meta_dict) def loglines( - self, stream: str, as_unicode: bool = True, meta_dict: Dict[str, Any] = None + self, + stream: str, + as_unicode: bool = True, + meta_dict: Optional[Dict[str, Any]] = None, ) -> Iterable[Tuple[datetime, str]]: """ Return an iterator over (utc_timestamp, logline) tuples. @@ -1453,7 +1456,7 @@ def loglines( Returns ------- - Iterable[(datetime, str)] + Iterable[Tuple[datetime, str]] Iterator over timestamp, logline pairs. """ from metaflow.mflog.mflog import merge_logs diff --git a/metaflow/current.py b/metaflow/current.py index a4b959bdee3..057b66adb0d 100644 --- a/metaflow/current.py +++ b/metaflow/current.py @@ -1,14 +1,14 @@ from collections import namedtuple import os -from typing import Any, Optional +from typing import Any, Optional, TYPE_CHECKING from metaflow.metaflow_config import TEMPDIR Parallel = namedtuple("Parallel", ["main_ip", "num_nodes", "node_index"]) # Can add this if we are ok with 3.5.2+ -# if typing.TYPE_CHECKING: -# from metaflow.client.core import Run, Task +if TYPE_CHECKING: + from metaflow import Run, Task class Current(object): diff --git a/metaflow/events.py b/metaflow/events.py index 70746827ed0..aa5eb9bae49 100644 --- a/metaflow/events.py +++ b/metaflow/events.py @@ -1,6 +1,11 @@ from collections import OrderedDict, namedtuple from datetime import datetime +from typing import List, Optional, TYPE_CHECKING, Union + +if TYPE_CHECKING: + from metaflow import Run + MetaflowEvent = namedtuple("MetaflowEvent", ["name", "id", "timestamp", "type"]) MetaflowEvent.__doc__ = """ Container of metadata that identifies the event that triggered @@ -50,7 +55,7 @@ def __init__(self, _meta=None): ] @classmethod - def from_runs(cls, run_objs): + def from_runs(cls, run_objs: List["Run"]): run_objs.sort(key=lambda x: x.finished_at, reverse=True) trigger = Trigger( [ @@ -67,7 +72,7 @@ def from_runs(cls, run_objs): return trigger @property - def event(self): + def event(self) -> Optional[MetaflowEvent]: """ The `MetaflowEvent` object corresponding to the triggering event. @@ -81,7 +86,7 @@ def event(self): return next(iter(self._events), None) @property - def events(self): + def events(self) -> Optional[List[MetaflowEvent]]: """ The list of `MetaflowEvent` objects correspondings to all the triggering events. @@ -93,7 +98,7 @@ def events(self): return list(self._events) or None @property - def run(self): + def run(self) -> Optional["Run"]: """ The corresponding `Run` object if the triggering event is a Metaflow run. @@ -110,7 +115,7 @@ def run(self): return next(iter(self._runs), None) @property - def runs(self): + def runs(self) -> Optional[List["Run"]]: """ The list of `Run` objects in the triggering events. Returns `None` if none of the triggering events are `Run` objects. @@ -136,14 +141,14 @@ def runs(self): return list(self._runs) or None - def __getitem__(self, key): + def __getitem__(self, key: str) -> Union["Run", MetaflowEvent]: """ If triggering events are runs, `key` corresponds to the flow name of the triggering run. Otherwise, `key` corresponds to the event name and a `MetaflowEvent` object is returned. Returns ------- - Run or MetaflowEvent + Union[Run, MetaflowEvent] `Run` object if triggered by a run. Otherwise returns a `MetaflowEvent`. """ if self.runs: @@ -161,8 +166,8 @@ def __iter__(self): return iter(self.events) return iter([]) - def __contains__(self, id): + def __contains__(self, ident: str) -> bool: try: - return bool(self.__getitem__(id)) + return bool(self.__getitem__(ident)) except KeyError: return False diff --git a/metaflow/includefile.py b/metaflow/includefile.py index 140077c4955..1cdb9e249cd 100644 --- a/metaflow/includefile.py +++ b/metaflow/includefile.py @@ -6,7 +6,7 @@ import os from hashlib import sha1 -from typing import Any, Dict, Optional +from typing import Any, Callable, Dict, Optional from metaflow._vendor import click @@ -246,7 +246,7 @@ class IncludeFile(Parameter): ---------- name : str User-visible parameter name. - default : str or a function + default : Union[str, Callable[ParameterContext, str]] Default path to a local file. A function implies that the parameter corresponds to a *deploy-time parameter*. is_text : bool, default: True diff --git a/metaflow/plugins/cards/card_decorator.py b/metaflow/plugins/cards/card_decorator.py index efa13a1ec90..09fd54958b4 100644 --- a/metaflow/plugins/cards/card_decorator.py +++ b/metaflow/plugins/cards/card_decorator.py @@ -40,6 +40,21 @@ class CardDecorator(StepDecorator): Options passed to the card. The contents depend on the card type. timeout : int, default: 45 Interrupt reporting if it takes more than this many seconds. + + MF Add To Current + ----------------- + card -> metaflow.plugins.cards.component_serializer.CardComponentCollector + The `@card` decorator makes the cards available through the `current.card` + object. If multiple `@card` decorators are present, you can add an `ID` to + distinguish between them using `@card(id=ID)` as the decorator. You will then + be able to access that specific card using `current.card[ID]. + + Methods available are `append` and `extend` + + Returns + ------- + CardComponentCollector + The or one of the cards attached to this step. """ name = "card" @@ -89,7 +104,6 @@ def _increment_step_counter(cls): def step_init( self, flow, graph, step_name, decorators, environment, flow_datastore, logger ): - self._flow_datastore = flow_datastore self._environment = environment self._logger = logger @@ -200,7 +214,6 @@ def _options(mapping): yield to_unicode(value) def _create_top_level_args(self): - top_level_options = { "quiet": True, "metadata": self._metadata.TYPE, diff --git a/metaflow/plugins/cards/card_modules/card.py b/metaflow/plugins/cards/card_modules/card.py index 35a639299dd..f0d8d7d5706 100644 --- a/metaflow/plugins/cards/card_modules/card.py +++ b/metaflow/plugins/cards/card_modules/card.py @@ -49,7 +49,7 @@ def _get_mustache(self): except ImportError: return None - def render(self, task) -> str: + def render(self, task: "Task") -> str: """ Produce custom card contents in HTML. diff --git a/metaflow/plugins/events_decorator.py b/metaflow/plugins/events_decorator.py index f64572f31e5..679bb60e867 100644 --- a/metaflow/plugins/events_decorator.py +++ b/metaflow/plugins/events_decorator.py @@ -1,5 +1,3 @@ -import json -import time import re from metaflow import current @@ -45,12 +43,22 @@ class TriggerDecorator(FlowDecorator): Parameters ---------- - event : Union[str, dict], optional + event : Union[str, Dict[str, Any]], optional Event dependency for this flow. - events : List[Union[str, dict]], optional + events : List[Union[str, Dict[str, Any]]], optional Events dependency for this flow. - options : dict, optional + options : Dict[str, Any], optional Backend-specific configuration for tuning eventing behavior. + + MF Add To Current + ----------------- + trigger -> metaflow.events.Trigger + Returns `Trigger` if the current run is triggered by an event + + Returns + ------- + Trigger + `Trigger` if triggered by an event """ name = "trigger" @@ -221,10 +229,20 @@ class TriggerOnFinishDecorator(FlowDecorator): ---------- flow : Union[str, Dict[str, str]], optional Upstream flow dependency for this flow. - flows : List[Union[str, Dict[str, str]], optional + flows : List[Union[str, Dict[str, str]]], optional Upstream flow dependencies for this flow. - options : dict, optional + options : Dict[str, Any], optional Backend-specific configuration for tuning eventing behavior. + + MF Add To Current + ----------------- + trigger -> metaflow.events.Trigger + Returns `Trigger` if the current run is triggered by an event + + Returns + ------- + Trigger + `Trigger` if triggered by an event """ name = "trigger_on_finish" diff --git a/metaflow/plugins/project_decorator.py b/metaflow/plugins/project_decorator.py index 37ff93efe6b..6567bac6d87 100644 --- a/metaflow/plugins/project_decorator.py +++ b/metaflow/plugins/project_decorator.py @@ -25,6 +25,50 @@ class ProjectDecorator(FlowDecorator): Project name. Make sure that the name is unique amongst all projects that use the same production scheduler. The name may contain only lowercase alphanumeric characters and underscores. + + MF Add To Current + ----------------- + project_name -> str + The name of the project assigned to this flow, i.e. `X` in `@project(name=X)`. + + Returns + ------- + str + Project name. + + project_flow_name -> str + The flow name prefixed with the current project and branch. This name identifies + the deployment on a production scheduler. + + Returns + ------- + str + Flow name prefixed with project information. + + branch_name -> str + The current branch, i.e. `X` in `--branch=X` set during deployment or run. + + Returns + ------- + str + Branch name. + + is_user_branch -> bool + True if the flow is deployed without a specific `--branch` or a `--production` + flag. + + Returns + ------- + bool + True if the deployment does not correspond to a specific branch. + + is_production -> bool + True if the flow is deployed with the `--production` flag + + Returns + ------- + bool + True if the flow is deployed with `--production`. """ name = "project" @@ -83,7 +127,6 @@ def get_top_level_options(self): def format_name(flow_name, project_name, deploy_prod, given_branch, user_name): - if not project_name: # an empty string is not a valid project name raise MetaflowException(