[card][tests] verify the behavior of .refresh / .components

Netflix · Oct 25, 2023 · 83d7347 · 83d7347
1 parent 2c4a009
commit 83d7347
Show file tree

Hide file tree

Showing 11 changed files with 553 additions and 6 deletions.
diff --git a/metaflow/plugins/__init__.py b/metaflow/plugins/__init__.py
@@ -166,6 +166,8 @@ def get_plugin_cli():
     TestNonEditableCard,
     TestPathSpecCard,
     TestTimeoutCard,
+    TestRefreshCard,
+    TestRefreshComponentCard,
 )
 
 CARDS = [
@@ -182,5 +184,7 @@ def get_plugin_cli():
     TestNonEditableCard,
     BlankCard,
     DefaultCardJSON,
+    TestRefreshCard,
+    TestRefreshComponentCard,
 ]
 merge_lists(CARDS, MF_EXTERNAL_CARDS, "type")
diff --git a/metaflow/plugins/cards/card_cli.py b/metaflow/plugins/cards/card_cli.py
@@ -703,6 +703,13 @@ def create(
                 )
                 rendered_content = rendered_info.data
             except:
+                rendered_info = CardRenderInfo(
+                    mode=mode,
+                    is_implemented=True,
+                    data=None,
+                    timed_out=False,
+                    timeout_stack_trace=None,
+                )
                 if render_error_card:
                     error_stack_trace = str(UnrenderableCardException(type, options))
                 else:

diff --git a/metaflow/plugins/cards/card_modules/test_cards.py b/metaflow/plugins/cards/card_modules/test_cards.py
@@ -1,13 +1,20 @@
+import json
 from .card import MetaflowCard, MetaflowCardComponent
+from .renderer_tools import render_safely
 
 
 class TestStringComponent(MetaflowCardComponent):
+    REALTIME_UPDATABLE = True
+
     def __init__(self, text):
         self._text = text
 
     def render(self):
         return str(self._text)
 
+    def update(self, text):
+        self._text = text
+
 
 class TestPathSpecCard(MetaflowCard):
     type = "test_pathspec_card"
@@ -98,3 +105,110 @@ def render(self, task):
 
         time.sleep(self._timeout)
         return "%s" % task.pathspec
+
+
+REFRESHABLE_HTML_TEMPLATE = """
+<html>
+<script> 
+var METAFLOW_RELOAD_TOKEN = "[METAFLOW_RELOAD_TOKEN]"
+
+window.metaflow_card_update = function(data) {
+    document.querySelector("h1").innerHTML = JSON.stringify(data);
+}
+</script>
+<h1>[PATHSPEC]</h1>
+<h1>[REPLACE_CONTENT_HERE]</h1>
+</html>
+"""
+
+
+class TestJSONComponent(MetaflowCardComponent):
+
+    REALTIME_UPDATABLE = True
+
+    def __init__(self, data):
+        self._data = data
+
+    @render_safely
+    def render(self):
+        return self._data
+
+    def update(self, data):
+        self._data = data
+
+
+class TestRefreshCard(MetaflowCard):
+
+    """
+    This card takes no components and helps test the `current.card.refresh(data)` interface.
+    """
+
+    HTML_TEMPLATE = REFRESHABLE_HTML_TEMPLATE
+
+    RUNTIME_UPDATABLE = True
+
+    ALLOW_USER_COMPONENTS = True
+
+    # Not implementing Reload Policy here since the reload Policy is set to always
+    RELOAD_POLICY = MetaflowCard.RELOAD_POLICY_ALWAYS
+
+    type = "test_refresh_card"
+
+    def render(self, task, data) -> str:
+        return self.HTML_TEMPLATE.replace(
+            "[REPLACE_CONTENT_HERE]", json.dumps(data["user"])
+        ).replace("[PATHSPEC]", task.pathspec)
+
+    def render_runtime(self, task, data):
+        return self.render(task, data)
+
+    def refresh(self, task, data):
+        return data
+
+
+import hashlib
+
+
+def _component_values_to_hash(components):
+    comma_str = ",".join(["".join(x) for v in components.values() for x in v])
+    return hashlib.sha256(comma_str.encode("utf-8")).hexdigest()
+
+
+class TestRefreshComponentCard(MetaflowCard):
+
+    """
+    This card takes components and helps test the `current.card.components["A"].update()`
+    interface
+    """
+
+    HTML_TEMPLATE = REFRESHABLE_HTML_TEMPLATE
+
+    RUNTIME_UPDATABLE = True
+
+    ALLOW_USER_COMPONENTS = True
+
+    # Not implementing Reload Policy here since the reload Policy is set to always
+    RELOAD_POLICY = MetaflowCard.RELOAD_POLICY_ONCHANGE
+
+    type = "test_component_refresh_card"
+
+    def __init__(self, options={}, components=[], graph=None):
+        self._components = components
+
+    def render(self, task, data) -> str:
+        # Calling `render`/`render_runtime` wont require the `data` object
+        return self.HTML_TEMPLATE.replace(
+            "[REPLACE_CONTENT_HERE]", json.dumps(self._components)
+        ).replace("[PATHSPEC]", task.pathspec)
+
+    def render_runtime(self, task, data):
+        return self.render(task, data)
+
+    def refresh(self, task, data):
+        # Govers the information passed in the data update
+        return data["components"]
+
+    def reload_content_token(self, task, data):
+        if task.finished:
+            return "final"
+        return "runtime-%s" % _component_values_to_hash(data["components"])
diff --git a/test/core/contexts.json b/test/core/contexts.json
@@ -27,7 +27,39 @@
             "checks": [ "python3-cli", "python3-metadata"],
             "disabled_tests": [
                 "LargeArtifactTest",
-                "S3FailureTest"
+                "S3FailureTest",
+                "CardComponentRefreshTest",
+                "CardWithRefreshTest"
+            ]
+        },
+        {
+            "name": "python3-all-local-cards-realtime",
+            "disabled": true,
+            "env": {
+                "METAFLOW_USER": "tester",
+                "METAFLOW_RUN_BOOL_PARAM": "False",
+                "METAFLOW_RUN_NO_DEFAULT_PARAM": "test_str",
+                "METAFLOW_DEFAULT_METADATA": "local"
+            },
+            "python": "python3",
+            "top_options": [
+                "--metadata=local",
+                "--datastore=local",
+                "--environment=local",
+                "--event-logger=nullSidecarLogger",
+                "--no-pylint",
+                "--quiet"
+            ],
+            "run_options": [
+                "--max-workers", "50",
+                "--max-num-splits", "10000",
+                "--tag", "\u523a\u8eab means sashimi",
+                "--tag", "multiple tags should be ok"
+            ],
+            "checks": [ "python3-cli", "python3-metadata"],
+            "enabled_tests": [
+                "CardComponentRefreshTest",
+                "CardWithRefreshTest"
             ]
         },
         {

diff --git a/test/core/graphs/branch.json b/test/core/graphs/branch.json
@@ -2,8 +2,8 @@
     "name": "single-and-branch",
     "graph": {
         "start": {"branch": ["a", "b"], "quals": ["split-and"]},
-        "a": {"linear": "join"},
-        "b": {"linear": "join"},
+        "a": {"linear": "join", "quals": ["single-branch-split"]},
+        "b": {"linear": "join", "quals": ["single-branch-split"]},
         "join": {"linear": "end", "join": true, "quals": ["join-and"]},
         "end": {}
     }

diff --git a/test/core/metaflow_test/__init__.py b/test/core/metaflow_test/__init__.py
@@ -2,6 +2,8 @@
 import os
 from metaflow.exception import MetaflowException
 from metaflow import current
+from metaflow.cards import get_cards
+from metaflow.plugins.cards.exception import CardNotPresentException
 
 
 def steps(prio, quals, required=False):
@@ -31,6 +33,39 @@ def truncate(var):
     return var
 
 
+def retry_untill_timeout(cb_fn, *args, timeout=4, **kwargs):
+    """
+    certain operations in metaflow may not be synchronous and may be running fully asynchronously.
+    This creates a problem in writing tests that verify some behaviour at runtime. This function
+    is a helper that allows us to wait for a certain amount of time for a callback function to
+    return a non-False value.
+    """
+    import time
+
+    start = time.time()
+    while True:
+        cb_val = cb_fn(*args, **kwargs)
+        if cb_val is not False:
+            return cb_val
+        if time.time() - start > timeout:
+            raise TimeoutError("Timeout waiting for callback to return non-False value")
+        time.sleep(1)
+
+
+def try_to_get_card(id=None, timeout=4):
+    """
+    Safetly try to get the card object until a timeout value.
+    """
+
+    def _get_card(card_id):
+        container = get_card_container(id=card_id)
+        if container is None:
+            return False
+        return container[0]
+
+    return retry_untill_timeout(_get_card, id, timeout=timeout)
+
+
 class AssertArtifactFailed(Exception):
     pass
 
@@ -66,6 +101,16 @@ def __init__(self):
         super(TestRetry, self).__init__("This is not an error. " "Testing retry...")
 
 
+def get_card_container(id=None):
+    """
+    Safetly try to load the card_container object.
+    """
+    try:
+        return get_cards(current.pathspec, id=id)
+    except CardNotPresentException:
+        return None
+
+
 def is_resumed():
     return current.origin_run_id is not None
 
@@ -144,6 +189,12 @@ def assert_log(self, step, logtype, value, exact_match=True):
     def get_card(self, step, task, card_type):
         raise NotImplementedError()
 
+    def get_card_data(self, step, task, card_type, card_id=None):
+        """
+        returns : (card_present, card_data)
+        """
+        raise NotImplementedError()
+
     def list_cards(self, step, task, card_type=None):
         raise NotImplementedError()
 

diff --git a/test/core/metaflow_test/formatter.py b/test/core/metaflow_test/formatter.py
@@ -85,7 +85,7 @@ def _flow_lines(self):
 
         yield 0, "# -*- coding: utf-8 -*-"
         yield 0, "from metaflow import FlowSpec, step, Parameter, project, IncludeFile, JSONType, current, parallel"
-        yield 0, "from metaflow_test import assert_equals, assert_equals_metadata, assert_exception, ExpectationFailed, is_resumed, ResumeFromHere, TestRetry"
+        yield 0, "from metaflow_test import assert_equals, assert_equals_metadata, assert_exception, ExpectationFailed, is_resumed, ResumeFromHere, TestRetry, try_to_get_card"
         if tags:
             yield 0, "from metaflow import %s" % ",".join(tags)
 

diff --git a/test/core/metaflow_test/metadata_check.py b/test/core/metaflow_test/metadata_check.py
@@ -167,6 +167,24 @@ def assert_card(
             )
         return True
 
+    def get_card_data(self, step, task, card_type, card_id=None):
+        """
+        returns : (card_present, card_data)
+        """
+        from metaflow.plugins.cards.exception import CardNotPresentException
+
+        try:
+            card_iter = self.get_card(step, task, card_type, card_id=card_id)
+        except CardNotPresentException:
+            return False, None
+        if card_id is None:
+            # Return the first piece of card_data we can find.
+            return True, card_iter[0]._get_data()
+        for card in card_iter:
+            if card.id == card_id:
+                return True, card._get_data()
+        return False, None
+
     def get_log(self, step, logtype):
         return "".join(getattr(task, logtype) for task in self.run[step])