go back to using get_pinned_conda_libs as a source for conda dependen…

…cies. (#1567)
Netflix · Oct 9, 2023 · f52d03d · f52d03d
1 parent f165ea3
commit f52d03d
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 18 deletions.
diff --git a/metaflow/metaflow_config.py b/metaflow/metaflow_config.py
@@ -430,7 +430,6 @@ def get_version(pkg):
     return pkg_resources.get_distribution(pkg).version
 
 
-# TODO: This is no longer in use and can be dispensed with.
 # PINNED_CONDA_LIBS are the libraries that metaflow depends on for execution
 # and are needed within a conda environment
 def get_pinned_conda_libs(python_version, datastore_type):

diff --git a/metaflow/plugins/pypi/conda_environment.py b/metaflow/plugins/pypi/conda_environment.py
@@ -14,6 +14,7 @@
 
 import requests
 
+from metaflow.metaflow_config import get_pinned_conda_libs
 from metaflow.exception import MetaflowException
 from metaflow.metaflow_environment import MetaflowEnvironment
 from metaflow.metaflow_profile import profile
@@ -196,22 +197,18 @@ def get_environment(self, step):
                     }
                 else:
                     return {}
+        # Resolve conda environment for @pypi's Python, falling back on @conda's
+        # Python
+        env_python = (
+            environment.get("pypi", environment["conda"]).get("python")
+            or environment["conda"]["python"]
+        )
         # TODO: Support dependencies for `--metadata`.
         # TODO: Introduce support for `--telemetry` as a follow up.
         # Certain packages are required for metaflow runtime to function correctly.
         # Ensure these packages are available both in Conda channels and PyPI
         # repostories.
-        pinned_packages = {"requests": ">=2.21.0"}
-        if self.datastore_type == "s3":
-            pinned_packages.update({"boto3": ">=1.14.0"})
-        elif self.datastore_type == "azure":
-            pinned_packages.update(
-                {"azure-identity": ">=1.10.0", "azure-storage-blob": ">=12.12.0"}
-            )
-        elif self.datastore_type == "gs":
-            pinned_packages.update(
-                {"google-cloud-storage": ">=2.5.0", "google-auth": ">=2.11.0"}
-            )
+        pinned_packages = get_pinned_conda_libs(env_python, self.datastore_type)
 
         # PyPI dependencies are prioritized over Conda dependencies.
         environment.get("pypi", environment["conda"])["packages"] = {
@@ -253,12 +250,8 @@ def get_environment(self, step):
                 {target_platform, conda_platform()}
             )
             environment["pypi"]["platforms"] = [target_platform]
-            # Resolve conda environment for @pypi's Python, falling back on @conda's
-            # Python
-            environment["pypi"]["python"] = environment["conda"]["python"] = (
-                environment["pypi"].get("python", environment["conda"]["python"])
-                or environment["conda"]["python"]
-            )
+            # Match PyPI and Conda python versions with the resolved environment Python.
+            environment["pypi"]["python"] = environment["conda"]["python"] = env_python
 
         # Z combinator for a recursive lambda
         deep_sort = (lambda f: f(f))(