@pypi and faster @conda (#1505)

* @pip draft

* chore: pip deco fixes (#1523)

* correctly choose conda platform

* fix conda step_init comment

* delete unused files from pypi plugin

* add TODO and add missing default param to executable()

* add support for legacy libraries= in conda decorator (#1525)

* chore: fix micromamba detection (#1526)

* add installation step for micromamba binary if none can be found

* add error handling to micromamba install

* @pip draft

* vend

* rebase

* more changes

* more changes

* enable conda channels

* more comments

* introduce conda_base

* more changes

* address comments

* address nousersite issue

* handle cross-platform cache check optimization

* handle missing tarballs

* update comments

* fix biug

* address more todos

* remove stray file

* fix comments

* fix mac version

* fix passing multiple channels (#1536)

* chore: pip decorator fixes (#1535)

* add docstrings to conda flow and step decorators

* unify micromamba binary download locations

* explicitly except KeyError

* unused import

* conditional import for packaging for python 3.5 (#1534)

* fix micromamba download extraction command (#1539)

* support extra indices

* support @pypi(python=)

* support extensions

* support @pypi_base

* more changes

* @pypi(disabled=)

* fix issue

* fix issue

* fix

* address @ville's issue

* support extensions

* env escape support

* fix

* drop pip resolution times by 65%

* fix up INFO

* cleanup

* remove spurious todos

* address minor nits

* remove references to py3.5

* add stub object for py3.5

---------

Co-authored-by: Sakari Ikonen <64256562+saikonen@users.noreply.github.com>

metaflow/_vendor/vendor_any.txt

@@ -1,2 +1,2 @@
 click==7.1.2
-packaging==23.0
+packaging==23.0

metaflow/metaflow_config.py

@@ -429,6 +429,7 @@ def get_version(pkg):
     return pkg_resources.get_distribution(pkg).version
 
 
+# TODO: This is no longer in use and can be dispensed with.
 # PINNED_CONDA_LIBS are the libraries that metaflow depends on for execution
 # and are needed within a conda environment
 def get_pinned_conda_libs(python_version, datastore_type):

metaflow/plugins/__init__.py

@@ -41,10 +41,11 @@
         "unbounded_test_foreach_internal",
         ".test_unbounded_foreach_decorator.InternalTestUnboundedForeachDecorator",
     ),
-    ("conda", ".conda.conda_step_decorator.CondaStepDecorator"),
     ("card", ".cards.card_decorator.CardDecorator"),
     ("pytorch_parallel", ".frameworks.pytorch.PytorchParallelDecorator"),
     ("airflow_internal", ".airflow.airflow_decorator.AirflowInternalDecorator"),
+    ("pypi", ".pypi.pypi_decorator.PyPIStepDecorator"),
+    ("conda", ".pypi.conda_decorator.CondaStepDecorator"),
 ]
 
 # Add new flow decorators here
@@ -53,15 +54,19 @@
 # careful with the choice of name though - they become top-level
 # imports from the metaflow package.
 FLOW_DECORATORS_DESC = [
-    ("conda_base", ".conda.conda_flow_decorator.CondaFlowDecorator"),
     ("schedule", ".aws.step_functions.schedule_decorator.ScheduleDecorator"),
     ("project", ".project_decorator.ProjectDecorator"),
     ("trigger", ".events_decorator.TriggerDecorator"),
     ("trigger_on_finish", ".events_decorator.TriggerOnFinishDecorator"),
+    ("pypi_base", ".pypi.pypi_decorator.PyPIFlowDecorator"),
+    ("conda_base", ".pypi.conda_decorator.CondaFlowDecorator"),
 ]
 
 # Add environments here
-ENVIRONMENTS_DESC = [("conda", ".conda.conda_environment.CondaEnvironment")]
+ENVIRONMENTS_DESC = [
+    ("conda", ".pypi.conda_environment.CondaEnvironment"),
+    ("pypi", ".pypi.pypi_environment.PyPIEnvironment"),
+]
 
 # Add metadata providers here
 METADATA_PROVIDERS_DESC = [