From 11fd153f1fb2fe675c959b4c5f334cd92f49b94e Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Fri, 31 Mar 2023 13:27:30 +0200 Subject: [PATCH 1/9] Fallback to metadata.version before giving up --- ctapipe/core/provenance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctapipe/core/provenance.py b/ctapipe/core/provenance.py index 312f43884ee..01e902dad4f 100644 --- a/ctapipe/core/provenance.py +++ b/ctapipe/core/provenance.py @@ -53,7 +53,7 @@ def get_module_version(name): try: module = import_module(name) return module.__version__ - except AttributeError: + except (AttributeError, ModuleNotFoundError): try: return version(name) except Exception: From e539eb06b4773f17081c55cede6537eb093abf43 Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Fri, 31 Mar 2023 13:28:07 +0200 Subject: [PATCH 2/9] Read dependencies from setup.cfg, filter versions --- ctapipe/tools/info.py | 54 +++++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/ctapipe/tools/info.py b/ctapipe/tools/info.py index 66b28ba97c8..57a6b9fe900 100644 --- a/ctapipe/tools/info.py +++ b/ctapipe/tools/info.py @@ -1,7 +1,10 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst """ print information about ctapipe and its command-line tools. """ +from configparser import ConfigParser import logging import os +from pathlib import Path +from re import split import sys from ..core import Provenance, get_module_version @@ -16,28 +19,43 @@ __all__ = ["info"] -# TODO: this list should be global (or generated at install time) -_dependencies = sorted( - [ - "astropy", - "matplotlib", - "numpy", - "traitlets", - "sklearn", - "scipy", - "numba", - "pytest", - "iminuit", - "tables", - "eventio", - ] -) +conf = ConfigParser() +conf.read(Path(__file__).parent / Path("../../setup.cfg")) +setup_cfg_options = dict(conf.items("options")) +setup_cfg_options_extras = dict(conf.items("options.extras_require")) + +_dependencies = sorted(setup_cfg_options["install_requires"].split("\n")[1:]) _optional_dependencies = sorted( - ["ctapipe_resources", "pytest", "graphviz", "matplotlib"] + setup_cfg_options_extras["tests"].split("\n")[1:] + + setup_cfg_options_extras["docs"].split("\n")[1:] ) +def get_package_name_setuptools(name): + """Extract a package name from setuptools syntax. + + Parameters + ---------- + name: str + Name of the package as specified in files like + ``setup.cfg``, ``setup.py`` or ``pyproject.toml``. + + Returns + ------- + module_name: str + Name of package without version specification. + """ + + version_delimeters = [" ", "=", "~", ">", "<"] + if any([i in name for i in version_delimeters]): + package_name = split(r"|".join(version_delimeters), name)[0] + else: + package_name = name + + return package_name + + def main(args=None): parser = get_parser(info) parser.add_argument("--version", action="store_true", help="Print version number") @@ -167,12 +185,14 @@ def _info_dependencies(): print("\n*** ctapipe core dependencies ***\n") for name in _dependencies: + name = get_package_name_setuptools(name) version = get_module_version(name) print(f"{name:>20s} -- {version}") print("\n*** ctapipe optional dependencies ***\n") for name in _optional_dependencies: + name = get_package_name_setuptools(name) version = get_module_version(name) print(f"{name:>20s} -- {version}") From a1adba0c880848b673684c8a93c6b6b83f6a88b8 Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Fri, 31 Mar 2023 13:28:26 +0200 Subject: [PATCH 3/9] Add pytest-console-scripts test dependency --- environment.yml | 1 + setup.cfg | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index e2d90f14896..0376642eac9 100644 --- a/environment.yml +++ b/environment.yml @@ -27,6 +27,7 @@ dependencies: - psutil - pytables - pytest + - pytest-console-scripts - pytest-cov - pytest-runner - pytest-astropy-header diff --git a/setup.cfg b/setup.cfg index 8085914b2d9..fb7508a2a31 100644 --- a/setup.cfg +++ b/setup.cfg @@ -53,7 +53,7 @@ tests = tomli pytest_astropy_header h5py - + pytest-console-scripts docs = sphinx ~=3.5 From c2cdb69bcf342c39b7e79bf4e8e9c356d6aceb0b Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Fri, 31 Mar 2023 13:28:36 +0200 Subject: [PATCH 4/9] Test info tool properly --- ctapipe/tools/tests/test_info.py | 78 ++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 ctapipe/tools/tests/test_info.py diff --git a/ctapipe/tools/tests/test_info.py b/ctapipe/tools/tests/test_info.py new file mode 100644 index 00000000000..06e5d8ae78a --- /dev/null +++ b/ctapipe/tools/tests/test_info.py @@ -0,0 +1,78 @@ +"""Test ctapipe-info functionality.""" + + +def test_info_version(script_runner): + + result = script_runner.run( + "ctapipe-info", + "--version", + ) + + assert result.success + assert result.stderr == "" + + +def test_info_tools(script_runner): + + result = script_runner.run( + "ctapipe-info", + "--tools", + ) + + assert result.success + assert result.stderr == "" + + +def test_info_dependencies(script_runner): + + result = script_runner.run( + "ctapipe-info", + "--dependencies", + ) + + assert result.success + assert result.stderr == "" + + +def test_info_system(script_runner): + + result = script_runner.run( + "ctapipe-info", + "--system", + ) + + assert result.success + assert result.stderr == "" + + +def test_info_plugins(script_runner): + + result = script_runner.run( + "ctapipe-info", + "--plugins", + ) + + assert result.success + assert result.stderr == "" + + +def test_info_eventsources(script_runner): + + result = script_runner.run( + "ctapipe-info", + "--event-sources", + ) + + assert result.success + assert result.stderr == "" + + +def test_info_all(script_runner): + + result = script_runner.run( + "ctapipe-info", + "--all", + ) + + assert result.success + assert result.stderr == "" From e9fc7a7bed6efa0b00e1998c3abbfbfa7eab5622 Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Fri, 31 Mar 2023 13:29:19 +0200 Subject: [PATCH 5/9] Remove old test for info tool --- ctapipe/tools/tests/test_tools.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ctapipe/tools/tests/test_tools.py b/ctapipe/tools/tests/test_tools.py index c31c7736456..5ab7e3af826 100644 --- a/ctapipe/tools/tests/test_tools.py +++ b/ctapipe/tools/tests/test_tools.py @@ -53,12 +53,6 @@ def test_display_dl1(tmp_path, dl1_image_file, dl1_parameters_file): run_tool(DisplayDL1Calib(), ["--help-all"], raises=True) -def test_info(): - from ctapipe.tools.info import info - - info(show_all=True) - - def test_fileinfo(tmp_path, dl1_image_file): """check we can run ctapipe-fileinfo and get results""" import yaml From 7400781de2ce920673e8494660f91666fba3c7a9 Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Fri, 31 Mar 2023 15:43:29 +0200 Subject: [PATCH 6/9] Add news fragment for PR 2303 --- docs/changes/2303.optimization.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 docs/changes/2303.optimization.rst diff --git a/docs/changes/2303.optimization.rst b/docs/changes/2303.optimization.rst new file mode 100644 index 00000000000..785fa3d6671 --- /dev/null +++ b/docs/changes/2303.optimization.rst @@ -0,0 +1,2 @@ +Optimize ``ctapipe-info --dependencies`` by reading them from the package installation code. +Use ``pytest-console-scripts`` to test this tool. \ No newline at end of file From d53b9e322fe7c918a8d19be8f86d4d4b72491a36 Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Fri, 31 Mar 2023 15:47:34 +0200 Subject: [PATCH 7/9] Run isort on ctapipe/tools/info.py --- ctapipe/tools/info.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ctapipe/tools/info.py b/ctapipe/tools/info.py index 57a6b9fe900..1d3e8665403 100644 --- a/ctapipe/tools/info.py +++ b/ctapipe/tools/info.py @@ -1,11 +1,11 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst """ print information about ctapipe and its command-line tools. """ -from configparser import ConfigParser import logging import os +import sys +from configparser import ConfigParser from pathlib import Path from re import split -import sys from ..core import Provenance, get_module_version from ..core.plugins import detect_and_import_plugins @@ -286,7 +286,8 @@ def _info_reconstructors(): def _info_datamodel(): - from ctapipe.io.datawriter import DATA_MODEL_CHANGE_HISTORY, DATA_MODEL_VERSION + from ctapipe.io.datawriter import (DATA_MODEL_CHANGE_HISTORY, + DATA_MODEL_VERSION) from ctapipe.io.hdf5eventsource import COMPATIBLE_DATA_MODEL_VERSIONS print("\n*** ctapipe data model ***\n") From f2e4c43a8a9e78b76347e2bc4d5fb1e0a1642dec Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Fri, 31 Mar 2023 16:00:30 +0200 Subject: [PATCH 8/9] Fix isort complain in ctapipe/tools/info.py --- ctapipe/tools/info.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ctapipe/tools/info.py b/ctapipe/tools/info.py index 1d3e8665403..ebcba44d777 100644 --- a/ctapipe/tools/info.py +++ b/ctapipe/tools/info.py @@ -286,8 +286,7 @@ def _info_reconstructors(): def _info_datamodel(): - from ctapipe.io.datawriter import (DATA_MODEL_CHANGE_HISTORY, - DATA_MODEL_VERSION) + from ctapipe.io.datawriter import DATA_MODEL_CHANGE_HISTORY, DATA_MODEL_VERSION from ctapipe.io.hdf5eventsource import COMPATIBLE_DATA_MODEL_VERSIONS print("\n*** ctapipe data model ***\n") From 5c76087b947cfdc4bf74f15b3ca27b103d313330 Mon Sep 17 00:00:00 2001 From: Michele Peresano Date: Mon, 8 May 2023 12:05:46 +0200 Subject: [PATCH 9/9] Use importlib.metadata and print all extras --- ctapipe/tools/info.py | 81 +++++++++++++++++-------------------------- 1 file changed, 31 insertions(+), 50 deletions(-) diff --git a/ctapipe/tools/info.py b/ctapipe/tools/info.py index ebcba44d777..80a6cb11027 100644 --- a/ctapipe/tools/info.py +++ b/ctapipe/tools/info.py @@ -2,10 +2,9 @@ """ print information about ctapipe and its command-line tools. """ import logging import os +import re import sys -from configparser import ConfigParser -from pathlib import Path -from re import split +from importlib.metadata import metadata, requires from ..core import Provenance, get_module_version from ..core.plugins import detect_and_import_plugins @@ -19,42 +18,6 @@ __all__ = ["info"] -conf = ConfigParser() -conf.read(Path(__file__).parent / Path("../../setup.cfg")) -setup_cfg_options = dict(conf.items("options")) -setup_cfg_options_extras = dict(conf.items("options.extras_require")) - -_dependencies = sorted(setup_cfg_options["install_requires"].split("\n")[1:]) - -_optional_dependencies = sorted( - setup_cfg_options_extras["tests"].split("\n")[1:] - + setup_cfg_options_extras["docs"].split("\n")[1:] -) - - -def get_package_name_setuptools(name): - """Extract a package name from setuptools syntax. - - Parameters - ---------- - name: str - Name of the package as specified in files like - ``setup.cfg``, ``setup.py`` or ``pyproject.toml``. - - Returns - ------- - module_name: str - Name of package without version specification. - """ - - version_delimeters = [" ", "=", "~", ">", "<"] - if any([i in name for i in version_delimeters]): - package_name = split(r"|".join(version_delimeters), name)[0] - else: - package_name = name - - return package_name - def main(args=None): parser = get_parser(info) @@ -100,6 +63,12 @@ def main(args=None): info(**vars(args)) +def pretty_print_requires(package): + pack_name = re.split(";|=|>|<|@|~| ", package)[0] + entry = f"{pack_name} -- {get_module_version(pack_name)}" + return entry + + def info( version=False, tools=False, @@ -182,19 +151,32 @@ def _info_tools(): def _info_dependencies(): """Print info about dependencies.""" - print("\n*** ctapipe core dependencies ***\n") - for name in _dependencies: - name = get_package_name_setuptools(name) - version = get_module_version(name) - print(f"{name:>20s} -- {version}") + meta = metadata("ctapipe") + extras = [v for k, v in meta.items() if k == "Provides-Extra"] + + all_dependencies = set(requires("ctapipe")) - print("\n*** ctapipe optional dependencies ***\n") + optional_dependencies = {extra: [] for extra in extras} - for name in _optional_dependencies: - name = get_package_name_setuptools(name) - version = get_module_version(name) - print(f"{name:>20s} -- {version}") + required_dependencies = [] + for package in all_dependencies: + if "extra" in package: + for extra in extras: + if extra in package: + optional_dependencies[extra].append(pretty_print_requires(package)) + else: + required_dependencies.append(pretty_print_requires(package)) + + print("\n*** ctapipe core dependencies ***\n") + + for package in required_dependencies: + print(package) + + for extra in optional_dependencies: + print(f"\n*** ctapipe optional dependencies [{extra}] ***\n") + for package in optional_dependencies[extra]: + print(package) def _info_resources(): @@ -239,7 +221,6 @@ def _info_system(): system_prov = prov.current_activity.provenance["system"] for section in ["platform", "python"]: - print("\n====== ", section, " ======== \n") sysinfo = system_prov[section]