From d3f3a88617516e2a7f925685a81255f207a2a7e6 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Wed, 28 Aug 2024 17:00:28 -0700 Subject: [PATCH 1/3] Exclude testing 3.7 on mac --- .github/workflows/build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c89a46b..58d04cf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,6 +17,10 @@ jobs: matrix: python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] os: [macOS-latest, ubuntu-latest, windows-latest] + exclude: + # macOS-latest are now on arm64 + - os: macOS-latest + python-version: "3.7" steps: - name: Checkout From 271a5a60af35ba3ccf7cfca8f7694a4874d56939 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Wed, 28 Aug 2024 15:10:25 -0700 Subject: [PATCH 2/3] Support reading static metadata from a source tree. Currently this handles pep621, poetry, and setuptools in setup.cfg in a best-effort sort of way. Callers of this should check for nonempty deps to know whether it found anything. Because of the fragility with intreehooks this does not even look at the build-backend. --- .gitignore | 3 + Makefile | 2 +- metadata_please/__init__.py | 7 +- metadata_please/source_checkout.py | 210 +++++++++++++++++++++++ metadata_please/tests/__init__.py | 4 +- metadata_please/tests/source_checkout.py | 114 ++++++++++++ setup.cfg | 2 + 7 files changed, 338 insertions(+), 4 deletions(-) create mode 100644 metadata_please/source_checkout.py create mode 100644 metadata_please/tests/source_checkout.py diff --git a/.gitignore b/.gitignore index 0a4bafc..23eace5 100644 --- a/.gitignore +++ b/.gitignore @@ -105,3 +105,6 @@ venv.bak/ # Visual Studio Code .vscode/ + +# Vim swapfiles +*.sw[op] diff --git a/Makefile b/Makefile index 9283b49..c5bd1f2 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ format: lint: python -m ufmt check $(SOURCES) python -m flake8 $(SOURCES) - python -m checkdeps --allow-names metadata_please metadata_please + python -m checkdeps --allow-names metadata_please,toml metadata_please mypy --strict --install-types --non-interactive metadata_please .PHONY: release diff --git a/metadata_please/__init__.py b/metadata_please/__init__.py index f7ac5ed..51bf7d6 100644 --- a/metadata_please/__init__.py +++ b/metadata_please/__init__.py @@ -4,13 +4,16 @@ from_tar_sdist, from_zip_sdist, ) +from .source_checkout import basic_metadata_from_source_checkout, from_source_checkout from .wheel import basic_metadata_from_wheel, from_wheel __all__ = [ + "basic_metadata_from_source_checkout", "basic_metadata_from_tar_sdist", - "basic_metadata_from_zip_sdist", "basic_metadata_from_wheel", - "from_zip_sdist", + "basic_metadata_from_zip_sdist", + "from_source_checkout", "from_tar_sdist", "from_wheel", + "from_zip_sdist", ] diff --git a/metadata_please/source_checkout.py b/metadata_please/source_checkout.py new file mode 100644 index 0000000..0923e42 --- /dev/null +++ b/metadata_please/source_checkout.py @@ -0,0 +1,210 @@ +""" +Best-effort metadata extraction for "source checkouts" -- e.g. a local dir containing pyproject.toml. + +This is different from an (extracted) sdist, which *should* have a generated dist-info already. + +Prefers: +- PEP 621 metadata (pyproject.toml) +- Poetry metadata (pyproject.toml) +- Setuptools static metadata (setup.cfg) + +Notably, does not read setup.py or attempt to emulate anything that can't be read staticly. +""" + +from pathlib import Path + +try: + import tomllib as toml +except ImportError: + import toml # type: ignore[no-redef,unused-ignore] + +from configparser import NoOptionError, NoSectionError, RawConfigParser + +from packaging.utils import canonicalize_name + +from .types import BasicMetadata + + +def merge_markers(extra_name: str, value: str) -> str: + """Simulates what a dist-info requirement string would look like if also restricted to an extra.""" + if ";" not in value: + return f'{value} ; extra == "{extra_name}"' + else: + a, _, b = value.partition(";") + a = a.strip() + b = b.strip() + return f'{a} ; ({b}) and extra == "{extra_name}"' + + +def from_source_checkout(path: Path) -> bytes: + return ( + from_pep621_checkout(path) + or from_poetry_checkout(path) + or from_setup_cfg_checkout(path) + ) + + +def from_pep621_checkout(path: Path) -> bytes: + """ + Returns a metadata snippet (which is zero-length if this is none of this style). + """ + try: + data = (path / "pyproject.toml").read_text() + except FileNotFoundError: + return b"" + doc = toml.loads(data) + + buf: list[str] = [] + for dep in doc.get("project", {}).get("dependencies", ()): + buf.append(f"Requires-Dist: {dep}\n") + for k, v in doc.get("project", {}).get("optional-dependencies", {}).items(): + extra_name = canonicalize_name(k) + buf.append(f"Provides-Extra: {extra_name}\n") + for i in v: + buf.append("Requires-Dist: " + merge_markers(extra_name, i) + "\n") + + return "".join(buf).encode("utf-8") + + +def _translate_caret(specifier: str) -> str: + """ + Given a string like "^0.2.3" returns ">=0.2.3,<0.3.0". + """ + assert "," not in specifier + parts = specifier[1:].split(".") + while len(parts) < 3: + parts.append("0") + + for i in range(len(parts)): + if parts[i] != "0": + # The docs are not super clear about how this behaves, but let's + # assume integer-valued parts and just let the exception raise + # otherwise. + incremented = parts[:] + incremented[i] = str(int(parts[i]) + 1) + del incremented[i + 1 :] + incremented_version = ".".join(incremented) + break + else: + raise ValueError("All components were zero?") + return f">={specifier[1:]},<{incremented_version}" + + +def _translate_tilde(specifier: str) -> str: + """ + Given a string like "~1.2.3" returns ">=1.2.3,<1.3". + """ + assert "," not in specifier + parts = specifier[1:].split(".") + incremented = parts[:2] + incremented[-1] = str(int(incremented[-1]) + 1) + incremented_version = ".".join(incremented) + + return f">={specifier[1:]},<{incremented_version}" + + +def from_poetry_checkout(path: Path) -> bytes: + """ + Returns a metadata snippet (which is zero-length if this is none of this style). + """ + try: + data = (path / "pyproject.toml").read_text() + except FileNotFoundError: + return b"" + doc = toml.loads(data) + + saved_extra_constraints = {} + + buf: list[str] = [] + for k, v in doc.get("tool", {}).get("poetry", {}).get("dependencies", {}).items(): + if k == "python": + pass # TODO requires-python + else: + k = canonicalize_name(k) + if isinstance(v, dict): + version = v.get("version", "") + if "extras" in v: + extras = "[%s]" % (",".join(v["extras"])) + else: + extras = "" + optional = v.get("optional", False) + else: + version = v + extras = "" + optional = False + + if not version: + # e.g. git, path or url dependencies, skip for now + continue + + # https://python-poetry.org/docs/dependency-specification/#version-constraints + # 1.2.* type wildcards are supported natively in packaging + if version.startswith("^"): + version = _translate_caret(version) + elif version.startswith("~"): + version = _translate_tilde(version) + elif version == "*": + version = "" + + if version[:1].isdigit(): + version = "==" + version + + if optional: + saved_extra_constraints[k] = f"{extras}{version}" + else: + buf.append(f"Requires-Dist: {k}{extras}{version}\n") + + for k, v in doc.get("tool", {}).get("poetry", {}).get("extras", {}).items(): + k = canonicalize_name(k) + buf.append(f"Provides-Extra: {k}\n") + for vi in v: + vi = canonicalize_name(vi) + buf.append( + f"Requires-Dist: {vi}{merge_markers(k, saved_extra_constraints[vi])}" + ) + + return "".join(buf).encode("utf-8") + + +def from_setup_cfg_checkout(path: Path) -> bytes: + try: + data = (path / "setup.cfg").read_text() + except FileNotFoundError: + return b"" + + rc = RawConfigParser() + rc.read_string(data) + + buf: list[str] = [] + try: + for dep in rc.get("options", "install_requires").splitlines(): + dep = dep.strip() + if dep: + buf.append(f"Requires-Dist: {dep}\n") + except (NoOptionError, NoSectionError): + pass + + try: + section = rc["options.extras_require"] + except KeyError: + pass + else: + for k, v in section.items(): + extra_name = canonicalize_name(k) + buf.append(f"Provides-Extra: {extra_name}\n") + for i in v.splitlines(): + i = i.strip() + if i: + buf.append("Requires-Dist: " + merge_markers(extra_name, i) + "\n") + + return "".join(buf).encode("utf-8") + + +def basic_metadata_from_source_checkout(path: Path) -> BasicMetadata: + return BasicMetadata.from_metadata(from_source_checkout(path)) + + +if __name__ == "__main__": # pragma: no cover + import sys + + print(basic_metadata_from_source_checkout(Path(sys.argv[1]))) diff --git a/metadata_please/tests/__init__.py b/metadata_please/tests/__init__.py index 565d539..55602ca 100644 --- a/metadata_please/tests/__init__.py +++ b/metadata_please/tests/__init__.py @@ -1,8 +1,10 @@ from .sdist import TarSdistTest, ZipSdistTest +from .source_checkout import SourceCheckoutTest from .wheel import WheelTest __all__ = [ + "SourceCheckoutTest", + "TarSdistTest", "WheelTest", "ZipSdistTest", - "TarSdistTest", ] diff --git a/metadata_please/tests/source_checkout.py b/metadata_please/tests/source_checkout.py new file mode 100644 index 0000000..4218a6e --- /dev/null +++ b/metadata_please/tests/source_checkout.py @@ -0,0 +1,114 @@ +import tempfile + +import unittest +from pathlib import Path + +from ..source_checkout import basic_metadata_from_source_checkout +from ..types import BasicMetadata + + +class SourceCheckoutTest(unittest.TestCase): + def test_pep621_empty(self) -> None: + with tempfile.TemporaryDirectory() as d: + Path(d, "pyproject.toml").write_text("") + self.assertEqual( + BasicMetadata((), frozenset()), + basic_metadata_from_source_checkout(Path(d)), + ) + + def test_pep621_extras(self) -> None: + with tempfile.TemporaryDirectory() as d: + Path(d, "pyproject.toml").write_text( + """\ +[project] +dependencies = ["x"] + +[project.optional-dependencies] +dev = ["Foo <= 2"] +""" + ) + self.assertEqual( + BasicMetadata(["x", 'Foo <= 2 ; extra == "dev"'], frozenset(["dev"])), + basic_metadata_from_source_checkout(Path(d)), + ) + + def test_poetry_full(self) -> None: + with tempfile.TemporaryDirectory() as d: + Path(d, "pyproject.toml").write_text( + """\ +[tool.poetry.dependencies] +python = "^3.6" +a = "1.0" +a2 = "*" +b = "^1.2.3" +b2 = "^0.2.3" +c = "~1.2.3" +c2 = "~1.2" +c3 = "~1" +skipped = {git = "..."} +complex = {extras=["bar", "baz"], version="2"} +opt = { version = "^2.9", optional = true} +unused-extra = { version = "2", optional = true } + +[tool.poetry.extras] +Foo = ["Opt"] # intentionally uppercased +""" + ) + rv = basic_metadata_from_source_checkout(Path(d)) + self.assertEqual( + [ + "a==1.0", + "a2", + "b>=1.2.3,<2", + "b2>=0.2.3,<0.3", + "c>=1.2.3,<1.3", + "c2>=1.2,<1.3", + "c3>=1,<2", + "complex[bar,baz]==2", + 'opt>=2.9,<3 ; extra == "foo"', + ], + rv.reqs, + ) + self.assertEqual( + frozenset({"foo"}), + rv.provides_extra, + ) + + def test_setuptools_empty(self) -> None: + with tempfile.TemporaryDirectory() as d: + Path(d, "setup.cfg").write_text("") + self.assertEqual( + BasicMetadata((), frozenset()), + basic_metadata_from_source_checkout(Path(d)), + ) + + def test_setuptools_extras(self) -> None: + with tempfile.TemporaryDirectory() as d: + Path(d, "setup.cfg").write_text( + """\ +[options] +install_requires = + x + y + +[options.extras_require] +dev = + # comment + Foo <= 2 + # comment after +marker = + Bar ; python_version < "3" +""" + ) + self.assertEqual( + BasicMetadata( + [ + "x", + "y", + 'Foo <= 2 ; extra == "dev"', + 'Bar ; (python_version < "3") and extra == "marker"', + ], + frozenset(["dev", "marker"]), + ), + basic_metadata_from_source_checkout(Path(d)), + ) diff --git a/setup.cfg b/setup.cfg index 566e694..a2fb1fc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,6 +18,8 @@ setup_requires = include_package_data = true install_requires = packaging + configparser + toml; python_version < '3.11' [options.extras_require] dev = From c096436d1cf938690367fa2571c80b26d0d067c3 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Thu, 29 Aug 2024 18:05:20 -0700 Subject: [PATCH 3/3] Support extras and python on poetry deps --- metadata_please/source_checkout.py | 43 +++++++++++++++++++----- metadata_please/tests/source_checkout.py | 43 +++++++++++++++++------- 2 files changed, 65 insertions(+), 21 deletions(-) diff --git a/metadata_please/source_checkout.py b/metadata_please/source_checkout.py index 0923e42..e6e35d1 100644 --- a/metadata_please/source_checkout.py +++ b/metadata_please/source_checkout.py @@ -10,7 +10,7 @@ Notably, does not read setup.py or attempt to emulate anything that can't be read staticly. """ - +import re from pathlib import Path try: @@ -24,8 +24,20 @@ from .types import BasicMetadata +OPERATOR_RE = re.compile(r"([<>=~]+)(\d.*)") + + +def combine_markers(*markers: str) -> str: + filtered_markers = [m for m in markers if m and m.strip()] + if len(filtered_markers) == 0: + return "" + elif len(filtered_markers) == 1: + return filtered_markers[0] + else: + return " and ".join(f"({m})" for m in filtered_markers) + -def merge_markers(extra_name: str, value: str) -> str: +def merge_extra_marker(extra_name: str, value: str) -> str: """Simulates what a dist-info requirement string would look like if also restricted to an extra.""" if ";" not in value: return f'{value} ; extra == "{extra_name}"' @@ -33,7 +45,8 @@ def merge_markers(extra_name: str, value: str) -> str: a, _, b = value.partition(";") a = a.strip() b = b.strip() - return f'{a} ; ({b}) and extra == "{extra_name}"' + c = f'extra == "{extra_name}"' + return f"{a} ; {combine_markers(b, c)}" def from_source_checkout(path: Path) -> bytes: @@ -61,7 +74,7 @@ def from_pep621_checkout(path: Path) -> bytes: extra_name = canonicalize_name(k) buf.append(f"Provides-Extra: {extra_name}\n") for i in v: - buf.append("Requires-Dist: " + merge_markers(extra_name, i) + "\n") + buf.append("Requires-Dist: " + merge_extra_marker(extra_name, i) + "\n") return "".join(buf).encode("utf-8") @@ -127,10 +140,21 @@ def from_poetry_checkout(path: Path) -> bytes: extras = "[%s]" % (",".join(v["extras"])) else: extras = "" + markers = v.get("markers", "") + python = v.get("python", "") + if python: + m = OPERATOR_RE.fullmatch(python) + assert m is not None + # TODO do ^/~ work on python version? + python = f"python_version {m.group(1)} '{m.group(2)}'" + markers = combine_markers(markers, python) + if markers: + markers = " ; " + markers optional = v.get("optional", False) else: version = v extras = "" + markers = "" optional = False if not version: @@ -150,17 +174,18 @@ def from_poetry_checkout(path: Path) -> bytes: version = "==" + version if optional: - saved_extra_constraints[k] = f"{extras}{version}" + saved_extra_constraints[k] = (f"{extras}{version}", markers) else: - buf.append(f"Requires-Dist: {k}{extras}{version}\n") + buf.append(f"Requires-Dist: {k}{extras}{version}{markers}\n") for k, v in doc.get("tool", {}).get("poetry", {}).get("extras", {}).items(): k = canonicalize_name(k) buf.append(f"Provides-Extra: {k}\n") for vi in v: vi = canonicalize_name(vi) + constraints, markers = saved_extra_constraints[vi] buf.append( - f"Requires-Dist: {vi}{merge_markers(k, saved_extra_constraints[vi])}" + f"Requires-Dist: {vi}{constraints}{merge_extra_marker(k, markers)}" ) return "".join(buf).encode("utf-8") @@ -195,7 +220,9 @@ def from_setup_cfg_checkout(path: Path) -> bytes: for i in v.splitlines(): i = i.strip() if i: - buf.append("Requires-Dist: " + merge_markers(extra_name, i) + "\n") + buf.append( + "Requires-Dist: " + merge_extra_marker(extra_name, i) + "\n" + ) return "".join(buf).encode("utf-8") diff --git a/metadata_please/tests/source_checkout.py b/metadata_please/tests/source_checkout.py index 4218a6e..a5882f3 100644 --- a/metadata_please/tests/source_checkout.py +++ b/metadata_please/tests/source_checkout.py @@ -21,16 +21,25 @@ def test_pep621_extras(self) -> None: Path(d, "pyproject.toml").write_text( """\ [project] -dependencies = ["x"] +dependencies = ["x", "y ; platform_system != 'Windows'"] [project.optional-dependencies] dev = ["Foo <= 2"] +marker = ["Bar ; python_version < '3'", "Baz <= 2; python_version < '3'"] """ ) + rv = basic_metadata_from_source_checkout(Path(d)) self.assertEqual( - BasicMetadata(["x", 'Foo <= 2 ; extra == "dev"'], frozenset(["dev"])), - basic_metadata_from_source_checkout(Path(d)), + [ + "x", + "y ; platform_system != 'Windows'", + 'Foo <= 2 ; extra == "dev"', + "Bar ; (python_version < '3') and (extra == \"marker\")", + "Baz <= 2 ; (python_version < '3') and (extra == \"marker\")", + ], + rv.reqs, ) + self.assertEqual(frozenset({"dev", "marker"}), rv.provides_extra) def test_poetry_full(self) -> None: with tempfile.TemporaryDirectory() as d: @@ -45,6 +54,8 @@ def test_poetry_full(self) -> None: c = "~1.2.3" c2 = "~1.2" c3 = "~1" +d = {version="2", python="<3.11"} +e = {version="2", markers="sys_platform == 'darwin'"} skipped = {git = "..."} complex = {extras=["bar", "baz"], version="2"} opt = { version = "^2.9", optional = true} @@ -64,6 +75,8 @@ def test_poetry_full(self) -> None: "c>=1.2.3,<1.3", "c2>=1.2,<1.3", "c3>=1,<2", + "d==2 ; python_version < '3.11'", + "e==2 ; sys_platform == 'darwin'", "complex[bar,baz]==2", 'opt>=2.9,<3 ; extra == "foo"', ], @@ -98,17 +111,21 @@ def test_setuptools_extras(self) -> None: # comment after marker = Bar ; python_version < "3" + Baz <= 2; python_version < "3" """ ) + rv = basic_metadata_from_source_checkout(Path(d)) self.assertEqual( - BasicMetadata( - [ - "x", - "y", - 'Foo <= 2 ; extra == "dev"', - 'Bar ; (python_version < "3") and extra == "marker"', - ], - frozenset(["dev", "marker"]), - ), - basic_metadata_from_source_checkout(Path(d)), + [ + "x", + "y", + 'Foo <= 2 ; extra == "dev"', + 'Bar ; (python_version < "3") and (extra == "marker")', + 'Baz <= 2 ; (python_version < "3") and (extra == "marker")', + ], + rv.reqs, + ) + self.assertEqual( + frozenset({"dev", "marker"}), + rv.provides_extra, )