Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Upload local Pants PEXs to release #19450

Merged
merged 4 commits into from
Jul 14, 2023
Merged

Upload local Pants PEXs to release #19450

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1c51513
upload local PEXs to release
thejcannon Jul 11, 2023
e37a639
some tweaks
thejcannon Jul 11, 2023
176da6e
Merge branch 'main' into jcannon/uploadPEXs
thejcannon Jul 13, 2023
cd1e8a5
reorder
thejcannon Jul 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 67 additions & 24 deletions .github/workflows/release.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,23 +44,34 @@ jobs:
- env:
PANTS_CONFIG_FILES: +['pants.ci.toml','pants.ci.aarch64.toml']
name: Build wheels
run: './pants run src/python/pants_release/release.py -- build-local-pex

./pants run src/python/pants_release/release.py -- build-wheels

'
run: ./pants run src/python/pants_release/release.py -- build-wheels
- env:
PANTS_CONFIG_FILES: +['pants.ci.toml','pants.ci.aarch64.toml']
name: Build Pants PEX
run: ./pants package src/python/pants:pants-pex
- continue-on-error: true
if: always()
name: Upload pants.log
uses: actions/upload-artifact@v3
with:
name: logs-wheels-Linux-ARM64
name: logs-wheels-and-pex-Linux-ARM64
path: .pants.d/*.log
- env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
name: Deploy wheels to S3
run: ./pants run src/python/pants_release/deploy_to_s3.py
- if: needs.release_info.outputs.is-release == 'true'
name: Upload Pants PEX
run: 'LOCAL_TAG=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import
sys;major, minor = sys.version_info[:2];import os;uname = os.uname();print(f''cp{major}{minor}-{uname.sysname.lower()}_{uname.machine.lower()}'')")

mv dist/src.python.pants/pants-pex.pex dist/src.python.pants/pants.$LOCAL_TAG.pex

gh release upload --no-clobber ${{ needs.release_info.outputs.build-ref }}
dist/src.python.pants/pants.$LOCAL_TAG.pex

'
timeout-minutes: 90
build_wheels_linux_x86_64:
container:
Expand Down Expand Up @@ -103,23 +114,33 @@ jobs:
go-version: 1.19.5
- env: {}
name: Build wheels
run: './pants run src/python/pants_release/release.py -- build-local-pex

./pants run src/python/pants_release/release.py -- build-wheels

'
run: ./pants run src/python/pants_release/release.py -- build-wheels
- env: {}
name: Build Pants PEX
run: ./pants package src/python/pants:pants-pex
- continue-on-error: true
if: always()
name: Upload pants.log
uses: actions/upload-artifact@v3
with:
name: logs-wheels-Linux-x86_64
name: logs-wheels-and-pex-Linux-x86_64
path: .pants.d/*.log
- env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
name: Deploy wheels to S3
run: ./pants run src/python/pants_release/deploy_to_s3.py
- if: needs.release_info.outputs.is-release == 'true'
name: Upload Pants PEX
run: 'LOCAL_TAG=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import
sys;major, minor = sys.version_info[:2];import os;uname = os.uname();print(f''cp{major}{minor}-{uname.sysname.lower()}_{uname.machine.lower()}'')")

mv dist/src.python.pants/pants-pex.pex dist/src.python.pants/pants.$LOCAL_TAG.pex

gh release upload --no-clobber ${{ needs.release_info.outputs.build-ref }}
dist/src.python.pants/pants.$LOCAL_TAG.pex

'
timeout-minutes: 90
build_wheels_macos10_15_x86_64:
env:
Expand Down Expand Up @@ -162,23 +183,34 @@ jobs:
- env:
ARCHFLAGS: -arch x86_64
name: Build wheels
run: './pants run src/python/pants_release/release.py -- build-local-pex

./pants run src/python/pants_release/release.py -- build-wheels

'
run: ./pants run src/python/pants_release/release.py -- build-wheels
- env:
ARCHFLAGS: -arch x86_64
name: Build Pants PEX
run: ./pants package src/python/pants:pants-pex
- continue-on-error: true
if: always()
name: Upload pants.log
uses: actions/upload-artifact@v3
with:
name: logs-wheels-macOS10-15-x86_64
name: logs-wheels-and-pex-macOS10-15-x86_64
path: .pants.d/*.log
- env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
name: Deploy wheels to S3
run: ./pants run src/python/pants_release/deploy_to_s3.py
- if: needs.release_info.outputs.is-release == 'true'
name: Upload Pants PEX
run: 'LOCAL_TAG=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import
sys;major, minor = sys.version_info[:2];import os;uname = os.uname();print(f''cp{major}{minor}-{uname.sysname.lower()}_{uname.machine.lower()}'')")

mv dist/src.python.pants/pants-pex.pex dist/src.python.pants/pants.$LOCAL_TAG.pex

gh release upload --no-clobber ${{ needs.release_info.outputs.build-ref }}
dist/src.python.pants/pants.$LOCAL_TAG.pex

'
timeout-minutes: 90
build_wheels_macos11_arm64:
env:
Expand Down Expand Up @@ -221,23 +253,34 @@ jobs:
- env:
ARCHFLAGS: -arch arm64
name: Build wheels
run: './pants run src/python/pants_release/release.py -- build-local-pex

./pants run src/python/pants_release/release.py -- build-wheels

'
run: ./pants run src/python/pants_release/release.py -- build-wheels
- env:
ARCHFLAGS: -arch arm64
name: Build Pants PEX
run: ./pants package src/python/pants:pants-pex
- continue-on-error: true
if: always()
name: Upload pants.log
uses: actions/upload-artifact@v3
with:
name: logs-wheels-macOS11-ARM64
name: logs-wheels-and-pex-macOS11-ARM64
path: .pants.d/*.log
- env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
name: Deploy wheels to S3
run: ./pants run src/python/pants_release/deploy_to_s3.py
- if: needs.release_info.outputs.is-release == 'true'
name: Upload Pants PEX
run: 'LOCAL_TAG=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import
sys;major, minor = sys.version_info[:2];import os;uname = os.uname();print(f''cp{major}{minor}-{uname.sysname.lower()}_{uname.machine.lower()}'')")

mv dist/src.python.pants/pants-pex.pex dist/src.python.pants/pants.$LOCAL_TAG.pex

gh release upload --no-clobber ${{ needs.release_info.outputs.build-ref }}
dist/src.python.pants/pants.$LOCAL_TAG.pex

'
timeout-minutes: 90
publish:
if: github.repository_owner == 'pantsbuild' && needs.release_info.outputs.is-release
Expand Down
47 changes: 23 additions & 24 deletions .github/workflows/test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -315,17 +315,17 @@ jobs:
- env:
PANTS_CONFIG_FILES: +['pants.ci.toml','pants.ci.aarch64.toml']
name: Build wheels
run: './pants run src/python/pants_release/release.py -- build-local-pex

./pants run src/python/pants_release/release.py -- build-wheels

'
run: ./pants run src/python/pants_release/release.py -- build-wheels
- env:
PANTS_CONFIG_FILES: +['pants.ci.toml','pants.ci.aarch64.toml']
name: Build Pants PEX
run: ./pants package src/python/pants:pants-pex
- continue-on-error: true
if: always()
name: Upload pants.log
uses: actions/upload-artifact@v3
with:
name: logs-wheels-Linux-ARM64
name: logs-wheels-and-pex-Linux-ARM64
path: .pants.d/*.log
timeout-minutes: 90
build_wheels_linux_x86_64:
Expand Down Expand Up @@ -369,17 +369,16 @@ jobs:
go-version: 1.19.5
- env: {}
name: Build wheels
run: './pants run src/python/pants_release/release.py -- build-local-pex

./pants run src/python/pants_release/release.py -- build-wheels

'
run: ./pants run src/python/pants_release/release.py -- build-wheels
- env: {}
name: Build Pants PEX
run: ./pants package src/python/pants:pants-pex
- continue-on-error: true
if: always()
name: Upload pants.log
uses: actions/upload-artifact@v3
with:
name: logs-wheels-Linux-x86_64
name: logs-wheels-and-pex-Linux-x86_64
path: .pants.d/*.log
timeout-minutes: 90
build_wheels_macos10_15_x86_64:
Expand Down Expand Up @@ -424,17 +423,17 @@ jobs:
- env:
ARCHFLAGS: -arch x86_64
name: Build wheels
run: './pants run src/python/pants_release/release.py -- build-local-pex

./pants run src/python/pants_release/release.py -- build-wheels

'
run: ./pants run src/python/pants_release/release.py -- build-wheels
- env:
ARCHFLAGS: -arch x86_64
name: Build Pants PEX
run: ./pants package src/python/pants:pants-pex
- continue-on-error: true
if: always()
name: Upload pants.log
uses: actions/upload-artifact@v3
with:
name: logs-wheels-macOS10-15-x86_64
name: logs-wheels-and-pex-macOS10-15-x86_64
path: .pants.d/*.log
timeout-minutes: 90
build_wheels_macos11_arm64:
Expand Down Expand Up @@ -479,17 +478,17 @@ jobs:
- env:
ARCHFLAGS: -arch arm64
name: Build wheels
run: './pants run src/python/pants_release/release.py -- build-local-pex

./pants run src/python/pants_release/release.py -- build-wheels

'
run: ./pants run src/python/pants_release/release.py -- build-wheels
- env:
ARCHFLAGS: -arch arm64
name: Build Pants PEX
run: ./pants package src/python/pants:pants-pex
- continue-on-error: true
if: always()
name: Upload pants.log
uses: actions/upload-artifact@v3
with:
name: logs-wheels-macOS11-ARM64
name: logs-wheels-and-pex-macOS11-ARM64
path: .pants.d/*.log
timeout-minutes: 90
check_labels:
Expand Down
18 changes: 2 additions & 16 deletions docs/markdown/Contributions/releases/release-process.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,22 +196,8 @@ Then, run:

This will tag the release with your PGP key, and push the tag to origin, which will kick off a [`Release` job](https://github.com/pantsbuild/pants/actions/workflows/release.yaml) to build the wheels and publish them to PyPI.

Step 4: Release a Pants PEX
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

All this red makes me happy!

---------------------------

After the [`Release` job](https://github.com/pantsbuild/pants/actions/workflows/release.yaml) for your tag has completed, you should additionally build and publish the "universal" PEX to Github.

```bash
PANTS_PEX_RELEASE=STABLE ./pants run src/python/pants_release/release.py -- build-universal-pex
```

Then:

- Go to <https://github.com/pantsbuild/pants/releases>, find your release.
- Attach the PEX located at `dist/pex.pants.<version>.pex`.
- Click "Publish release"

Step 5: Test the release
Step 4: Test the release
------------------------

Run this script as a basic smoke test:
Expand All @@ -222,7 +208,7 @@ Run this script as a basic smoke test:

You should also [check PyPI](https://pypi.org/pypi/pantsbuild.pants) to ensure everything looks good. Click "Release history" to find the version you released, then click it and confirm the changelog is correct on the "Project description" page and that the `macOS` and `manylinux` wheels show up in the "Download files" page.

Step 6: Announce the change
Step 5: Announce the change
---------------------------

Announce the release to:
Expand Down
10 changes: 10 additions & 0 deletions src/python/pants/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,16 @@ python_distribution(
entry_points={"console_scripts": {"pants": "src/python/pants/bin:pants"}},
)

pex_binary(
name="pants-pex",
dependencies=[":pants-packaged"],
script="pants",
execution_mode="venv",
shebang="/usr/bin/env python",
strip_pex_env=False,
layout="zipapp",
)
Comment on lines +43 to +51
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was copied from release.py, but without the --platforms flag. As I understand it, this will resolve only for the local platform, so that's exactly what we want!

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I also am hardcoding zipapp PEX to guard against __defaults__ accidentally changing that.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe that you understand correctly.


# NB: we use `dummy.c` to avoid clang/gcc complaining `error: no input files` when building
# `:pants-packaged`. We don't actually need to use any meaningful file here, though, because we
# use `entry_points` to link to the actual native code, so clang/gcc do not need to build any
Expand Down
50 changes: 28 additions & 22 deletions src/python/pants_release/generate_github_workflows.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -534,26 +534,6 @@ def bootstrap_pants(self) -> Sequence[Step]:
self.native_binaries_upload(),
]

def build_wheels(self) -> list[Step]:
cmd = dedent(
# Note that the build-local-pex run is just for smoke-testing that pex
# builds work, and it must come *before* the build-wheels runs, since
# it cleans out `dist/deploy`, which the build-wheels runs populate for
# later attention by deploy_to_s3.py.
"""\
./pants run src/python/pants_release/release.py -- build-local-pex
./pants run src/python/pants_release/release.py -- build-wheels
"""
)

return [
{
"name": "Build wheels",
"run": cmd,
"env": self.platform_env(),
},
]

def upload_log_artifacts(self, name: str) -> Step:
return {
"name": "Upload pants.log",
Expand Down Expand Up @@ -867,9 +847,35 @@ def build_wheels_job(
"steps": [
*initial_steps,
*([] if platform == Platform.LINUX_ARM64 else [install_go()]),
*helper.build_wheels(),
helper.upload_log_artifacts(name="wheels"),
{
"name": "Build wheels",
"run": "./pants run src/python/pants_release/release.py -- build-wheels",
"env": helper.platform_env(),
},
{
"name": "Build Pants PEX",
"run": "./pants package src/python/pants:pants-pex",
thejcannon marked this conversation as resolved.
Show resolved Hide resolved
"env": helper.platform_env(),
},
helper.upload_log_artifacts(name="wheels-and-pex"),
*([deploy_to_s3("Deploy wheels to S3")] if for_deploy_ref else []),
*(
[
{
"name": "Upload Pants PEX",
"if": "needs.release_info.outputs.is-release == 'true'",
"run": dedent(
"""\
LOCAL_TAG=$(PEX_INTERPRETER=1 dist/src.python.pants/pants-pex.pex -c "import sys;major, minor = sys.version_info[:2];import os;uname = os.uname();print(f'cp{major}{minor}-{uname.sysname.lower()}_{uname.machine.lower()}')")
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is my assume-nothing way of generating the tag. Using the PEX itself means we'll land the right IC, and using Python means we'll be confident scie-pants's Python script can produce a similar tag.

The value looks like: cp39_linux_x86_64

mv dist/src.python.pants/pants-pex.pex dist/src.python.pants/pants.$LOCAL_TAG.pex
gh release upload --no-clobber ${{ needs.release_info.outputs.build-ref }} dist/src.python.pants/pants.$LOCAL_TAG.pex
"""
),
}
]
if for_deploy_ref
else []
),
],
},
}
Expand Down
Loading