From b542349309e815a3f3ca67c905dc69686cc428f1 Mon Sep 17 00:00:00 2001 From: Tomas Sebestik Date: Fri, 18 Aug 2023 08:08:04 +0200 Subject: [PATCH 1/3] feat(target-test-base): Add target-test-base images --- .../build_image_target-test-base_jammy-py.yml | 94 +++++++++++++++++++ target-test-base_jammy-py/Dockerfile | 62 ++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 .github/workflows/build_image_target-test-base_jammy-py.yml create mode 100644 target-test-base_jammy-py/Dockerfile diff --git a/.github/workflows/build_image_target-test-base_jammy-py.yml b/.github/workflows/build_image_target-test-base_jammy-py.yml new file mode 100644 index 0000000..e2ab128 --- /dev/null +++ b/.github/workflows/build_image_target-test-base_jammy-py.yml @@ -0,0 +1,94 @@ +name: Build and push Docker image 'target-test-base_jammy-py' +on: + push: + branches: + - master + paths: + - 'target-test-base_jammy-py/**' + - '.github/workflows/build_image_target-test-base_jammy-py.yml' + pull_request: + paths: + - 'target-test-base_jammy-py/**' + - '.github/workflows/build_image_target-test-base_jammy-py.yml' + workflow_dispatch: + inputs: + python_3_8_17: + description: 'Python 3.8.17' + required: false + type: boolean + default: true + python_3_9_17: + description: 'Python 3.9.17' + required: false + type: boolean + default: false + # Add more versions as needed... + +env: + IMAGE_DIR: ./target-test-base_jammy-py + IMAGE_BASE_NAME: ghcr.io/${{ github.repository }}/target-test-base-jammy + IMAGE_ARCHS: linux/amd64,linux/arm + +jobs: + build-and-push-docker-image: + runs-on: ubuntu-latest + strategy: + matrix: + python_version: ["3.8.17", "3.9.17"] # Add more versions as needed. + include: + - python_version: "3.8.17" + build_this_version: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.python_3_8_17 == 'true' || github.event_name != 'workflow_dispatch' }} + - python_version: "3.9.17" + build_this_version: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.python_3_9_17 == 'true' || github.event_name != 'workflow_dispatch' }} + + steps: + - name: Checkout Repository + if: matrix.build_this_version + uses: actions/checkout@v2 + + - name: Extract Major/Minor Version from Full Version + id: extract_version + run: | + echo "PYTHON_VERSION=${{ matrix.python_version }}" >> $GITHUB_ENV + echo "PYTHON_VERSION_MM=$(echo "${{ matrix.python_version }}" | cut -d'.' -f1,2)" >> $GITHUB_ENV + + - name: Setup QEMU for Multi-Arch Builds + uses: docker/setup-qemu-action@v1 + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to GitHub Container Registry (GHCR) + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Pull Docker image to use as cache (for master) + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + run: | + docker pull ${{ env.IMAGE_BASE_NAME }}:py${{ matrix.python_version }} || true + + - name: Cache Docker layers (for PRs) + if: github.event_name == 'pull_request' + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: buildx-${{ matrix.python_version }}-${{ github.sha }} + restore-keys: | + buildx-${{ matrix.python_version }}- + + - name: Build and Push Docker Image + if: matrix.build_this_version + uses: docker/build-push-action@v2 + with: + context: ${{ env.IMAGE_DIR }} + platforms: ${{env.IMAGE_ARCHS}} + tags: ${{ env.IMAGE_BASE_NAME }}:py${{ matrix.python_version }} + build-args: | + PYTHON_VERSION=${{ env.PYTHON_VERSION }} + PYTHON_VERSION_MM=${{ env.PYTHON_VERSION_MM }} + push: ${{ github.event_name != 'pull_request' }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/target-test-base_jammy-py/Dockerfile b/target-test-base_jammy-py/Dockerfile new file mode 100644 index 0000000..57f02aa --- /dev/null +++ b/target-test-base_jammy-py/Dockerfile @@ -0,0 +1,62 @@ +FROM ubuntu:22.04 AS build-python + +ARG PYTHON_VERSION_MM=3.8 +ARG PYTHON_VERSION=3.8.17 + +# Install essential build tools and necessary runtime libraries for Python compilation +RUN : \ + && apt-get update \ + && apt-get install -y \ + build-essential \ + wget \ + libffi-dev \ + libssl-dev \ + zlib1g-dev \ + liblzma-dev \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev \ + && : + +# Download, compile, and install Python from source code (to have this version available for both amd64 and armv7l) +WORKDIR /tmp +RUN : \ + && wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tar.xz \ + && tar xvf Python-${PYTHON_VERSION}.tar.xz \ + && cd Python-${PYTHON_VERSION} \ + && ./configure --enable-shared --enable-optimizations \ + && make -j$(nproc) \ + && make altinstall \ + && : + +# Update symbolic links for python3 and pip3 (to be able to use commands without version suffix) +RUN : \ + && ln -s /usr/local/bin/python${PYTHON_VERSION_MM} /usr/local/bin/python3 \ + && ln -s /usr/local/bin/pip${PYTHON_VERSION_MM} /usr/local/bin/pip3 \ + && ln -s /usr/local/bin/python${PYTHON_VERSION_MM} /usr/local/bin/python \ + && ln -s /usr/local/bin/pip${PYTHON_VERSION_MM} /usr/local/bin/pip \ + && : + +# ------------------------------------------------------------------------------------------------------------------------------- +FROM ubuntu:22.04 as main + +# Set up required development and runtime dependencies +RUN : \ + && apt-get update \ + && apt-get install -y \ + wget \ + libffi-dev \ + libssl-dev \ + zlib1g-dev \ + liblzma-dev \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && : + +# Copy Python binaries, libraries, includes, and virtualenv from the build-python stage +COPY --from=build-python /usr/local/ /usr/local/ + +CMD [ "/bin/bash" ] From 0777f1a0e1c334da52e231ee734d4682d386bf95 Mon Sep 17 00:00:00 2001 From: Tomas Sebestik Date: Fri, 18 Aug 2023 09:43:55 +0200 Subject: [PATCH 2/3] feat: Add more Python versions to target-test-base image --- .../build_image_target-test-base_jammy-py.yml | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_image_target-test-base_jammy-py.yml b/.github/workflows/build_image_target-test-base_jammy-py.yml index e2ab128..a2d11fd 100644 --- a/.github/workflows/build_image_target-test-base_jammy-py.yml +++ b/.github/workflows/build_image_target-test-base_jammy-py.yml @@ -13,16 +13,30 @@ on: workflow_dispatch: inputs: python_3_8_17: - description: 'Python 3.8.17' + description: 'Build Ubuntu 22.04 (Jammy) with Python 3.8.17' required: false type: boolean default: true python_3_9_17: - description: 'Python 3.9.17' + description: 'Build Ubuntu 22.04 (Jammy) with Python 3.9.17' required: false type: boolean default: false - # Add more versions as needed... + python_3_10_12: + description: 'Build Ubuntu 22.04 (Jammy) with Python 3.10.12' + required: false + type: boolean + default: false + python_3_11_4: + description: 'Build Ubuntu 22.04 (Jammy) with Python 3.11.4' + required: false + type: boolean + default: false + +# Kill running actions pipeline (for PR) when new changes are pushed to PR +concurrency: + group: pr-${{ github.event.pull_request.number }} + cancel-in-progress: true env: IMAGE_DIR: ./target-test-base_jammy-py @@ -34,12 +48,16 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python_version: ["3.8.17", "3.9.17"] # Add more versions as needed. + python_version: ["3.8.17", "3.9.17", "3.10.12", "3.11.4"] include: - python_version: "3.8.17" - build_this_version: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.python_3_8_17 == 'true' || github.event_name != 'workflow_dispatch' }} + build_this_version: true - python_version: "3.9.17" - build_this_version: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.python_3_9_17 == 'true' || github.event_name != 'workflow_dispatch' }} + build_this_version: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.python_3_9_17 == 'true' }} + - python_version: "3.10.12" + build_this_version: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.python_3_10_12 == 'true' }} + - python_version: "3.11.4" + build_this_version: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.python_3_11_4 == 'true' }} steps: - name: Checkout Repository @@ -89,6 +107,7 @@ jobs: build-args: | PYTHON_VERSION=${{ env.PYTHON_VERSION }} PYTHON_VERSION_MM=${{ env.PYTHON_VERSION_MM }} - push: ${{ github.event_name != 'pull_request' }} + push: true #TODO: remove this for production + # push: ${{ github.event_name != 'pull_request' }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,dest=/tmp/.buildx-cache From 65ab6d1dc765b47046b74ccdf9b1fae0f2731fce Mon Sep 17 00:00:00 2001 From: Tomas Sebestik Date: Fri, 18 Aug 2023 11:42:04 +0200 Subject: [PATCH 3/3] fix: Update matrix for build jobs --- .../build_image_target-test-base_jammy-py.yml | 40 +++++++++++++------ target-test-base_jammy-py/Dockerfile | 11 +++++ 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build_image_target-test-base_jammy-py.yml b/.github/workflows/build_image_target-test-base_jammy-py.yml index a2d11fd..9782593 100644 --- a/.github/workflows/build_image_target-test-base_jammy-py.yml +++ b/.github/workflows/build_image_target-test-base_jammy-py.yml @@ -44,24 +44,39 @@ env: IMAGE_ARCHS: linux/amd64,linux/arm jobs: + setup-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Set Matrix + id: set-matrix + run: | + matrix='["3.8.17"]' + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + if [ "${{ github.event.inputs.python_3_9_17 }}" = "true" ]; then + matrix=$(echo $matrix | jq ' . + ["3.9.17"] ') + fi + if [ "${{ github.event.inputs.python_3_10_12 }}" = "true" ]; then + matrix=$(echo $matrix | jq ' . + ["3.10.12"] ') + fi + if [ "${{ github.event.inputs.python_3_11_4 }}" = "true" ]; then + matrix=$(echo $matrix | jq ' . + ["3.11.4"] ') + fi + fi + echo $matrix + echo "::set-output name=matrix::$matrix" + shell: bash + + build-and-push-docker-image: + needs: setup-matrix runs-on: ubuntu-latest strategy: matrix: - python_version: ["3.8.17", "3.9.17", "3.10.12", "3.11.4"] - include: - - python_version: "3.8.17" - build_this_version: true - - python_version: "3.9.17" - build_this_version: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.python_3_9_17 == 'true' }} - - python_version: "3.10.12" - build_this_version: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.python_3_10_12 == 'true' }} - - python_version: "3.11.4" - build_this_version: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.python_3_11_4 == 'true' }} - + python_version: ${{fromJson(needs.setup-matrix.outputs.matrix)}} steps: - name: Checkout Repository - if: matrix.build_this_version uses: actions/checkout@v2 - name: Extract Major/Minor Version from Full Version @@ -98,7 +113,6 @@ jobs: buildx-${{ matrix.python_version }}- - name: Build and Push Docker Image - if: matrix.build_this_version uses: docker/build-push-action@v2 with: context: ${{ env.IMAGE_DIR }} diff --git a/target-test-base_jammy-py/Dockerfile b/target-test-base_jammy-py/Dockerfile index 57f02aa..9789e2d 100644 --- a/target-test-base_jammy-py/Dockerfile +++ b/target-test-base_jammy-py/Dockerfile @@ -59,4 +59,15 @@ RUN : \ # Copy Python binaries, libraries, includes, and virtualenv from the build-python stage COPY --from=build-python /usr/local/ /usr/local/ +# Update the dynamic linker run-time bindings (Python shared libraries are not in the default search path) +RUN ldconfig + +# This prefers the Espressif pypi wheel registry (https://dl.espressif.com/pypi/) over the public PyPI (fallback) +RUN pip install --upgrade pip +ENV PIP_INDEX_URL=https://dl.espressif.com/pypi/ +ENV PIP_EXTRA_INDEX_URL=https://pypi.org/simple + +# Prioritize using binary (wheel) packages over source packages for faster installations and to avoid unnecessary compilations. +ENV PIP_PREFER_BINARY=true + CMD [ "/bin/bash" ]