diff --git a/.github/workflows/base-glibc-busybox-bash.yaml b/.github/workflows/base-glibc-busybox-bash.yaml index fc037a2b..9c0c4169 100644 --- a/.github/workflows/base-glibc-busybox-bash.yaml +++ b/.github/workflows/base-glibc-busybox-bash.yaml @@ -14,74 +14,101 @@ on: jobs: build: name: Build & Push - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 env: # The base image is not intended to change often and should be used with # version tags or checksum IDs, but not via "latest". - IMAGE_VERSION: '2.1.0' + MAJOR_VERSION: 3 + MINOR_VERSION: 0 IMAGE_NAME: base-glibc-busybox-bash - BUSYBOX_VERSION: '1.32.1' - DEBIAN_VERSION: '10.9' + BUSYBOX_VERSION: '1.36.1' + DEBIAN_VERSION: '12.2' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: arm64 - name: Build - id: buildah-build + id: build run: | set -xeu cd 'images/${{ env.IMAGE_NAME }}' + image_name='${{ env.IMAGE_NAME }}' + tags=' + ${{ env.MAJOR_VERSION }} + ${{ env.MAJOR_VERSION }}.${{ env.MINOR_VERSION }} + latest + ' + printf %s\\n \ + "image=${image_name}" \ + "tags=$( echo ${tags} )" \ + >> $GITHUB_OUTPUT + + for tag in ${tags} ; do + buildah manifest create "${image_name}:${tag}" + done + iidfile="$( mktemp )" - buildah bud --layers \ + buildah bud \ --iidfile="${iidfile}" \ - --build-arg=busybox_version="${{ env.BUSYBOX_VERSION }}" \ - --build-arg=debian_version="${{ env.DEBIAN_VERSION }}" - image_id="$( cat "${iidfile}" )" + --build-arg=busybox_version='${{ env.BUSYBOX_VERSION }}' \ + --file=Dockerfile.busybox + busybox_image="$( cat "${iidfile}" )" rm "${iidfile}" - container="$( buildah from "${image_id}" )" - run() { buildah run "${container}" "${@}" ; } - deb_list="$( run cat /.deb.lst )" - pkg_list="$( run cat /.pkg.lst )" - glibc="$( run sh -c 'exec "$( find /lib -name libc.so.6 -print -quit )"' | sed '1!d' )" - busybox="$( run busybox | sed '1!d' )" - bash="$( run bash --version | sed '1!d' )" - buildah rm "${container}" - - container="$( buildah from "${image_id}" )" - buildah config --label=glibc="${glibc}" "${container}" - buildah config --label=busybox="${busybox}" "${container}" - buildah config --label=deb-list="${deb_list}" "${container}" - buildah config --label=pkg-list="${pkg_list}" "${container}" - - glibc_version="$( printf %s "${glibc}" | sed -E 's/.*version ([0-9.]*[0-9]).*/\1/' )" - busybox_version="$( printf %s "${busybox}" | sed -E '1 s/.*v([0-9.]*[0-9]).*/\1/' )" - bash_version="$( printf %s "${bash}" | sed -E 's/.*version ([0-9.]*[0-9]).*/\1/' )" - tags=" - ${{ env.IMAGE_VERSION }} - ${{ env.IMAGE_VERSION }}_${glibc_version}_${busybox_version}_${bash_version} - latest - " + for arch in amd64 arm64 ; do + iidfile="$( mktemp )" + buildah bud --layers \ + --arch="${arch}" \ + --iidfile="${iidfile}" \ + --build-arg=busybox_image="${busybox_image}" \ + --build-arg=debian_version='${{ env.DEBIAN_VERSION }}' + image_id="$( cat "${iidfile}" )" + rm "${iidfile}" - image_id="$( buildah commit "${container}" )" - buildah rm "${container}" - image_name='${{ env.IMAGE_NAME }}' + container="$( buildah from "${image_id}" )" + run() { buildah run "${container}" "${@}" ; } + deb_list="$( run cat /.deb.lst | tr '\n' '|' | sed 's/|$//' )" + pkg_list="$( run cat /.pkg.lst | tr '\n' '|' | sed 's/|$//' )" + glibc="$( run sh -c 'exec "$( find -xdev -name libc.so.6 -print -quit )"' | sed '1!d' )" + busybox="$( run busybox | sed '1!d' )" + bash="$( run bash --version | sed '1!d' )" + buildah rm "${container}" - for tag in ${tags} ; do - buildah tag "${image_id}" \ - "${image_name}":"${tag}" - done + container="$( buildah from "${image_id}" )" + buildah config \ + --label=glibc="${glibc}" \ + --label=busybox="${busybox}" \ + --label=bash="${bash}" \ + --label=deb-list="${deb_list}" \ + --label=pkg-list="${pkg_list}" \ + "${container}" - echo "::set-output name=image::${image_name}" - echo "::set-output name=tags::$( echo ${tags} )" + image_id="$( buildah commit "${container}" )" + buildah rm "${container}" + for tag in ${tags} ; do + buildah manifest add \ + "${image_name}:${tag}" \ + "${image_id}" + done + done - name: Test run: | - image='${{ steps.buildah-build.outputs.image }}' + image='${{ steps.build.outputs.image }}' ids="$( - for tag in ${{ steps.buildah-build.outputs.tags }} ; do - buildah images --quiet --no-trunc "${image}:${tag}" + for tag in ${{ steps.build.outputs.tags }} ; do + buildah manifest inspect "${image}:${tag}" \ + | jq -r '.manifests[]|.digest' \ + | while read id ; do + buildah images --format '{{.ID}}{{.Digest}}' \ + | sed -n "s/${id}//p" + done done )" ids="$( printf %s "${ids}" | sort -u )" @@ -98,38 +125,40 @@ jobs: run: | # FIX upstream: Quay.io does not support immutable images currently. # => Try to use the REST API to check for duplicate tags. - respone="$( + response="$( curl -sL \ - 'https://quay.io/api/v1/repository/bioconda/${{ steps.buildah-build.outputs.image }}/image' + 'https://quay.io/api/v1/repository/bioconda/${{ steps.build.outputs.image }}/tag/' )" existing_tags="$( - printf %s "${respone}" \ - | jq -r '.images[].tags[]' + printf %s "${response}" \ + | jq -r '.tags[]|.name' )" \ || { printf %s\\n \ 'Could not get list of image tags.' \ 'Does the repository exist on Quay.io?' \ 'Quay.io REST API response was:' \ - "${respone}" + "${response}" exit 1 } - for tag in ${{ steps.buildah-build.outputs.tags }} ; do - if [ \! "${tag}" = latest ] ; then - if printf %s "${existing_tags}" | grep -qxF "${tag}" ; then - printf 'Tag %s already exists!\n' "${tag}" - exit 1 - fi - fi + for tag in ${{ steps.build.outputs.tags }} ; do + case "${tag}" in + latest | '${{ env.MAJOR_VERSION }}' ) ;; + * ) + if printf %s "${existing_tags}" | grep -qxF "${tag}" ; then + printf 'Tag %s already exists!\n' "${tag}" + exit 1 + fi + esac done - if: ${{ github.ref == 'refs/heads/main' }} name: Push uses: redhat-actions/push-to-registry@v2 with: - image: ${{ steps.buildah-build.outputs.image }} - tags: ${{ steps.buildah-build.outputs.tags }} + image: ${{ steps.build.outputs.image }} + tags: ${{ steps.build.outputs.tags }} registry: ${{ secrets.QUAY_BIOCONDA_REPO }} username: ${{ secrets.QUAY_BIOCONDA_USERNAME }} password: ${{ secrets.QUAY_BIOCONDA_TOKEN }} @@ -137,10 +166,15 @@ jobs: - if: ${{ github.ref == 'refs/heads/main' }} name: Test Pushed run: | - image='${{ steps.buildah-build.outputs.image }}' + image='${{ env.IMAGE_NAME }}' ids="$( - for tag in ${{ steps.buildah-build.outputs.tags }} ; do - buildah images --quiet --no-trunc "${image}:${tag}" + for tag in ${{ steps.build.outputs.tags }} ; do + buildah manifest inspect "${image}:${tag}" \ + | jq -r '.manifests[]|.digest' \ + | while read id ; do + buildah images --format '{{.ID}}{{.Digest}}' \ + | sed -n "s/${id}//p" + done done )" ids="$( printf %s "${ids}" | sort -u )" diff --git a/.github/workflows/base-glibc-debian-bash.yaml b/.github/workflows/base-glibc-debian-bash.yaml index 539aa6a9..f0571986 100644 --- a/.github/workflows/base-glibc-debian-bash.yaml +++ b/.github/workflows/base-glibc-debian-bash.yaml @@ -14,68 +14,91 @@ on: jobs: build: name: Build & Push - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 env: # The base image is not intended to change often and should be used with # version tags or checksum IDs, but not via "latest". - IMAGE_VERSION: '2.1.0' + MAJOR_VERSION: 3 + MINOR_VERSION: 0 IMAGE_NAME: base-glibc-debian-bash - DEBIAN_VERSION: '10.9' + DEBIAN_VERSION: '12.2' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: arm64 - name: Build - id: buildah-build + id: build run: | set -xeu cd 'images/${{ env.IMAGE_NAME }}' - iidfile="$( mktemp )" - buildah bud --layers \ - --iidfile="${iidfile}" \ - --build-arg=debian_version="${{ env.DEBIAN_VERSION }}" - image_id="$( cat "${iidfile}" )" - rm "${iidfile}" + image_name='${{ env.IMAGE_NAME }}' + tags=' + ${{ env.MAJOR_VERSION }} + ${{ env.MAJOR_VERSION }}.${{ env.MINOR_VERSION }} + latest + ' + printf %s\\n \ + "image=${image_name}" \ + "tags=$( echo ${tags} )" \ + >> $GITHUB_OUTPUT - container="$( buildah from "${image_id}" )" - run() { buildah run "${container}" "${@}" ; } - glibc="$( run sh -c 'exec "$( find /lib -name libc.so.6 -print -quit )"' | sed '1!d' )" - debian="$( run cat /etc/debian_version | sed '1!d' )" - bash="$( run bash --version | sed '1!d' )" - buildah rm "${container}" + for tag in ${tags} ; do + buildah manifest create "${image_name}:${tag}" + done - container="$( buildah from "${image_id}" )" - buildah config --label=glibc="${glibc}" "${container}" - buildah config --label=debian="${debian}" "${container}" + for arch in amd64 arm64 ; do + iidfile="$( mktemp )" + buildah bud --layers \ + --arch="${arch}" \ + --iidfile="${iidfile}" \ + --build-arg=debian_version='${{ env.DEBIAN_VERSION }}' + image_id="$( cat "${iidfile}" )" + rm "${iidfile}" - glibc_version="$( printf %s "${glibc}" | sed -E 's/.*version ([0-9.]*[0-9]).*/\1/' )" - debian_version="$( printf %s "${debian}" | sed -E 's|/|_|g' )" - bash_version="$( printf %s "${bash}" | sed -E 's/.*version ([0-9.]*[0-9]).*/\1/' )" - tags=" - ${{ env.IMAGE_VERSION }} - ${{ env.IMAGE_VERSION }}_${glibc_version}_${debian_version}_${bash_version} - latest - " + container="$( buildah from "${image_id}" )" + run() { buildah run "${container}" "${@}" ; } + deb_list="$( run cat /.deb.lst | tr '\n' '|' | sed 's/|$//' )" + pkg_list="$( run cat /.pkg.lst | tr '\n' '|' | sed 's/|$//' )" + glibc="$( run sh -c 'exec "$( find -xdev -name libc.so.6 -print -quit )"' | sed '1!d' )" + debian="$( run cat /etc/debian_version | sed '1!d' )" + bash="$( run bash --version | sed '1!d' )" + buildah rm "${container}" - image_id="$( buildah commit "${container}" )" - buildah rm "${container}" - image_name='${{ env.IMAGE_NAME }}' + container="$( buildah from "${image_id}" )" + buildah config \ + --label=glibc="${glibc}" \ + --label=debian="${debian}" \ + --label=bash="${bash}" \ + --label=deb-list="${deb_list}" \ + --label=pkg-list="${pkg_list}" \ + "${container}" - for tag in ${tags} ; do - buildah tag "${image_id}" \ - "${image_name}":"${tag}" + image_id="$( buildah commit "${container}" )" + buildah rm "${container}" + for tag in ${tags} ; do + buildah manifest add \ + "${image_name}:${tag}" \ + "${image_id}" + done done - echo "::set-output name=image::${image_name}" - echo "::set-output name=tags::$( echo ${tags} )" - - name: Test run: | - image='${{ steps.buildah-build.outputs.image }}' + image='${{ steps.build.outputs.image }}' ids="$( - for tag in ${{ steps.buildah-build.outputs.tags }} ; do - buildah images --quiet --no-trunc "${image}:${tag}" + for tag in ${{ steps.build.outputs.tags }} ; do + buildah manifest inspect "${image}:${tag}" \ + | jq -r '.manifests[]|.digest' \ + | while read id ; do + buildah images --format '{{.ID}}{{.Digest}}' \ + | sed -n "s/${id}//p" + done done )" ids="$( printf %s "${ids}" | sort -u )" @@ -92,41 +115,40 @@ jobs: run: | # FIX upstream: Quay.io does not support immutable images currently. # => Try to use the REST API to check for duplicate tags. - response=$( - curl -H "Authorization: Bearer $TOKEN" \ - -sL \ - 'https://quay.io/api/v1/repository/bioconda/${{ steps.buildah-build.outputs.image }}/image' - ) + response="$( + curl -sL \ + 'https://quay.io/api/v1/repository/bioconda/${{ steps.build.outputs.image }}/tag/' + )" existing_tags="$( printf %s "${response}" \ - | jq -r '.images[].tags[]' + | jq -r '.tags[]|.name' )" \ || { printf %s\\n \ 'Could not get list of image tags.' \ 'Does the repository exist on Quay.io?' \ 'Quay.io REST API response was:' \ - "${respone}" + "${response}" exit 1 } - for tag in ${{ steps.buildah-build.outputs.tags }} ; do - if [ \! "${tag}" = latest ] ; then - if printf %s "${existing_tags}" | grep -qxF "${tag}" ; then - printf 'Tag %s already exists!\n' "${tag}" - exit 1 - fi - fi + for tag in ${{ steps.build.outputs.tags }} ; do + case "${tag}" in + latest | '${{ env.MAJOR_VERSION }}' ) ;; + * ) + if printf %s "${existing_tags}" | grep -qxF "${tag}" ; then + printf 'Tag %s already exists!\n' "${tag}" + exit 1 + fi + esac done - env: - TOKEN: ${{ secrets.secrets.QUAY_BIOCONDA_TOKEN }} - if: ${{ github.ref == 'refs/heads/main' }} name: Push uses: redhat-actions/push-to-registry@v2 with: - image: ${{ steps.buildah-build.outputs.image }} - tags: ${{ steps.buildah-build.outputs.tags }} + image: ${{ steps.build.outputs.image }} + tags: ${{ steps.build.outputs.tags }} registry: ${{ secrets.QUAY_BIOCONDA_REPO }} username: ${{ secrets.QUAY_BIOCONDA_USERNAME }} password: ${{ secrets.QUAY_BIOCONDA_TOKEN }} @@ -134,12 +156,17 @@ jobs: - if: ${{ github.ref == 'refs/heads/main' }} name: Test Pushed run: | - image='${{ steps.buildah-build.outputs.image }}' + image='${{ env.IMAGE_NAME }}' ids="$( - for tag in ${{ steps.buildah-build.outputs.tags }} ; do - buildah images --quiet --no-trunc "${image}:${tag}" + for tag in ${{ steps.build.outputs.tags }} ; do + buildah manifest inspect "${image}:${tag}" \ + | jq -r '.manifests[]|.digest' \ + | while read id ; do + buildah images --format '{{.ID}}{{.Digest}}' \ + | sed -n "s/${id}//p" + done done - )" + )" ids="$( printf %s "${ids}" | sort -u )" for id in ${ids} ; do podman history "${id}" diff --git a/images/base-glibc-busybox-bash/Dockerfile b/images/base-glibc-busybox-bash/Dockerfile index 1c877de2..e875a2d4 100644 --- a/images/base-glibc-busybox-bash/Dockerfile +++ b/images/base-glibc-busybox-bash/Dockerfile @@ -1,65 +1,93 @@ -ARG busybox_version=1.32.1 -ARG debian_version=10.8 - # Don't use Debian's busybox package since it only provides a smaller subset of # BusyBox's functions (e.g., no administrative tools like adduser etc.). -# Since we create a glibc image anyway, we can also use "busybox:glibc" as the -# base image, use a the slightly smaller dynamically linked binary and reuse -# base files (e.g., /etc/passwd) from that image. +# Since we create a glibc image anyway, we can also use a the slightly smaller +# dynamically linked binary. -FROM "busybox:${busybox_version}-glibc" AS target_base +ARG debian_version FROM "debian:${debian_version}-slim" AS build_base - - -# Build busybox ourselves to have more fine-grained control over what we want -# (or not want) to include. -# Use old Debian version to ensure compatible (low glibc requirement) binaries. -FROM debian:9-slim AS busybox_builder -RUN apt-get update && \ - apt-get install -y \ - bzip2 curl ca-certificates tar gcc gnupg dirmngr make -COPY build-busybox /usr/local/bin -ARG busybox_version -RUN build-busybox "${busybox_version}" /busybox +RUN [ ! -f /etc/apt/sources.list ] || sed --in-place= --regexp-extended \ + '/ stretch/ { s,-updates,-backports, ; s,/(deb|security)\.,/archive., }' \ + /etc/apt/sources.list FROM build_base AS rootfs_builder -WORKDIR /rootfs -COPY --from=target_base / ./ - -RUN find . -samefile ./bin/busybox -delete -COPY --from=busybox_builder /busybox/busybox ./ -RUN mkdir -p \ - ./bin ./usr/bin \ - ./sbin ./usr/sbin \ +ARG busybox_image +COPY --from="${busybox_image}" /build /build +WORKDIR /busybox-rootfs +RUN arch="$( uname -m )" \ && \ - chroot . /busybox --install \ + mkdir -p ./bin ./sbin ./usr/bin ./usr/sbin \ && \ - # Somehow (container layers?) busybox does not have the same inode if it's - # directly put at /bin/busybox => hardlink it manually afterwards. - rm ./busybox && ln ./bin/ln ./bin/busybox + cp -al "/build/busybox.${arch}" ./bin/busybox \ + && \ + ldd ./bin/busybox \ + | grep --only-matching --extended-regexp '/lib\S+' \ + | xargs -n1 sh -xc 'mkdir -p ".${1%/*}" && cp -aL "${1}" ".${1%/*}"' -- \ + && \ + chroot . /bin/busybox --install \ + && \ + rm -rf ./lib* -# Remove glibc files. They are incomplete and get substituted by `install-pkgs`. -RUN rm -rf ./lib ./lib64 +WORKDIR /rootfs + +RUN mkdir -p ./etc ./home ./opt ./root ./run /tmp ./usr ./var/log \ + && \ + for dir in bin lib sbin ; do \ + mkdir "./usr/${dir}" \ + && \ + if [ -L "/bin" ] ; then \ + ln -s "usr/${dir}" "./${dir}" ; \ + else \ + mkdir "./${dir}" ; \ + fi ; \ + done + +RUN find /busybox-rootfs -type f \ + -exec sh -c 'cp -al -- "${1}" "./${1#/busybox-rootfs/}"' -- '{}' ';' # Install helper tools used by install-pkgs. RUN apt-get update -qq \ && \ - apt-get install --yes --no-install-recommends \ + DEBIAN_FRONTEND=noninteractive \ + apt-get install --yes --no-install-recommends \ patchelf COPY install-pkgs /usr/local/bin RUN install-pkgs "$( pwd )" /tmp/work \ bash \ - ncurses-base \ + base-passwd \ libc-bin \ + login \ + ncurses-base \ && \ # Remove contents of /usr/local as downstream images overwrite those. find ./usr/local/ \ -mindepth 1 -depth \ -delete +RUN while IFS=: read _ _ uid gid _ home _ ; do \ + [ -n "${home##/var/run/*}" ] || home="${home#/var}" \ + && \ + [ -d "./${home#/}" ] || [ "${home}" = "/nonexistent" ] && continue ; \ + mkdir -p "./${home#/}" \ + && \ + chown "${uid}:${gid}" "./${home#/}" \ + && \ + chmod 775 "./${home#/}" \ + ; done < ./etc/passwd \ + && \ + pwck --read-only --root "$( pwd )" \ + | { ! grep -v -e 'no changes' -e '/nonexistent' ; } \ + && \ + grpck --read-only --root "$( pwd )" \ + && \ + find \ + -xdev -type f \! -path ./var/\* \! -path ./usr/share/\* \! -name \*.pl \ + | xargs -P0 -n100 sh -c \ + 'chroot . ldd -- "${@}" 2> /dev/null | sed -n "/:/h; /not found/{x;p;x;p}"' -- \ + | { ! grep . ; } + # env-activate.sh (+ optionally env-execute) should be overwritten downstream. # - env-activate.sh: # Is sourced (via symlink in /etc/profile.d/) to activate the /usr/local env. diff --git a/images/base-glibc-busybox-bash/Dockerfile.busybox b/images/base-glibc-busybox-bash/Dockerfile.busybox new file mode 100644 index 00000000..fcbd60bd --- /dev/null +++ b/images/base-glibc-busybox-bash/Dockerfile.busybox @@ -0,0 +1,23 @@ +# Build busybox ourselves to have more fine-grained control over what we want +# (or not want) to include. +# Use old Debian version to ensure compatible (low glibc requirement) binaries. +FROM debian:9-slim AS busybox_builder +RUN [ ! -f /etc/apt/sources.list ] || sed --in-place= --regexp-extended \ + '/ stretch/ { s,-updates,-backports, ; s,/(deb|security)\.,/archive., }' \ + /etc/apt/sources.list \ + && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive \ + apt-get install --yes --no-install-recommends \ + bzip2 curl ca-certificates tar \ + gcc libc6-dev \ + gcc-aarch64-linux-gnu libc6-dev-arm64-cross \ + make patch + +WORKDIR /build +COPY build-busybox ./ +ARG busybox_version +RUN ./build-busybox \ + "${busybox_version}" \ + x86_64 aarch64 + diff --git a/images/base-glibc-busybox-bash/Dockerfile.test b/images/base-glibc-busybox-bash/Dockerfile.test index d1f8e5d9..feba4402 100644 --- a/images/base-glibc-busybox-bash/Dockerfile.test +++ b/images/base-glibc-busybox-bash/Dockerfile.test @@ -17,9 +17,11 @@ RUN [ "$( sh -lc 'printf world' )" = 'world' ] \ printf '' \ > /usr/local/env-activate.sh -COPY --from=debian:9-slim /lib/x86_64-linux-gnu/libz.so* /lib/x86_64-linux-gnu/ -RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ +RUN arch=$(uname -m) \ && \ - sh ./Miniconda3-latest-Linux-x86_64.sh -bp /opt/conda \ + wget --quiet \ + "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-${arch}.sh" \ + && \ + sh ./Miniforge3-Linux-${arch}.sh -bp /opt/conda \ && \ /opt/conda/bin/conda info --all diff --git a/images/base-glibc-busybox-bash/build-busybox b/images/base-glibc-busybox-bash/build-busybox index 7221a0da..902b3375 100755 --- a/images/base-glibc-busybox-bash/build-busybox +++ b/images/base-glibc-busybox-bash/build-busybox @@ -1,54 +1,140 @@ #! /bin/sh set -xeu -version="${1}" -shift -work="${1}" -shift - -mkdir -p "${work}" -cd "${work}" - -curl -L \ - "https://busybox.net/downloads/busybox-${busybox_version}.tar.bz2" \ - | tar -xjf- --strip-components=1 - -make defconfig -mv .config .defconfig -# Set CONFIG_SUBST_WCHAR=0 for better Unicode support and remove big components. -printf %s\\n \ - CONFIG_AR=y \ - CONFIG_FEATURE_AR_CREATE=y \ - CONFIG_FEATURE_AR_LONG_FILENAMES=y \ - CONFIG_SUBST_WCHAR=0 \ - CONFIG_RPM=n \ - CONFIG_RPM2CPIO=n \ - CONFIG_FSCK_MINIX=n \ - CONFIG_MKFS_MINIX=n \ - CONFIG_BC=n \ - CONFIG_DC=n \ - CONFIG_HDPARM=n \ - CONFIG_HEXEDIT=n \ - CONFIG_I2CGET=n \ - CONFIG_I2CSET=n \ - CONFIG_I2CDUMP=n \ - CONFIG_I2CDETECT=n \ - CONFIG_I2CTRANSFER=n \ - CONFIG_DNSD=n \ - CONFIG_FTPD=n \ - CONFIG_HTTPD=n \ - CONFIG_TCPSVD=n \ - CONFIG_UDPSVD=n \ - CONFIG_UDHCPD=n \ - CONFIG_SH_IS_ASH=n \ - CONFIG_SH_IS_NONE=y \ - CONFIG_SHELL_ASH=n \ - CONFIG_ASH=n \ - CONFIG_HUSH=n \ - CONFIG_SHELL_HUSH=n \ - | cat - .defconfig \ - > .config -# make still asks for choosing a although CONFIG_SH_IS_NONE=y is set!? -printf \\n | make oldconfig - -make -j "$( nproc )" busybox +download() { + curl --location --silent \ + "https://busybox.net/downloads/busybox-${version}.tar.bz2" \ + | tar -xjf- --strip-components=1 +} + +patch() { + case "${version}" in 1.36.* ) + # Small fix to let it build with older glibc versions. + curl --location --silent \ + 'https://git.busybox.net/busybox/patch/miscutils/seedrng.c?id=200a9669fbf6f06894e4243cccc9fc11a1a6073a' \ + 'https://git.busybox.net/busybox/patch/miscutils/seedrng.c?id=cb57abb46f06f4ede8d9ccbdaac67377fdf416cf' \ + | command patch --strip=1 + esac + + # Add support for running busybox wget without OpenSSL under QEMU. + # (NB: If we run into other QEMU+BusyBox problems that needs debugging: That + # vfork issue might affect other BusyBox parts, so check for it first.) + command patch --strip=1 <<'EOP' +From e7b57533ffcd5842fa93f5aa96949b3eaed54b67 Mon Sep 17 00:00:00 2001 +From: Marcel Bargull +Date: Sat, 14 Oct 2023 22:58:42 +0200 +Subject: [PATCH] wget: don't assume vfork blocking for openssl exec + +Under QEMU, busybox wget fails to fallback to busybox ssl_client in case +openssl s_client can't be executed because QEMU's vfork does not block. +Ref.: https://man7.org/linux/man-pages/man2/vfork.2.html#VERSIONS + +Signed-off-by: Marcel Bargull +--- + networking/wget.c | 24 +++++++++++++++++++++--- + 1 file changed, 21 insertions(+), 3 deletions(-) + +diff --git a/networking/wget.c b/networking/wget.c +index 9ec0e67b9..4bcc26e86 100644 +--- a/networking/wget.c ++++ b/networking/wget.c +@@ -683,3 +683,9 @@ static int spawn_https_helper_openssl(const char *host, unsigned port) + int pid; +- IF_FEATURE_WGET_HTTPS(volatile int child_failed = 0;) ++ ++# if ENABLE_FEATURE_WGET_HTTPS ++ struct fd_pair status; ++ int exec_errno = 0; ++ ++ xpiped_pair(status); ++# endif + +@@ -701,2 +707,7 @@ static int spawn_https_helper_openssl(const char *host, unsigned port) + ++# if ENABLE_FEATURE_WGET_HTTPS ++ close(status.rd); ++ if (fcntl(status.wr, F_SETFD, FD_CLOEXEC) != 0) ++ bb_simple_perror_msg_and_die("fcntl"); ++# endif + close(sp[0]); +@@ -743,5 +754,8 @@ static int spawn_https_helper_openssl(const char *host, unsigned port) + BB_EXECVP(argv[0], argv); ++ exec_errno = errno; + xmove_fd(3, 2); + # if ENABLE_FEATURE_WGET_HTTPS +- child_failed = 1; ++ if (write(status.wr, &exec_errno, sizeof(exec_errno)) != sizeof(exec_errno)) ++ bb_simple_perror_msg_and_die("write"); ++ close(status.wr); + xfunc_die(); +@@ -758,3 +772,7 @@ static int spawn_https_helper_openssl(const char *host, unsigned port) + # if ENABLE_FEATURE_WGET_HTTPS +- if (child_failed) { ++ close(status.wr); ++ if (read(status.rd, &exec_errno, sizeof(exec_errno)) == -1) ++ bb_simple_perror_msg_and_die("read"); ++ close(status.rd); ++ if (exec_errno) { + close(sp[0]); +EOP +} + +config() { + make defconfig + mv .config .defconfig + # Set CONFIG_SUBST_WCHAR=0 for better Unicode support and remove big components. + printf %s\\n \ + CONFIG_AR=y \ + CONFIG_FEATURE_AR_CREATE=y \ + CONFIG_FEATURE_AR_LONG_FILENAMES=y \ + CONFIG_SUBST_WCHAR=0 \ + CONFIG_RPM=n \ + CONFIG_RPM2CPIO=n \ + CONFIG_FSCK_MINIX=n \ + CONFIG_MKFS_MINIX=n \ + CONFIG_BC=n \ + CONFIG_DC=n \ + CONFIG_HDPARM=n \ + CONFIG_HEXEDIT=n \ + CONFIG_I2CGET=n \ + CONFIG_I2CSET=n \ + CONFIG_I2CDUMP=n \ + CONFIG_I2CDETECT=n \ + CONFIG_I2CTRANSFER=n \ + CONFIG_DNSD=n \ + CONFIG_FTPD=n \ + CONFIG_HTTPD=n \ + CONFIG_TCPSVD=n \ + CONFIG_UDPSVD=n \ + CONFIG_UDHCPD=n \ + CONFIG_SH_IS_ASH=n \ + CONFIG_SH_IS_NONE=y \ + CONFIG_SHELL_ASH=n \ + CONFIG_ASH=n \ + CONFIG_HUSH=n \ + CONFIG_SHELL_HUSH=n \ + | cat - .defconfig \ + > .config + # make still asks which shell to use for sh although CONFIG_SH_IS_NONE=y is set!? + printf \\n | make oldconfig +} + +build() { + make -j "$( nproc )" busybox +} + +main() { + version="${1}" + shift + download + patch + for target ; do + export MAKEFLAGS="ARCH=${target} CROSS_COMPILE=${target}-linux-gnu-" + make clean + config + build + cp -al ./busybox "./busybox.${target}" + done +} + +main "${@}" diff --git a/images/base-glibc-busybox-bash/install-pkgs b/images/base-glibc-busybox-bash/install-pkgs index 7a9a14fe..fdb483dd 100755 --- a/images/base-glibc-busybox-bash/install-pkgs +++ b/images/base-glibc-busybox-bash/install-pkgs @@ -1,15 +1,34 @@ #! /bin/sh set -xeu +arch=$(uname -m) prepare_remove_docs() { # remove lintian and docs (apart from copyright) - rm -rf ./usr/share/{lintian,man} + rm -rf \ + ./usr/share/lintian \ + ./usr/share/man find ./usr/share/doc/ -type f ! -name copyright -delete find ./usr/share/doc/ -type d -empty -delete } +prepare_usrmerge() { + # If we are on Debian >=12, /bin et al. are symlinks to /usr/ counterparts. + # Since we don't do full apt installs, we accomodate for it here. + if [ -L "${root_fs}/bin" ] ; then + for dir in bin lib* sbin ; do + [ -d "./${dir}" ] || continue + [ -L "./${dir}" ] && continue + mkdir -p ./usr + cp -ral "./${dir}" ./usr/ + rm -rf "./${dir}" + ln -s "usr/${dir}" "${dir}" + done + fi +} + + add_rpath() { local binary="${1}" shift @@ -41,61 +60,92 @@ prepare() { # Update gconv-modules accordingly. # NOTE: When adding/removing any, check required dyn. linked libs! - local gconv_path='./usr/lib/x86_64-linux-gnu/gconv' - local gconv_modules_regex='UTF-\w+|UNICODE|ISO8859-(1|2|15)|CP125(0|1|2)|MACINTOSH' - local gconv_modules_file_tmp='./.tmp.gconv-modules' - - mv "${gconv_path}"/gconv-modules "${gconv_modules_file_tmp}" + local gconv_path="./usr/lib/${arch}-linux-gnu/gconv" + local gconv_modules_regex + if [ -e "${gconv_path}/gconv-modules.d/gconv-modules-extra.conf" ] ; then + gconv_modules_regex="$( + sed -nE 's/^module\s+\S+\s+\S+\s+(\S+)\s+.*/\1/p' \ + < "${gconv_path}/gconv-modules" \ + | sort -u \ + | tr '\n' '|' \ + | sed 's/|$//' + )" + : > "${gconv_path}/gconv-modules.d/gconv-modules-extra.conf" + else + gconv_modules_regex='UTF-\w+|UNICODE|ISO8859-(1|15)|CP1252|ANSI_X3\.110' + local gconv_modules_file_tmp='./.tmp.gconv-modules' + + mv "${gconv_path}"/gconv-modules "${gconv_modules_file_tmp}" + + grep -E \ + '^\s*$|^#|^(alias\s+.*|module\s+[^\s]+\s+[^\s]+)\s+\<('"${gconv_modules_regex}"')(//|\s)' \ + "${gconv_modules_file_tmp}" \ + | sed -nEe '1N;N;/^(#.*)\n.*\1/{D;D};P;D' | cat -s \ + > "${gconv_path}"/gconv-modules + rm "${gconv_modules_file_tmp}" + fi find "${gconv_path}" \ -mindepth 1 -maxdepth 1 \ + -name '*.so' \ + -type f \ -regextype posix-extended \ ! -regex '.*/('"${gconv_modules_regex}"').so' \ -print -delete - grep -E \ - '^\s*$|^#|^(alias\s+.*|module\s+[^\s]+\s+[^\s]+)\s+\<('"${gconv_modules_regex}"')(//|\s)' \ - "${gconv_modules_file_tmp}" \ - | sed -nEe '1N;N;/^(#.*)\n.*\1/{D;D};P;D' | cat -s \ - > "${gconv_path}"/gconv-modules - rm "${gconv_modules_file_tmp}" iconvconfig --prefix ./ ;; bash ) rm -rf ./usr/share/locale # Add custom rpath for libtinfo (see below) to bash binaries. - local new_rpath='/lib/x86_64-linux-gnu/terminfo:/usr/lib/x86_64-linux-gnu/terminfo' + local new_rpath="/lib/${arch}-linux-gnu/terminfo:/usr/lib/${arch}-linux-gnu/terminfo" add_rpath ./bin/bash "${new_rpath}" add_rpath ./usr/bin/clear_console "${new_rpath}" ;; - libtinfo5 | \ - libtinfo6 ) + libtinfo* ) # Move libtinfo libraries to a custom path to ensure it is not # unintentionally used in downstream images. - find ./usr/lib/x86_64-linux-gnu -type f \ + find ./usr/lib/${arch}-linux-gnu -type f \ | { while read binary ; do - add_rpath "${binary}" '/lib/x86_64-linux-gnu/terminfo' + add_rpath "${binary}" "/lib/${arch}-linux-gnu/terminfo" done } - mv ./lib/x86_64-linux-gnu ./temp - mkdir ./lib/x86_64-linux-gnu - mv ./temp ./lib/x86_64-linux-gnu/terminfo + mv ./lib/${arch}-linux-gnu ./temp + mkdir ./lib/${arch}-linux-gnu + mv ./temp ./lib/${arch}-linux-gnu/terminfo - mv ./usr/lib/x86_64-linux-gnu ./temp - mkdir ./usr/lib/x86_64-linux-gnu - mv ./temp ./usr/lib/x86_64-linux-gnu/terminfo + mv ./usr/lib/${arch}-linux-gnu ./temp + mkdir ./usr/lib/${arch}-linux-gnu + mv ./temp ./usr/lib/${arch}-linux-gnu/terminfo + ;; + base-passwd ) + # The dependencies libdebconfclient0 (and libselinux1 for Debian>=12) + # are needed for update-passwd, but we ignore them => remove the binary. + rm ./usr/sbin/update-passwd ;; + login ) + rm -rf ./usr/share/locale + # The following binaries provided by BusyBox or pull in more dependencies + # (PAM, libselinux1, and their dependencies) => remove them. + rm -f \ + ./bin/login \ + ./bin/su \ + ./usr/bin/lastlog \ + ./usr/bin/newgrp \ + ./usr/bin/sg + ;; libc-bin | \ libgcc1 | \ base-files | \ - gcc-6-base | \ - gcc-8-base | \ - gcc-10-base | \ + gcc-*-base | \ libcrypt1 | \ libgcc-s1 | \ + libdebconfclient0 | \ + libpcre* | \ + libselinux1 | \ ncurses-base | \ zlib1g ) : @@ -104,9 +154,10 @@ prepare() { # Abort if we get an unexpected package. printf %s\\n "\`prepare\` not defined for ${pkg}" >&2 return 1 - ;; - esac - prepare_remove_docs + ;; + esac + prepare_remove_docs + prepare_usrmerge } @@ -133,6 +184,25 @@ postinst() { chroot ./ sh /base-files-postinst configure rm ./base-files-postinst ;; + base-passwd ) + mkdir -p "${destdir}/etc" + cp -p --remove-destination \ + "${destdir}/usr/share/base-passwd/group.master" \ + ./etc/group + cp -p --remove-destination \ + "${destdir}/usr/share/base-passwd/passwd.master" \ + ./etc/passwd + DPKG_ROOT="$( pwd )" \ + shadowconfig on + ;; + login ) + for file in /var/log/faillog /etc/subuid /etc/subgid ; do + [ -f "./${file}" ] || continue + touch "${file}" + chown 0:0 "${file}" + chmod 644 "${file}" + done + ;; bash ) # Replace BusyBox's sh by Bash rm -f ./bin/sh @@ -153,17 +223,17 @@ postinst() { EOF ;; libc6 | \ + libdebconfclient0 | \ libgcc1 | \ libcrypt1 | \ libgcc-s1 | \ - libtinfo5 | \ - libtinfo6 | \ + libpcre* | \ + libselinux1 | \ + libtinfo* | \ zlib1g ) postinst_ldconfig_trigger ;; - gcc-6-base | \ - gcc-8-base | \ - gcc-10-base | \ + gcc-*-base | \ ncurses-base ) : ;; @@ -222,7 +292,10 @@ get_deps() { local ignore_pkgs ignore_pkgs="$( - printf %s\\n base-files '' debianutils dash \ + printf %s\\n \ + base-files '' debianutils dash \ + libdebconfclient0 libselinux1 \ + libaudit1 libpam-modules libpam-runtime libpam0g \ | grep -vFx "$( printf %s\\n "${@}" )" )" [ -f "${root_fs}/.pkg.lst" ] && \ @@ -270,7 +343,7 @@ main() { # Unconditionally install glibc (package libc6). # Also install dependencies acc. to `apt-cache depends`: # - libgcc1 only consists of libgcc_s.so.1 (+ docs, which we remove). - # - gcc-{6,7,8}-base only has empty directories (+ docs, which we remove). + # - gcc-*-base only has empty directories (+ docs, which we remove). install_with_deps libc6 # libc-bin must be in ${@} for Unicode support (C.UTF-8 locale). diff --git a/images/base-glibc-debian-bash/Dockerfile b/images/base-glibc-debian-bash/Dockerfile index 377e3476..c0adc292 100644 --- a/images/base-glibc-debian-bash/Dockerfile +++ b/images/base-glibc-debian-bash/Dockerfile @@ -1,29 +1,45 @@ -ARG debian_version=10.8 +ARG debian_version FROM "debian:${debian_version}-slim" - -RUN apt-get update -qq \ +RUN [ ! -f /etc/apt/sources.list ] || sed --in-place= --regexp-extended \ + '/ stretch/ { s,-updates,-backports, ; s,/(deb|security)\.,/archive., }' \ + /etc/apt/sources.list \ && \ - apt-get install --yes \ - --no-install-recommends \ - libgl1-mesa-glx \ - locales \ - openssh-client \ - procps \ + apt-get update -qq \ && \ # Add en_US.UTF-8 locale. - sed -i \ - 's/^# *\(en_US.UTF-8\)/\1/' \ - /etc/locale.gen \ + printf '%s\n' 'en_US.UTF-8 UTF-8' \ + >> /etc/locale.gen \ && \ - locale-gen \ + DEBIAN_FRONTEND=noninteractive \ + apt-get install --yes --no-install-recommends \ + $( \ + . /etc/os-release \ + && \ + [ "${VERSION_ID-10}" -lt 10 ] \ + && \ + printf '%s\n' \ + libegl1-mesa \ + libgl1-mesa-glx \ + || \ + printf '%s\n' \ + libegl1 \ + libgl1 \ + libglx-mesa0 \ + ) \ + libglvnd0 \ + libopengl0 \ + locales \ + openssh-client \ + procps \ && \ # Remove "locales" package, but keep the generated locale. sed -i \ 's/\s*rm .*locale-archive$/: &/' \ /var/lib/dpkg/info/locales.prerm \ && \ - apt-get remove --yes \ + DEBIAN_FRONTEND=noninteractive \ + apt-get remove --yes \ locales \ && \ # On Debian 10 (and 11) libgl1-mesa-glx pulls in libgl1-mesa-dri (which in @@ -33,7 +49,8 @@ RUN apt-get update -qq \ '/^Depends:/ s/, libgl1-mesa-dri\>//g' \ /var/lib/dpkg/status \ && \ - apt-get autoremove --yes \ + DEBIAN_FRONTEND=noninteractive \ + apt-get autoremove --yes \ && \ # Remove apt package lists. rm -rf /var/lib/apt/lists/* \ @@ -43,6 +60,38 @@ RUN apt-get update -qq \ -mindepth 1 -depth \ -delete +RUN dpkg-query --show --showformat \ + '${db:Status-Status} ${Package}\n' \ + | sed -n 's/:/%3a/g ; s/^installed //p' \ + > /.pkg.lst \ + && \ + dpkg-query --show --showformat \ + '${db:Status-Status} ${Package}_${Version}_${Architecture}\n' \ + | sed -n 's/:/%3a/g ; s/$/.deb/ ; s/^installed //p' \ + > /.deb.lst + +RUN while IFS=: read _ _ uid gid _ home _ ; do \ + [ -n "${home##/var/run/*}" ] || home="${home#/var}" \ + && \ + [ -d "./${home#/}" ] || [ "${home}" = "/nonexistent" ] && continue ; \ + mkdir -p "./${home#/}" \ + && \ + chown "${uid}:${gid}" "./${home#/}" \ + && \ + chmod 775 "./${home#/}" \ + ; done < ./etc/passwd \ + && \ + pwck --read-only --root "$( pwd )" \ + | { ! grep -v -e 'no changes' -e '/nonexistent' ; } \ + && \ + grpck --read-only --root "$( pwd )" \ + && \ + find \ + -xdev -type f \! -path ./var/\* \! -path ./usr/share/\* \! -name \*.pl \ + | xargs -P0 -n100 sh -c \ + 'chroot . ldd -- "${@}" 2> /dev/null | sed -n "/:/h; /not found/{x;p;x;p}"' -- \ + | { ! grep . ; } + # Bash 4.* did not have default key bindings for control-arrow-key key # combinations. Add some for convenience: RUN >> /etc/inputrc \ diff --git a/images/base-glibc-debian-bash/Dockerfile.test b/images/base-glibc-debian-bash/Dockerfile.test index 30bdfa7f..f2f0bace 100644 --- a/images/base-glibc-debian-bash/Dockerfile.test +++ b/images/base-glibc-debian-bash/Dockerfile.test @@ -24,12 +24,16 @@ RUN locale -a | grep -i 'c\.utf-\?8' \ RUN apt-get update -qq \ && \ - apt-get install --yes --no-install-recommends \ + DEBIAN_FRONTEND=noninteractive \ + apt-get install --yes --no-install-recommends \ ca-certificates \ wget \ && \ - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh \ + arch=$(uname -m) \ && \ - sh ./Miniconda3-latest-Linux-x86_64.sh -bp /opt/conda \ + wget --quiet \ + "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-${arch}.sh" \ + && \ + sh ./Miniforge3-Linux-${arch}.sh -bp /opt/conda \ && \ /opt/conda/bin/conda info --all