From 9790963b15f2773dd51e107410a68ed87160df94 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Wed, 31 Mar 2021 22:15:47 +0100 Subject: [PATCH 01/11] Disable psycopg2-binary on arm64 --- images/hub/Dockerfile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/images/hub/Dockerfile b/images/hub/Dockerfile index 26177f5aa7..166f8b8f4b 100644 --- a/images/hub/Dockerfile +++ b/images/hub/Dockerfile @@ -38,7 +38,17 @@ COPY requirements.txt /tmp/requirements.txt RUN pip3 install --upgrade --no-cache-dir \ setuptools \ pip -RUN PYCURL_SSL_LIBRARY=openssl \ + +# Automatically set by docker buildx +ARG TARGETPLATFORM + +# psycopg2-binary not available for arm64 +RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ + echo "Removing psycopg2-binary from dependencies ($TARGETPLATFORM)"; \ + sed -i '/psycopg2-binary==/d' /tmp/requirements.txt; \ + cat /tmp/requirements.txt; \ + fi; \ + PYCURL_SSL_LIBRARY=openssl \ pip install --no-cache-dir \ -r /tmp/requirements.txt From ca11c5e84dd7b1d76bbafc561d51ae1834e4d204 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Wed, 31 Mar 2021 22:59:46 +0100 Subject: [PATCH 02/11] secret-sync uses arch specific tini-static --- images/secret-sync/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/images/secret-sync/Dockerfile b/images/secret-sync/Dockerfile index 60a1480969..30149d4374 100644 --- a/images/secret-sync/Dockerfile +++ b/images/secret-sync/Dockerfile @@ -2,8 +2,11 @@ FROM python:3.8-alpine # VULN_SCAN_TIME=2021-03-27_00:01:53 +# Automatically set by docker buildx +ARG TARGETPLATFORM + # Note that we use tini-static, it embeds dependencies missing in alpine -RUN wget -qO /tini https://github.com/krallin/tini/releases/download/v0.19.0/tini-static \ +RUN wget -qO /tini "https://github.com/krallin/tini/releases/download/v0.19.0/tini-static-${TARGETPLATFORM#*/}" \ && chmod +x /tini # Ensures written logs are made available directly From 09279cd0bfec20659ab08cfb9746cc1ac2647b1f Mon Sep 17 00:00:00 2001 From: Simon Li Date: Thu, 1 Apr 2021 20:32:47 +0100 Subject: [PATCH 03/11] Use `uname -m` instead of TARGETPLATFORM This ensures the image can be built with plain `docker build` --- images/secret-sync/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/images/secret-sync/Dockerfile b/images/secret-sync/Dockerfile index 30149d4374..29983030d6 100644 --- a/images/secret-sync/Dockerfile +++ b/images/secret-sync/Dockerfile @@ -2,11 +2,11 @@ FROM python:3.8-alpine # VULN_SCAN_TIME=2021-03-27_00:01:53 -# Automatically set by docker buildx -ARG TARGETPLATFORM - # Note that we use tini-static, it embeds dependencies missing in alpine -RUN wget -qO /tini "https://github.com/krallin/tini/releases/download/v0.19.0/tini-static-${TARGETPLATFORM#*/}" \ +RUN ARCH=`uname -m`; date; \ + if [ "$ARCH" = x86_64 ]; then ARCH=amd64; fi; \ + if [ "$ARCH" = aarch64 ]; then ARCH=arm64; fi; \ + wget -qO /tini "https://github.com/krallin/tini/releases/download/v0.19.0/tini-static-$ARCH" \ && chmod +x /tini # Ensures written logs are made available directly From 7aab4ef464fb09a243f95cfe2700800cb485b181 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Thu, 1 Apr 2021 20:49:47 +0100 Subject: [PATCH 04/11] Install libpq-dev on architectures that aren't x86_64 --- images/hub/Dockerfile | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/images/hub/Dockerfile b/images/hub/Dockerfile index 166f8b8f4b..3e14d30632 100644 --- a/images/hub/Dockerfile +++ b/images/hub/Dockerfile @@ -4,7 +4,13 @@ FROM ubuntu:20.04 ENV DEBIAN_FRONTEND=noninteractive \ LANG=C.UTF-8 -RUN apt-get update && \ + +# psycopg2-binary in requirements.txt is not compiled for linux/arm64 +# TODO: Use build stages to compile psycopg2-binary separately instead of +# bloating the image size +RUN EXTRA_PACKAGES=; \ + if [ `uname -m` != 'x86_64' ]; then EXTRA_PACKAGES=libpq-dev; fi; \ + apt-get update && \ apt-get install -y --no-install-recommends \ git \ vim \ @@ -20,6 +26,7 @@ RUN apt-get update && \ sqlite3 \ curl \ dnsutils \ + $EXTRA_PACKAGES \ && \ rm -rf /var/lib/apt/lists/* @@ -38,17 +45,7 @@ COPY requirements.txt /tmp/requirements.txt RUN pip3 install --upgrade --no-cache-dir \ setuptools \ pip - -# Automatically set by docker buildx -ARG TARGETPLATFORM - -# psycopg2-binary not available for arm64 -RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ - echo "Removing psycopg2-binary from dependencies ($TARGETPLATFORM)"; \ - sed -i '/psycopg2-binary==/d' /tmp/requirements.txt; \ - cat /tmp/requirements.txt; \ - fi; \ - PYCURL_SSL_LIBRARY=openssl \ +RUN PYCURL_SSL_LIBRARY=openssl \ pip install --no-cache-dir \ -r /tmp/requirements.txt From c1e40100c7e7982c79d1ba3b3eab9108339216b3 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Thu, 1 Apr 2021 21:16:36 +0100 Subject: [PATCH 05/11] test-docker-build.yaml --- .github/workflows/test-docker-build.yaml | 68 ++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 .github/workflows/test-docker-build.yaml diff --git a/.github/workflows/test-docker-build.yaml b/.github/workflows/test-docker-build.yaml new file mode 100644 index 0000000000..fc3d382f7e --- /dev/null +++ b/.github/workflows/test-docker-build.yaml @@ -0,0 +1,68 @@ +# This is a GitHub workflow defining a set of jobs with a set of steps. +# ref: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions +# +name: Test docker multiarch build + +# Trigger the workflow's on all PRs and pushes so that other contributors can +# run tests in their own forks. Avoid triggering these tests on changes to +# documentation only changes. +on: + pull_request: + paths-ignore: + - "doc/**" + - "**/test-docs.yaml" + - "**.md" + - "**/schema.yaml" + push: + paths-ignore: + - "doc/**" + - "**/test-docs.yaml" + - "**.md" + - "**/schema.yaml" + branches-ignore: + - "dependabot/**" + workflow_dispatch: + +jobs: + # TODO: this is just a quick test to check the arm64 docker images + # this should instead be done using chartpress + # Based on + # https://github.com/docker/build-push-action/blob/v2.3.0/docs/advanced/local-registry.md + # https://github.com/docker/build-push-action/blob/v2.3.0/docs/advanced/multi-platform.md + build_images: + runs-on: ubuntu-20.04 + strategy: + # Keep running even if one variation of the job fails + fail-fast: false + matrix: + image: + - hub + - image-awaiter + - network-tools + - secret-sync + # - singleuser-sample + + steps: + - uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + # https://github.com/docker/login-action/tree/v1.8.0#github-container-registry + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push to local registry + uses: docker/build-push-action@v2 + with: + context: images/${{ matrix.image }} + platforms: linux/amd64,linux/arm64 + push: true + tags: ghcr.io/${{ github.repository_owner }}/z2jh-${{ matrix.image }}:dev From f3e72b33a5ea0324de63319e89cdaab8631350d2 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Fri, 2 Apr 2021 21:31:33 +0100 Subject: [PATCH 06/11] hub/Dockerfile use $EXTRA_APT_PACKAGES --- images/hub/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/images/hub/Dockerfile b/images/hub/Dockerfile index 3e14d30632..d6ed141ae5 100644 --- a/images/hub/Dockerfile +++ b/images/hub/Dockerfile @@ -8,8 +8,8 @@ ENV DEBIAN_FRONTEND=noninteractive \ # psycopg2-binary in requirements.txt is not compiled for linux/arm64 # TODO: Use build stages to compile psycopg2-binary separately instead of # bloating the image size -RUN EXTRA_PACKAGES=; \ - if [ `uname -m` != 'x86_64' ]; then EXTRA_PACKAGES=libpq-dev; fi; \ +RUN EXTRA_APT_PACKAGES=; \ + if [ `uname -m` != 'x86_64' ]; then EXTRA_APT_PACKAGES=libpq-dev; fi; \ apt-get update && \ apt-get install -y --no-install-recommends \ git \ @@ -26,7 +26,7 @@ RUN EXTRA_PACKAGES=; \ sqlite3 \ curl \ dnsutils \ - $EXTRA_PACKAGES \ + $EXTRA_APT_PACKAGES \ && \ rm -rf /var/lib/apt/lists/* From 81fdd6e014960f925ceee274062d3a7747d5c9a8 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Sat, 10 Apr 2021 16:59:15 +0100 Subject: [PATCH 07/11] Try building/pushing arm64 images with chartpress dev version --- .github/workflows/test-docker-build.yaml | 31 ++++++++++-------------- chartpress.yaml | 3 +++ 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test-docker-build.yaml b/.github/workflows/test-docker-build.yaml index fc3d382f7e..89d69064ab 100644 --- a/.github/workflows/test-docker-build.yaml +++ b/.github/workflows/test-docker-build.yaml @@ -25,26 +25,22 @@ on: jobs: # TODO: this is just a quick test to check the arm64 docker images - # this should instead be done using chartpress # Based on # https://github.com/docker/build-push-action/blob/v2.3.0/docs/advanced/local-registry.md # https://github.com/docker/build-push-action/blob/v2.3.0/docs/advanced/multi-platform.md build_images: runs-on: ubuntu-20.04 - strategy: - # Keep running even if one variation of the job fails - fail-fast: false - matrix: - image: - - hub - - image-awaiter - - network-tools - - secret-sync - # - singleuser-sample - steps: - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: "3.8" + + # https://github.com/jupyterhub/chartpress/pull/124 + - name: Install chartpress buildx-platforms-dev version + run: pip install --no-cache-dir chartpress@git+https://github.com/manics/chartpress.git@buildx-platforms-skip + - name: Set up QEMU uses: docker/setup-qemu-action@v1 @@ -60,9 +56,8 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push to local registry - uses: docker/build-push-action@v2 - with: - context: images/${{ matrix.image }} - platforms: linux/amd64,linux/arm64 - push: true - tags: ghcr.io/${{ github.repository_owner }}/z2jh-${{ matrix.image }}:dev + run: >- + chartpress --push --tag dev + --builder docker-buildx + --platform linux/amd64 --platform linux/arm64 + --image-prefix ghcr.io/${{ github.repository_owner }}/z2jh- diff --git a/chartpress.yaml b/chartpress.yaml index 7b814ad989..df93fdbf6d 100644 --- a/chartpress.yaml +++ b/chartpress.yaml @@ -46,5 +46,8 @@ charts: valuesPath: prePuller.hook.image # singleuser-sample, a primitive user container to start with. + # Image is based on https://github.com/jupyter/docker-stacks/ which is amd64 only singleuser-sample: valuesPath: singleuser.image + skipPlatforms: + - linux/arm64 From f05eef000adcaffb2080da5af01fbe985a75150a Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 12 Apr 2021 10:16:27 +0200 Subject: [PATCH 08/11] Bump to chartpress 1.1.0 for multiarch build features --- .github/workflows/test-docker-build.yaml | 5 ++--- dev-requirements.txt | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-docker-build.yaml b/.github/workflows/test-docker-build.yaml index 89d69064ab..cec19354f5 100644 --- a/.github/workflows/test-docker-build.yaml +++ b/.github/workflows/test-docker-build.yaml @@ -37,9 +37,8 @@ jobs: with: python-version: "3.8" - # https://github.com/jupyterhub/chartpress/pull/124 - - name: Install chartpress buildx-platforms-dev version - run: pip install --no-cache-dir chartpress@git+https://github.com/manics/chartpress.git@buildx-platforms-skip + - name: Install chartpress + run: pip install chartpress - name: Set up QEMU uses: docker/setup-qemu-action@v1 diff --git a/dev-requirements.txt b/dev-requirements.txt index ad2cf86d0b..a999cc71cd 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -5,7 +5,7 @@ # # ref: https://github.com/jupyterhub/chartpress # -chartpress>=1.0.4 +chartpress>=1.1.0 # pytest run tests that require requests and pyyaml pytest>=3.7.1 From 1eac93ce252d818a37bcde4c5a42f40f1ffb32ea Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 12 Apr 2021 11:39:27 +0200 Subject: [PATCH 09/11] ci: make chart versions semver2 compliant for helm3 --- .github/workflows/test-docker-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-docker-build.yaml b/.github/workflows/test-docker-build.yaml index cec19354f5..ff03fbfa8e 100644 --- a/.github/workflows/test-docker-build.yaml +++ b/.github/workflows/test-docker-build.yaml @@ -56,7 +56,7 @@ jobs: - name: Build and push to local registry run: >- - chartpress --push --tag dev + chartpress --push --tag 0.0.1-dev --builder docker-buildx --platform linux/amd64 --platform linux/arm64 --image-prefix ghcr.io/${{ github.repository_owner }}/z2jh- From 6508e1af9548be11b5e86c39a5328dad08beeee3 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Mon, 12 Apr 2021 21:45:12 +0100 Subject: [PATCH 10/11] publish workflow: build amd64 and arm64 --- ci/publish | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ci/publish b/ci/publish index 996395d64f..a2b7482090 100755 --- a/ci/publish +++ b/ci/publish @@ -6,6 +6,11 @@ # Exit on errors, assert env vars, log commands set -eux +PUBLISH_ARGS="--push --publish-chart \ + --builder docker-buildx \ + --platform linux/amd64 --platform linux/arm64 \ + " + # chartpress use git to push to our Helm chart repository, which is the gh-pages # branch of jupyterhub/helm-chart. We have installed a private SSH key within # the ~/.ssh folder with permissions to push to jupyterhub/helm-chart. @@ -26,11 +31,11 @@ if [[ $GITHUB_REF != refs/tags/* ]]; then PR_OR_HASH=$(git log -1 --pretty=%h-%B | head -n1 | sed 's/^.*\(#[0-9]*\).*/\1/' | sed 's/^\([0-9a-f]*\)-.*/@\1/') LATEST_COMMIT_TITLE=$(git log -1 --pretty=%B | head -n1) EXTRA_MESSAGE="${GITHUB_REPOSITORY}${PR_OR_HASH} ${LATEST_COMMIT_TITLE}" - chartpress --push --publish-chart --extra-message "${EXTRA_MESSAGE}" + chartpress $PUBLISH_ARGS --extra-message "${EXTRA_MESSAGE}" else # Setting a tag explicitly enforces a rebuild if this tag had already been # built and we wanted to override it. - chartpress --push --publish-chart --tag "${GITHUB_REF:10}" + chartpress $PUBLISH_ARGS --tag "${GITHUB_REF:10}" fi # Let us log the changes chartpress did, it should include replacements for From 4d9c7b77dfbf8b95b8884618de6a8b184a284c97 Mon Sep 17 00:00:00 2001 From: Simon Li Date: Mon, 12 Apr 2021 21:46:04 +0100 Subject: [PATCH 11/11] test-docker-build.yaml workflow: Remove push --- .github/workflows/test-docker-build.yaml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test-docker-build.yaml b/.github/workflows/test-docker-build.yaml index ff03fbfa8e..070abba17d 100644 --- a/.github/workflows/test-docker-build.yaml +++ b/.github/workflows/test-docker-build.yaml @@ -46,17 +46,8 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - # https://github.com/docker/login-action/tree/v1.8.0#github-container-registry - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push to local registry + - name: Build a multiple architecture Docker image run: >- - chartpress --push --tag 0.0.1-dev + chartpress --builder docker-buildx --platform linux/amd64 --platform linux/arm64 - --image-prefix ghcr.io/${{ github.repository_owner }}/z2jh-