ci: move all x86_64 runners to the host-x86_64 directory

We need to add runners designed for an aarch64 host system, and it'd be
nice to return an error message if someone tries to run an image
designed for an host architecture in another one.

To start the work on this, this commit moves all the existing builders
in the host-x86_64 directory, and changes the run.sh script to look up
the image in the correct directory based on the host architecture.

Author: pietroalbini

.github/workflows/ci.yml

@@ -428,7 +428,7 @@ jobs:
             os: windows-latest-xl
           - name: x86_64-msvc-tools
             env:
-              SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
+              SCRIPT: src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json"
             os: windows-latest-xl
           - name: i686-mingw-1

src/ci/azure-pipelines/auto.yml

@@ -151,7 +151,7 @@ jobs:
         NO_LLVM_ASSERTIONS: 1
       # MSVC tools tests
       x86_64-msvc-tools:
-        SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py
+        SCRIPT: src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py
         INITIAL_RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json
 
       # 32/64-bit MinGW builds.

src/ci/docker/README.md

@@ -20,7 +20,7 @@ RUN dpkg --add-architecture i386 && \
 
 COPY scripts/android-sdk.sh /scripts/
 COPY scripts/android-sdk-manager.py /scripts/
-COPY arm-android/android-sdk.lock /android/sdk/android-sdk.lock
+COPY host-x86_64/arm-android/android-sdk.lock /android/sdk/android-sdk.lock
 RUN /scripts/android-sdk.sh
 
 ENV PATH=$PATH:/android/sdk/emulator

src/ci/docker/arm-android/Dockerfile src/ci/docker/host-x86_64/arm-android/Dockerfile

@@ -31,7 +31,7 @@ WORKDIR /build
 # The `vexpress_config` config file was a previously generated config file for
 # the kernel. This file was generated by running `make vexpress_defconfig`
 # followed by `make menuconfig` and then enabling the IPv6 protocol page.
-COPY armhf-gnu/vexpress_config /build/.config
+COPY host-x86_64/armhf-gnu/vexpress_config /build/.config
 RUN curl https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.4.42.tar.xz | \
       tar xJf - && \
       cd /build/linux-4.4.42 && \

src/ci/docker/arm-android/android-sdk.lock src/ci/docker/host-x86_64/arm-android/android-sdk.lock

@@ -31,7 +31,7 @@ WORKDIR /build
 # The `config` config file was a previously generated config file for
 # the kernel. This file was generated by running `make defconfig`
 # followed by `make menuconfig` and then enabling the IPv6 protocol page.
-COPY aarch64-gnu/config /build/.config
+COPY host-x86_64/aarch64-gnu/config /build/.config
 RUN curl https://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.4.42.tar.xz | \
       tar xJf - && \
       cd /build/linux-4.4.42 && \

src/ci/docker/armhf-gnu/Dockerfile src/ci/docker/host-x86_64/armhf-gnu/Dockerfile

@@ -18,8 +18,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   pkg-config
 
 
-COPY dist-x86_64-dragonfly/build-toolchain.sh /tmp/
-COPY dist-x86_64-dragonfly/patch-toolchain /tmp/
+COPY host-x86_64/dist-x86_64-dragonfly/build-toolchain.sh /tmp/
+COPY host-x86_64/dist-x86_64-dragonfly/patch-toolchain /tmp/
 RUN /tmp/build-toolchain.sh /tmp/patch-toolchain
 
 COPY scripts/sccache.sh /scripts/

src/ci/docker/armhf-gnu/vexpress_config src/ci/docker/host-x86_64/armhf-gnu/vexpress_config

@@ -25,15 +25,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   xz-utils \
   zlib1g-dev
 
-COPY dist-x86_64-haiku/llvm-config.sh /bin/llvm-config-haiku
+COPY host-x86_64/dist-x86_64-haiku/llvm-config.sh /bin/llvm-config-haiku
 
 ENV ARCH=x86_64
 
 WORKDIR /tmp
-COPY dist-x86_64-haiku/build-toolchain.sh /tmp/
+COPY host-x86_64/dist-x86_64-haiku/build-toolchain.sh /tmp/
 RUN /tmp/build-toolchain.sh $ARCH
 
-COPY dist-x86_64-haiku/fetch-packages.sh /tmp/
+COPY host-x86_64/dist-x86_64-haiku/fetch-packages.sh /tmp/
 RUN /tmp/fetch-packages.sh
 
 COPY scripts/sccache.sh /scripts/

src/ci/docker/disabled/aarch64-gnu/Dockerfile src/ci/docker/host-x86_64/disabled/aarch64-gnu/Dockerfile

@@ -7,7 +7,7 @@ COPY scripts/crosstool-ng.sh /scripts/
 RUN sh /scripts/crosstool-ng.sh
 
 WORKDIR /tmp
-COPY dist-various-1/install-x86_64-redox.sh /scripts/
+COPY host-x86_64/dist-various-1/install-x86_64-redox.sh /scripts/
 RUN sh /scripts/install-x86_64-redox.sh
 
 COPY scripts/sccache.sh /scripts/

src/ci/docker/disabled/aarch64-gnu/config src/ci/docker/host-x86_64/disabled/aarch64-gnu/config

@@ -31,7 +31,7 @@ ENV CROSS_COMPILE=riscv64-linux-gnu-
 WORKDIR /build
 
 # From https://github.com/michaeljclark/busybear-linux/blob/master/conf/linux.config
-COPY riscv64gc-linux/linux.config /build
+COPY host-x86_64/riscv64gc-linux/linux.config /build
 
 # Compile the kernel that we're going to be emulating with. This is
 # basically just done to be compatible with the QEMU target that we're going
@@ -48,7 +48,7 @@ RUN rm -rf linux-5.6.16
 # binary which we will boot into. Only trick here is configuring busybox to
 # build static binaries.
 RUN curl https://busybox.net/downloads/busybox-1.31.1.tar.bz2 | tar xjf -
-COPY riscv64gc-linux/0001-Remove-stime-function-calls.patch /build/busybox-1.31.1/
+COPY host-x86_64/riscv64gc-linux/0001-Remove-stime-function-calls.patch /build/busybox-1.31.1/
 RUN cd /build/busybox-1.31.1 && \
     patch -p1 -i 0001-Remove-stime-function-calls.patch && \
     make defconfig && \

src/ci/docker/disabled/asmjs/Dockerfile src/ci/docker/host-x86_64/disabled/asmjs/Dockerfile

@@ -16,7 +16,7 @@ RUN sh /scripts/rustbuild-setup.sh
 USER rustbuild
 WORKDIR /tmp
 
-COPY dist-aarch64-linux/aarch64-linux-gnu.config dist-aarch64-linux/build-toolchains.sh /tmp/
+COPY host-x86_64/dist-aarch64-linux/aarch64-linux-gnu.config host-x86_64/dist-aarch64-linux/build-toolchains.sh /tmp/
 RUN ./build-toolchains.sh
 
 USER root

src/ci/docker/disabled/dist-aarch64-android/Dockerfile src/ci/docker/host-x86_64/disabled/dist-aarch64-android/Dockerfile

@@ -11,7 +11,7 @@ RUN sh /scripts/rustbuild-setup.sh
 USER rustbuild
 WORKDIR /tmp
 
-COPY dist-arm-linux/arm-linux-gnueabi.config dist-arm-linux/build-toolchains.sh /tmp/
+COPY host-x86_64/dist-arm-linux/arm-linux-gnueabi.config host-x86_64/dist-arm-linux/build-toolchains.sh /tmp/
 RUN ./build-toolchains.sh
 
 USER root

src/ci/docker/disabled/dist-armv7-android/Dockerfile src/ci/docker/host-x86_64/disabled/dist-armv7-android/Dockerfile

@@ -11,7 +11,7 @@ RUN sh /scripts/rustbuild-setup.sh
 USER rustbuild
 WORKDIR /tmp
 
-COPY dist-armhf-linux/arm-linux-gnueabihf.config dist-armhf-linux/build-toolchains.sh /tmp/
+COPY host-x86_64/dist-armhf-linux/arm-linux-gnueabihf.config host-x86_64/dist-armhf-linux/build-toolchains.sh /tmp/
 RUN ./build-toolchains.sh
 
 USER root

src/ci/docker/disabled/dist-i686-android/Dockerfile src/ci/docker/host-x86_64/disabled/dist-i686-android/Dockerfile

@@ -11,7 +11,7 @@ RUN sh /scripts/rustbuild-setup.sh
 USER rustbuild
 WORKDIR /tmp
 
-COPY dist-armv7-linux/build-toolchains.sh dist-armv7-linux/armv7-linux-gnueabihf.config /tmp/
+COPY host-x86_64/dist-armv7-linux/build-toolchains.sh host-x86_64/dist-armv7-linux/armv7-linux-gnueabihf.config /tmp/
 RUN ./build-toolchains.sh
 
 USER root

src/ci/docker/disabled/dist-powerpcspe-linux/Dockerfile src/ci/docker/host-x86_64/disabled/dist-powerpcspe-linux/Dockerfile

@@ -29,13 +29,13 @@ ENV PATH=/rustroot/bin:$PATH
 ENV LD_LIBRARY_PATH=/rustroot/lib64:/rustroot/lib
 ENV PKG_CONFIG_PATH=/rustroot/lib/pkgconfig
 WORKDIR /tmp
-COPY dist-x86_64-linux/shared.sh /tmp/
+COPY host-x86_64/dist-x86_64-linux/shared.sh /tmp/
 
 # We need a build of openssl which supports SNI to download artifacts from
 # static.rust-lang.org. This'll be used to link into libcurl below (and used
 # later as well), so build a copy of OpenSSL with dynamic libraries into our
 # generic root.
-COPY dist-x86_64-linux/build-openssl.sh /tmp/
+COPY host-x86_64/dist-x86_64-linux/build-openssl.sh /tmp/
 RUN ./build-openssl.sh
 
 # The `curl` binary on CentOS doesn't support SNI which is needed for fetching
@@ -44,48 +44,48 @@ RUN ./build-openssl.sh
 #
 # Note that we also disable a bunch of optional features of curl that we don't
 # really need.
-COPY dist-x86_64-linux/build-curl.sh /tmp/
+COPY host-x86_64/dist-x86_64-linux/build-curl.sh /tmp/
 RUN ./build-curl.sh
 
 # binutils < 2.22 has a bug where the 32-bit executables it generates
 # immediately segfault in Rust, so we need to install our own binutils.
 #
 # See https://github.com/rust-lang/rust/issues/20440 for more info
-COPY dist-x86_64-linux/build-binutils.sh /tmp/
+COPY host-x86_64/dist-x86_64-linux/build-binutils.sh /tmp/
 RUN ./build-binutils.sh
 
 # libssh2 (a dependency of Cargo) requires cmake 2.8.11 or higher but CentOS
 # only has 2.6.4, so build our own
-COPY dist-x86_64-linux/build-cmake.sh /tmp/
+COPY host-x86_64/dist-x86_64-linux/build-cmake.sh /tmp/
 RUN ./build-cmake.sh
 
 # Need a newer version of gcc than centos has to compile LLVM nowadays
-COPY dist-x86_64-linux/build-gcc.sh /tmp/
+COPY host-x86_64/dist-x86_64-linux/build-gcc.sh /tmp/
 RUN ./build-gcc.sh
 
 # CentOS 5.5 has Python 2.4 by default, but LLVM needs 2.7+
-COPY dist-x86_64-linux/build-python.sh /tmp/
+COPY host-x86_64/dist-x86_64-linux/build-python.sh /tmp/
 RUN ./build-python.sh
 
 # Now build LLVM+Clang 7, afterwards configuring further compilations to use the
 # clang/clang++ compilers.
-COPY dist-x86_64-linux/build-clang.sh dist-x86_64-linux/llvm-project-centos.patch /tmp/
+COPY host-x86_64/dist-x86_64-linux/build-clang.sh host-x86_64/dist-x86_64-linux/llvm-project-centos.patch /tmp/
 RUN ./build-clang.sh
 ENV CC=clang CXX=clang++
 
 # Apparently CentOS 5.5 desn't have `git` in yum, but we're gonna need it for
 # cloning, so download and build it here.
-COPY dist-x86_64-linux/build-git.sh /tmp/
+COPY host-x86_64/dist-x86_64-linux/build-git.sh /tmp/
 RUN ./build-git.sh
 
 # for sanitizers, we need kernel headers files newer than the ones CentOS ships
 # with so we install newer ones here
-COPY dist-x86_64-linux/build-headers.sh /tmp/
+COPY host-x86_64/dist-x86_64-linux/build-headers.sh /tmp/
 RUN ./build-headers.sh
 
 # OpenSSL requires a more recent version of perl
 # with so we install newer ones here
-COPY dist-x86_64-linux/build-perl.sh /tmp/
+COPY host-x86_64/dist-x86_64-linux/build-perl.sh /tmp/
 RUN ./build-perl.sh
 
 COPY scripts/sccache.sh /scripts/

src/ci/docker/disabled/dist-sparc64-linux/Dockerfile src/ci/docker/host-x86_64/disabled/dist-sparc64-linux/Dockerfile

@@ -16,8 +16,8 @@ RUN sh /scripts/rustbuild-setup.sh
 USER rustbuild
 WORKDIR /tmp
 
-COPY dist-powerpc-linux/patches/ /tmp/patches/
-COPY dist-powerpc-linux/powerpc-linux-gnu.config dist-powerpc-linux/build-powerpc-toolchain.sh /tmp/
+COPY host-x86_64/dist-powerpc-linux/patches/ /tmp/patches/
+COPY host-x86_64/dist-powerpc-linux/powerpc-linux-gnu.config host-x86_64/dist-powerpc-linux/build-powerpc-toolchain.sh /tmp/
 RUN ./build-powerpc-toolchain.sh
 
 USER root

src/ci/docker/disabled/dist-x86_64-android/Dockerfile src/ci/docker/host-x86_64/disabled/dist-x86_64-android/Dockerfile

@@ -17,8 +17,8 @@ RUN sh /scripts/rustbuild-setup.sh
 USER rustbuild
 WORKDIR /tmp
 
-COPY dist-powerpc64-linux/patches/ /tmp/patches/
-COPY dist-powerpc64-linux/shared.sh dist-powerpc64-linux/powerpc64-linux-gnu.config dist-powerpc64-linux/build-powerpc64-toolchain.sh /tmp/
+COPY host-x86_64/dist-powerpc64-linux/patches/ /tmp/patches/
+COPY host-x86_64/dist-powerpc64-linux/shared.sh host-x86_64/dist-powerpc64-linux/powerpc64-linux-gnu.config host-x86_64/dist-powerpc64-linux/build-powerpc64-toolchain.sh /tmp/
 RUN ./build-powerpc64-toolchain.sh
 
 USER root

src/ci/docker/disabled/dist-x86_64-dragonfly/Dockerfile src/ci/docker/host-x86_64/disabled/dist-x86_64-dragonfly/Dockerfile

@@ -19,7 +19,7 @@ WORKDIR /tmp
 USER root
 
 RUN apt-get install -y --no-install-recommends rpm2cpio cpio
-COPY dist-powerpc64le-linux/shared.sh dist-powerpc64le-linux/build-powerpc64le-toolchain.sh /tmp/
+COPY host-x86_64/dist-powerpc64le-linux/shared.sh host-x86_64/dist-powerpc64le-linux/build-powerpc64le-toolchain.sh /tmp/
 RUN ./build-powerpc64le-toolchain.sh
 
 COPY scripts/sccache.sh /scripts/

src/ci/docker/disabled/dist-x86_64-dragonfly/build-toolchain.sh src/ci/docker/host-x86_64/disabled/dist-x86_64-dragonfly/build-toolchain.sh

@@ -16,8 +16,8 @@ RUN sh /scripts/rustbuild-setup.sh
 USER rustbuild
 WORKDIR /tmp
 
-COPY dist-s390x-linux/patches/ /tmp/patches/
-COPY dist-s390x-linux/s390x-linux-gnu.config dist-s390x-linux/build-s390x-toolchain.sh /tmp/
+COPY host-x86_64/dist-s390x-linux/patches/ /tmp/patches/
+COPY host-x86_64/dist-s390x-linux/s390x-linux-gnu.config host-x86_64/dist-s390x-linux/build-s390x-toolchain.sh /tmp/
 RUN ./build-s390x-toolchain.sh
 
 USER root

src/ci/docker/disabled/dist-x86_64-dragonfly/patch-toolchain src/ci/docker/host-x86_64/disabled/dist-x86_64-dragonfly/patch-toolchain

@@ -47,7 +47,7 @@ RUN add-apt-repository ppa:team-gcc-arm-embedded/ppa && \
     apt-get update && \
     apt-get install -y --no-install-recommends gcc-arm-embedded
 
-COPY scripts/rustbuild-setup.sh dist-various-1/build-riscv-toolchain.sh dist-various-1/riscv64-unknown-linux-gnu.config dist-various-1/crosstool-ng.sh /build/
+COPY scripts/rustbuild-setup.sh host-x86_64/dist-various-1/build-riscv-toolchain.sh host-x86_64/dist-various-1/riscv64-unknown-linux-gnu.config host-x86_64/dist-various-1/crosstool-ng.sh /build/
 RUN ./crosstool-ng.sh
 
 # Crosstool-ng will refuse to build as root
@@ -59,19 +59,19 @@ RUN ./build-riscv-toolchain.sh
 USER root
 ENV PATH=/x-tools/riscv64-unknown-linux-gnu/bin:$PATH
 
-COPY dist-various-1/build-rumprun.sh /build
+COPY host-x86_64/dist-various-1/build-rumprun.sh /build
 RUN ./build-rumprun.sh
 
-COPY dist-various-1/install-x86_64-redox.sh /build
+COPY host-x86_64/dist-various-1/install-x86_64-redox.sh /build
 RUN ./install-x86_64-redox.sh
 
-COPY dist-various-1/install-mips-musl.sh /build
+COPY host-x86_64/dist-various-1/install-mips-musl.sh /build
 RUN ./install-mips-musl.sh
 
-COPY dist-various-1/install-mipsel-musl.sh /build
+COPY host-x86_64/dist-various-1/install-mipsel-musl.sh /build
 RUN ./install-mipsel-musl.sh
 
-COPY dist-various-1/install-aarch64-none-elf.sh /build
+COPY host-x86_64/dist-various-1/install-aarch64-none-elf.sh /build
 RUN ./install-aarch64-none-elf.sh
 
 # Suppress some warnings in the openwrt toolchains we downloaded

src/ci/docker/disabled/dist-x86_64-haiku/Dockerfile src/ci/docker/host-x86_64/disabled/dist-x86_64-haiku/Dockerfile

@@ -60,20 +60,20 @@ RUN env \
     rm -rf /build/*
 
 WORKDIR /tmp
-COPY dist-various-2/shared.sh /tmp/
-COPY dist-various-2/build-cloudabi-toolchain.sh /tmp/
+COPY host-x86_64/dist-various-2/shared.sh /tmp/
+COPY host-x86_64/dist-various-2/build-cloudabi-toolchain.sh /tmp/
 RUN /tmp/build-cloudabi-toolchain.sh x86_64-unknown-cloudabi
-COPY dist-various-2/build-fuchsia-toolchain.sh /tmp/
+COPY host-x86_64/dist-various-2/build-fuchsia-toolchain.sh /tmp/
 RUN /tmp/build-fuchsia-toolchain.sh
-COPY dist-various-2/build-solaris-toolchain.sh /tmp/
+COPY host-x86_64/dist-various-2/build-solaris-toolchain.sh /tmp/
 RUN /tmp/build-solaris-toolchain.sh x86_64  amd64   solaris-i386
 RUN /tmp/build-solaris-toolchain.sh sparcv9 sparcv9 solaris-sparc
-COPY dist-various-2/build-x86_64-fortanix-unknown-sgx-toolchain.sh /tmp/
-COPY dist-various-2/x86_64-fortanix-unknown-sgx-clang-wrap.sh /usr/bin/x86_64-fortanix-unknown-sgx-clang-11
+COPY host-x86_64/dist-various-2/build-x86_64-fortanix-unknown-sgx-toolchain.sh /tmp/
+COPY host-x86_64/dist-various-2/x86_64-fortanix-unknown-sgx-clang-wrap.sh /usr/bin/x86_64-fortanix-unknown-sgx-clang-11
 RUN ln -s /usr/bin/x86_64-fortanix-unknown-sgx-clang-11 /usr/bin/x86_64-fortanix-unknown-sgx-clang++-11
 RUN /tmp/build-x86_64-fortanix-unknown-sgx-toolchain.sh
 
-COPY dist-various-2/build-wasi-toolchain.sh /tmp/
+COPY host-x86_64/dist-various-2/build-wasi-toolchain.sh /tmp/
 RUN /tmp/build-wasi-toolchain.sh
 
 COPY scripts/sccache.sh /scripts/