unix: install pip and setuptools from wheels

This commit refactors the install of pip and setuptools.

Previously, we installed both of these by invoking `setup.py` from the
host Python. This worked, but it was somewhat old school. And in the
case of cross-compiling, it installed the packages to the host Python,
not the target Python.

After this commit, we download wheels for both pip and setuptools and
the install invokes the host Python but runs the pip wheel to invoke
pip to install the packages. This is conceptually similar to how the
`ensurepip` module works.

The end result is that pip is used to install itself and setuptools
into the appropriate output directory. This works with both native
and cross builds.

Aside from minor changes to the directory layout, the end result is
the same as far as I can tell. But since cross-compiles now install
pip and setuptools correctly, this fixes #84.

Because pip now bootstraps self and this doesn't work on musl builds
without a patch, we had to move the patching of pip to before it
is invoked. We moved the patching of setuptools as well, because it
is related.

Author: indygreg

cpython-unix/build-cpython.sh

@@ -20,8 +20,74 @@ export LLVM_PROFDATA=${TOOLS_PATH}/${TOOLCHAIN}/bin/llvm-profdata
 find ${TOOLS_PATH}/deps -name '*.so*' -exec rm {} \;
 
 tar -xf Python-${PYTHON_VERSION}.tar.xz
-tar -xf setuptools-${SETUPTOOLS_VERSION}.tar.gz
-tar -xf pip-${PIP_VERSION}.tar.gz
+
+PIP_WHEEL="${ROOT}/pip-${PIP_VERSION}-py3-none-any.whl"
+SETUPTOOLS_WHEEL="${ROOT}/setuptools-${SETUPTOOLS_VERSION}-py3-none-any.whl"
+
+chmod 644 "${PIP_WHEEL}"
+chmod 644 "${SETUPTOOLS_WHEEL}"
+
+# pip and setuptools don't properly handle the case where the current executable
+# isn't dynamic. This is tracked by https://github.com/pypa/pip/issues/6543.
+# We need to patch both.
+#
+# Ideally we'd do this later in the build. However, since we use the pip
+# wheel to bootstrap itself, we need to patch the wheel before it is used.
+#
+# Wheels are zip files. So we simply unzip, patch, and rezip.
+mkdir pip-tmp
+pushd pip-tmp
+unzip "${PIP_WHEEL}"
+
+patch -p1 <<EOF
+diff --git a/pip/_internal/utils/glibc.py b/pip/_internal/utils/glibc.py
+index 819979d80..4ae91e364 100644
+--- a/pip/_internal/utils/glibc.py
++++ b/pip/_internal/utils/glibc.py
+@@ -47,7 +47,10 @@ def glibc_version_string_ctypes():
+     # manpage says, "If filename is NULL, then the returned handle is for the
+     # main program". This way we can let the linker do the work to figure out
+     # which libc our process is actually using.
+-    process_namespace = ctypes.CDLL(None)
++    try:
++        process_namespace = ctypes.CDLL(None)
++    except OSError:
++        return None
+     try:
+         gnu_get_libc_version = process_namespace.gnu_get_libc_version
+     except AttributeError:
+EOF
+
+zip -r "${PIP_WHEEL}" *
+popd
+rm -rf pip-tmp
+
+mkdir setuptools-tmp
+pushd setuptools-tmp
+unzip "${SETUPTOOLS_WHEEL}"
+
+patch -p1 <<EOF
+diff --git a/setuptools/_vendor/packaging/tags.py b/setuptools/_vendor/packaging/tags.py
+index 9064910b..c541e648 100644
+--- a/setuptools/_vendor/packaging/tags.py
++++ b/setuptools/_vendor/packaging/tags.py
+@@ -475,7 +475,10 @@ def _glibc_version_string_ctypes():
+     # which libc our process is actually using.
+     #
+     # Note: typeshed is wrong here so we are ignoring this line.
+-    process_namespace = ctypes.CDLL(None)  # type: ignore
++    try:
++        process_namespace = ctypes.CDLL(None)  # type: ignore
++    except OSError:
++        return None
+     try:
+         gnu_get_libc_version = process_namespace.gnu_get_libc_version
+     except AttributeError:
+EOF
+
+zip -r "${SETUPTOOLS_WHEEL}" *
+popd
+rm -rf setuptools-tmp
 
 # If we are cross-compiling, we need to build a host Python to use during
 # the build.
@@ -50,7 +116,9 @@ if [ "${BUILD_TRIPLE}" != "${TARGET_TRIPLE}" ]; then
       ;;
   esac
 
-  CC="${HOST_CC}" CFLAGS="${EXTRA_HOST_CFLAGS}" CPPFLAGS="${EXTRA_HOST_CFLAGS}" LDFLAGS="${EXTRA_HOST_LDFLAGS}" ./configure --prefix "${TOOLS_PATH}/pyhost"
+  CC="${HOST_CC}" CFLAGS="${EXTRA_HOST_CFLAGS}" CPPFLAGS="${EXTRA_HOST_CFLAGS}" LDFLAGS="${EXTRA_HOST_LDFLAGS}" ./configure \
+    --prefix "${TOOLS_PATH}/pyhost" \
+    --without-ensurepip
 
   # When building on macOS 10.15 (and possibly earlier) using the 11.0
   # SDK, the _ctypes extension fails to import due to a missing symbol on
@@ -741,63 +809,17 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then
     fi
 fi
 
-# Install pip so we can patch it to work with non-dynamic executables
-# and work around https://github.com/pypa/pip/issues/6543. But pip's bundled
-# setuptools has the same bug! So we need to install a patched version.
-pushd ${ROOT}/setuptools-${SETUPTOOLS_VERSION}
-patch -p1 <<EOF
-diff --git a/setuptools/_vendor/packaging/tags.py b/setuptools/_vendor/packaging/tags.py
-index 9064910b..c541e648 100644
---- a/setuptools/_vendor/packaging/tags.py
-+++ b/setuptools/_vendor/packaging/tags.py
-@@ -475,7 +475,10 @@ def _glibc_version_string_ctypes():
-     # which libc our process is actually using.
-     #
-     # Note: typeshed is wrong here so we are ignoring this line.
--    process_namespace = ctypes.CDLL(None)  # type: ignore
-+    try:
-+        process_namespace = ctypes.CDLL(None)  # type: ignore
-+    except OSError:
-+        return None
-     try:
-         gnu_get_libc_version = process_namespace.gnu_get_libc_version
-     except AttributeError:
-EOF
-
-${BUILD_PYTHON} setup.py install
-popd
-
-pushd ${ROOT}/pip-${PIP_VERSION}
-
-# pip 21 shipped DOS line endings. https://github.com/pypa/pip/issues/9638.
-# Let's fix that.
-if [ "${PYBUILD_PLATFORM}" = "macos" ]; then
-    find . -name '*.py' -exec perl -i -pe 's/\r\n$/\n/g' {} \;
-else
-    find . -name '*.py' -exec sed -i 's/\r$//g' {} \;
-fi
-
-patch -p1 <<EOF
-diff --git a/src/pip/_internal/utils/glibc.py b/src/pip/_internal/utils/glibc.py
-index 819979d80..4ae91e364 100644
---- a/src/pip/_internal/utils/glibc.py
-+++ b/src/pip/_internal/utils/glibc.py
-@@ -47,7 +47,10 @@ def glibc_version_string_ctypes():
-     # manpage says, "If filename is NULL, then the returned handle is for the
-     # main program". This way we can let the linker do the work to figure out
-     # which libc our process is actually using.
--    process_namespace = ctypes.CDLL(None)
-+    try:
-+        process_namespace = ctypes.CDLL(None)
-+    except OSError:
-+        return None
-     try:
-         gnu_get_libc_version = process_namespace.gnu_get_libc_version
-     except AttributeError:
-EOF
+# Install setuptools and pip as they are common tools that should be in any
+# Python distribution.
+#
+# We disabled ensurepip because we insist on providing our own pip and don't
+# want the final product to possibly be contaminated by another version.
+#
+# It is possible for the Python interpreter to run wheels directly. So we
+# simply use our pip to install self. Kinda crazy, but it works!
 
-${BUILD_PYTHON} setup.py install
-popd
+${BUILD_PYTHON} "${PIP_WHEEL}/pip" install --prefix="${ROOT}/out/python/install" --no-cache-dir --no-index "${PIP_WHEEL}"
+${BUILD_PYTHON} "${PIP_WHEEL}/pip" install --prefix="${ROOT}/out/python/install" --no-cache-dir --no-index "${SETUPTOOLS_WHEEL}"
 
 # Emit metadata to be used in PYTHON.json.
 cat > ${ROOT}/generate_metadata.py << EOF

cpython-unix/build.Dockerfile

@@ -23,4 +23,5 @@ RUN apt-get install \
     tar \
     xz-utils \
     unzip \
+    zip \
     zlib1g-dev

cpython-unix/build.cross.Dockerfile

@@ -43,6 +43,7 @@ RUN apt-get install \
     tar \
     xz-utils \
     unzip \
+    zip \
     zlib1g-dev
 
 # Cross-building.

pythonbuild/downloads.py

@@ -286,9 +286,9 @@
         "version": "0.12",
     },
     "pip": {
-        "url": "https://files.pythonhosted.org/packages/52/e1/06c018197d8151383f66ebf6979d951995cf495629fc54149491f5d157d0/pip-21.2.4.tar.gz",
-        "size": 1564487,
-        "sha256": "0eb8a1516c3d138ae8689c0c1a60fde7143310832f9dc77e11d8a4bc62de193b",
+        "url": "https://files.pythonhosted.org/packages/ca/31/b88ef447d595963c01060998cb329251648acf4a067721b0452c45527eb8/pip-21.2.4-py3-none-any.whl",
+        "size": 1555100,
+        "sha256": "fa9ebb85d3fd607617c0c44aca302b1b45d87f9c2a1649b46c26167ca4296323",
         "version": "21.2.4",
     },
     "readline": {
@@ -301,9 +301,9 @@
         "license_file": "LICENSE.readline.txt",
     },
     "setuptools": {
-        "url": "https://files.pythonhosted.org/packages/db/e2/c0ced9ccffb61432305665c22842ea120c0f649eec47ecf2a45c596707c4/setuptools-57.4.0.tar.gz",
-        "size": 2141309,
-        "sha256": "6bac238ffdf24e8806c61440e755192470352850f3419a52f26ffe0a1a64f465",
+        "url": "https://files.pythonhosted.org/packages/bd/25/5bdf7f1adeebd4e3fa76b2e2f045ae53ee208e40a4231ad0f0c3007e4353/setuptools-57.4.0-py3-none-any.whl",
+        "size": 819017,
+        "sha256": "a49230977aa6cfb9d933614d2f7b79036e9945c4cdd7583163f4e920b83418d6",
         "version": "57.4.0",
     },
     "sqlite": {