Trac #29950: Build sagelib from build/pkgs/sagelib/src, fix setup.py sdist, add spkg-src and tox.ini

As a follow-up to #29411, we set up a separate directory
`build/pkgs/sagelib/src` with symlinks into `SAGE_ROOT/src` so that
sagelib's `setup.py` does not have access to the whole tree
`SAGE_ROOT/src`.

We fix `setup.py sdist`, which can be tested using `spkg-src` (which
builds a pip-installable tarball in `upstream/`) and `tox`:
{{{
./sage -sh -c '(cd build/pkgs/sagelib/src && tox -v -v)'
./sage -sh -c 'build/pkgs/sagelib/src/.tox/python/bin/sage-ipython'
}}}

The call of `./sage -sh` is necessary so that `SAGE_LOCAL` is set.
Moreover, for scripts that use `sage-env`, also `SAGE_ENV_SOURCED` is
crucial for the correct operation; otherwise `sage-env` sets things
based on `SAGE_LOCAL` again.

URL: https://trac.sagemath.org/29950
Reported by: mkoeppe
Ticket author(s): Matthias Koeppe
Reviewer(s): Dima Pasechnik

Author: Release Manager

.gitignore

@@ -101,6 +101,12 @@ gitlab-build-docker.log
 /src/bin/sage-env-config
 /build/bin/sage-build-env-config
 
+/build/pkgs/*/src/build
+/build/pkgs/*/src/dist
+/build/pkgs/*/src/MANIFEST
+/build/pkgs/*/src/*.egg-info
+/build/pkgs/*/src/.tox
+
 #######################
 # tox generated files #
 #######################

build/make/Makefile.in

@@ -331,10 +331,10 @@ clean:
 	@echo "Deleting package build directories..."
 	rm -rf "$(SAGE_LOCAL)/var/tmp/sage/build"
 
-# c_lib .cython_version are from old sage versions
+# "c_lib", ".cython_version", "build" in $(SAGE_SRC) are from old sage versions
 sagelib-clean:
 	@echo "Deleting Sage library build artifacts..."
-	cd "$(SAGE_SRC)" && (rm -rf c_lib .cython_version; rm -rf build; find . -name '*.pyc' | xargs rm -f; rm -rf sage/ext/interpreters)
+	(cd "$(SAGE_SRC)" && rm -rf c_lib .cython_version; rm -rf build; find . -name '*.pyc' | xargs rm -f; rm -rf sage/ext/interpreters) && (cd "$(SAGE_ROOT)/build/pkgs/sagelib/src/" && rm -rf build)
 
 build-clean: clean doc-clean sagelib-clean
 

build/pkgs/sagelib/spkg-install

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-cd "$SAGE_SRC"
+cd src
 ## All sagelib-building is done by setup.py.
 ## This is so that sagelib can be installed by standard Python procedures,
 ## such as "./setup.py install" or "pip install ."

build/pkgs/sagelib/spkg-src

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+#
+# Script to prepare an sdist tarball for sagelib
+# This script is not used during build.
+#
+# HOW TO MAKE THE TARBALL:
+# ./sage --sh build/pkgs/sagelib/spkg-src
+
+if [ -z "$SAGE_ROOT" ] ; then
+    echo >&2 "Error - SAGE_ROOT undefined ... exiting"
+    echo >&2 "Maybe run 'sage -sh'?"
+    exit 1
+fi
+
+# Exit on failure
+set -e
+
+cd build/pkgs/sagelib
+
+cd src
+sage-python23 -u setup.py --no-user-cfg sdist --dist-dir "$SAGE_DISTFILES"

build/pkgs/sagelib/src/MANIFEST.in

@@ -0,0 +1,8 @@
+global-include *.c *.cc *.cpp *.h *.hh *.hpp *.inc *.py *.pyx *.pxd *.pxi *.rst *.txt *.tex
+
+include MANIFEST.in
+
+prune .tox
+
+graft   sage/libs/gap/test
+prune sage/ext/interpreters   # In particular, __init__.py must not be present in the distribution; or sage_setup.autogen.interpreters.rebuild will not generate the code

build/pkgs/sagelib/src/bin

@@ -0,0 +1 @@
+../../../../src/bin

build/pkgs/sagelib/src/requirements.txt

@@ -0,0 +1,19 @@
+#sage_conf
+#sage_setup
+six             # use of six should be removed from sage_setup
+Cython==0.29.17
+pkgconfig
+cysignals
+gmpy2==2.1.0b1
+
+numpy          # already needed by sage.env
+jinja2         # sage_setup.autogen.interpreters
+
+cypari2        # but building bdist_wheel of cypari2 fails with recent pip... https://github.com/sagemath/cypari2/issues/93
+
+########## Runtime
+
+psutil
+pexpect
+pplpy
+ipython<=5.8

build/pkgs/sagelib/src/sage

@@ -0,0 +1 @@
+../../../../src/sage

build/pkgs/sagelib/src/sage_setup

@@ -0,0 +1 @@
+../../../../src/sage_setup

build/pkgs/sagelib/src/setup.py

@@ -0,0 +1 @@
+../../../../src/setup.py

build/pkgs/sagelib/src/tox.ini

@@ -0,0 +1,35 @@
+# First pip-install tox:
+#
+#   ./sage -pip install tox
+#
+# To build and test in the tox environment:
+#
+# ./sage -sh -c '(cd build/pkgs/sagelib/src && tox -v -v)'
+#
+# To test interactively:
+#
+# build/pkgs/sagelib/src/.tox/python/bin/python
+#
+[tox]
+
+[testenv]
+deps = -rrequirements.txt
+
+passenv =
+    SAGE_LOCAL
+
+setenv =
+    # Sage scripts such as sage-runtests like to use $HOME/.sage
+    HOME={envdir}
+
+whitelist_externals =
+    bash
+
+commands =
+    # Beware of the treacherous non-src layout. "./sage/" shadows the install sage package.
+    python -c 'import sys; "" in sys.path and sys.path.remove(""); import sage.all; print(sage.all.__file__)'
+
+    # FIXME: The following loads sage-env, which loads the wrong Python.
+    sage -c 'import sys; print("sys.path =", sys.path); import sage.all; print(sage.all.__file__)'
+
+    #sage -t --all

src/MANIFEST.in

@@ -77,8 +77,8 @@ def _find_stale_files(site_packages, python_packages, python_modules, ext_module
     course. We check that when the doctest is being run, that is,
     after installation, there are no stale files::
 
-        sage: from sage.env import SAGE_SRC, SAGE_LIB
-        sage: cythonized_dir = os.path.join(SAGE_SRC, "build", "cythonized")
+        sage: from sage.env import SAGE_SRC, SAGE_LIB, SAGE_ROOT
+        sage: cythonized_dir = os.path.join(SAGE_ROOT, "build", "pkgs", "sagelib", "src", "build", "cythonized")
         sage: from sage_setup.find import find_python_sources, find_extra_files
         sage: python_packages, python_modules, cython_modules = find_python_sources(
         ....:     SAGE_SRC, ['sage', 'sage_setup'])

src/sage_setup/clean.py

@@ -207,8 +207,8 @@ def find_extra_files(src_dir, modules, cythonized_dir, special_filenames=[]):
     EXAMPLES::
 
         sage: from sage_setup.find import find_extra_files
-        sage: from sage.env import SAGE_SRC
-        sage: cythonized_dir = os.path.join(SAGE_SRC, "build", "cythonized")
+        sage: from sage.env import SAGE_SRC, SAGE_ROOT
+        sage: cythonized_dir = os.path.join(SAGE_ROOT, "build", "pkgs", "sagelib", "src", "build", "cythonized")
         sage: extras = find_extra_files(SAGE_SRC, ["sage"], cythonized_dir)
         sage: extras["sage/libs/mpfr"]
         [...sage/libs/mpfr/types.pxd...]