Merge tag '9.7.beta3' into t/32423/update_numpy_to_1_22_x__scipy_1_8_x___requires_dropping_python_3_7

SageMath version 9.7.beta3, Release Date: 2022-06-19

Author: Matthias Koeppe

.github/workflows/build.yml

@@ -29,11 +29,13 @@ jobs:
 
       - name: Prepare
         run: |
+          # Install test tools.
+          # Installation of python3-venv can be removed as soon as a
+          # base image with a release including #33822 is available
+          apt-get install -y git python3-venv
           # Reuse built SAGE_LOCAL contained in the Docker image 
           ./bootstrap
           ./configure --enable-build-as-root --prefix=/sage/local --with-sage-venv --enable-editable --enable-download-from-upstream-url
-          # Install test tools
-          apt-get install -y git
       
       - name: Static code check with pyright
         run: pyright    
@@ -44,6 +46,16 @@ jobs:
           MAKE: make -j2
           SAGE_NUM_THREADS: 2
 
+      - name: Pytest
+        if: contains(github.ref, 'pytest')
+        run: |
+          ../sage -python -m pip install coverage pytest-xdist
+          ../sage -python -m coverage run -m pytest -c tox.ini --doctest-modules || true
+        working-directory: ./src
+        env:
+          # Increase the length of the lines in the "short summary"
+          COLUMNS: 120
+
       - name: Test
         run: |
           ../sage -python -m pip install coverage

.github/workflows/doc-build.yml

@@ -24,6 +24,7 @@ jobs:
           # Reuse built SAGE_LOCAL contained in the Docker image 
           ./bootstrap
           ./configure --enable-build-as-root --prefix=/sage/local --with-sage-venv --enable-download-from-upstream-url
+          make furo
 
       - name: Build
         run: make doc-html

.github/workflows/tox.yml

@@ -38,7 +38,7 @@ jobs:
       fail-fast: false
       max-parallel: 20
       matrix:
-        tox_system_factor: [gitpod, ubuntu-trusty-toolchain-gcc_9, ubuntu-xenial-toolchain-gcc_9, ubuntu-bionic, ubuntu-focal, ubuntu-hirsute, ubuntu-impish, ubuntu-jammy, ubuntu-kinetic, debian-stretch, debian-buster, debian-bullseye, debian-bookworm, debian-sid, linuxmint-19, linuxmint-19.3, linuxmint-20.1, linuxmint-20.2, linuxmint-20.3, linuxmint-21, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, fedora-35, fedora-36, fedora-37, centos-7-devtoolset-gcc_11, centos-stream-8, gentoo-python3.9, gentoo-python3.10, archlinux-latest, opensuse-15.3, opensuse-tumbleweed, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386-devtoolset-gcc_11]
+        tox_system_factor: [ubuntu-trusty-toolchain-gcc_9, ubuntu-xenial-toolchain-gcc_9, ubuntu-bionic, ubuntu-focal, ubuntu-hirsute, ubuntu-impish, ubuntu-jammy, ubuntu-kinetic, debian-stretch, debian-buster, debian-bullseye, debian-bookworm, debian-sid, linuxmint-19, linuxmint-19.3, linuxmint-20.1, linuxmint-20.2, linuxmint-20.3, linuxmint-21, fedora-26, fedora-27, fedora-28, fedora-29, fedora-30, fedora-31, fedora-32, fedora-33, fedora-34, fedora-35, fedora-36, fedora-37, centos-7-devtoolset-gcc_11, centos-stream-8, gentoo-python3.9, gentoo-python3.10, archlinux-latest, opensuse-15.3, opensuse-tumbleweed, conda-forge, ubuntu-bionic-i386, manylinux-2_24-i686, debian-buster-i386, centos-7-i386-devtoolset-gcc_11]
         tox_packages_factor: [minimal, standard]
     env:
       TOX_ENV: docker-${{ matrix.tox_system_factor }}-${{ matrix.tox_packages_factor }}

.gitignore

@@ -166,10 +166,16 @@ build/bin/sage-build-env-config
 
 /pkgs/sagemath-objects/setup.cfg
 /pkgs/sagemath-categories/setup.cfg
+/pkgs/sagemath-environment/setup.cfg
+/pkgs/sagemath-repl/setup.cfg
 /pkgs/sagemath-objects/pyproject.toml
 /pkgs/sagemath-categories/pyproject.toml
+/pkgs/sagemath-environment/pyproject.toml
+/pkgs/sagemath-repl/pyproject.toml
 /pkgs/sagemath-objects/requirements.txt
 /pkgs/sagemath-categories/requirements.txt
+/pkgs/sagemath-environment/requirements.txt
+/pkgs/sagemath-repl/requirements.txt
 /pkgs/sagemath-categories/MANIFEST.in
 
 # same for old locations - before Trac #31577
@@ -181,6 +187,7 @@ build/bin/sage-build-env-config
 
 # Generated by docbuild
 /src/doc/en/reference/*/sage
+/src/doc/en/reference/*/sage_docbuild
 /src/doc/en/reference/sage
 /src/doc/en/reference/spkg/*.rst
 /src/doc/output

.gitpod.yml

@@ -1,41 +1,32 @@
 # Use custom docker image. https://www.gitpod.io/docs/config-docker
 image:
-# Each of these two options works:
-# - Directly use the "-with-targets" image built by .github/workflows/tox.yml
-#   image: ghcr.io/sagemath/sage/sage-docker-gitpod-standard-with-targets:dev
-# - or go through a custom Dockerfile, which builds a smaller image
-#   based on the "-with-system-packages" image built by .github/workflows/tox.yml
-#   with the built SAGE_LOCAL from the "-with-targets" image copied in.
-    file: docker/.gitpod.Dockerfile
-
-# We use the following layout:
-#
-# $HOME/sage            Source tree used by the Docker build; see SAGE_ROOT/tox.ini (gitpod).
-#                       - We delete it in every invocation of the 'before' script
-#                         and replace it by a symlink to /workspace/...
-#                         (This symlink is needed because the package-removal scripts
-#                         ({prefix,venv}/var/lib/sage/scripts/*/{spkg-piprm,spkg-prerm,spkg-postrm)
-#                         hardcode SAGE_ROOT and SAGE_SRC from package installation time)
-# $HOME/sage/logs       Logs of the Docker build.
-#                       - In the first invocation of the 'before' script, we move it
-#                         to /workspace/.../logs
-# $HOME/sage-local      The configured prefix (SAGE_LOCAL) of the Sage installation.
-#                       - During the Docker build, this is the physical location.
-#                       - In the first invocation of the 'before' script, we move it
-#                         to the new physical location /workspace/.../local
-#                         (because gitpod only preserves the contents of /workspace)
-#                         and replace it by a symlink to the new physical location.
-#                       - In subsequent invocations of the 'before' script, we
-#                         remove it and replace it by a symlink to the physical
-#                         location /workspace/.../local
-# /worktree/.../local   The physical location of the Sage installation,
-#                       established in the first run of the 'before' script and
-#                       preserved by gitpod.
+  file: docker/.gitpod.Dockerfile
 
 # Start up tasks. https://www.gitpod.io/docs/config-start-tasks/
 tasks:
   - name: Setup
-    before: |
+    init: |
+      # Create conda environment
+      ./bootstrap-conda
+      mamba env create --file src/environment-dev.yml --prefix venv
+      conda config --append envs_dirs /workspace/sagetrac-mirror
+      conda activate /workspace/sagetrac-mirror/venv
+      
+      # Build sage
+      ./bootstrap
+      ./configure --enable-build-as-root --with-python=$CONDA_PREFIX/bin/python --prefix=$CONDA_PREFIX 
+      pip install --no-build-isolation -v -v -e ./pkgs/sage-conf ./pkgs/sage-setup 
+      pip install --no-build-isolation -v -v -e ./src
+
+    command: |
+      # Activate conda environment
+      conda config --append envs_dirs /workspace/sagetrac-mirror
+      conda activate /workspace/sagetrac-mirror/venv
+
+      # RestructuredText extension recommends python extension, although we have already installed it
+      ## So disable the recommendation dialog
+      echo "{\"restructuredtext.pythonRecommendation.disabled\": true}" > /workspace/.vscode-remote/data/Machine/settings.json
+
       # Setup trac as remote
       ## In order to push to trac, generate a new key with `ssh-keygen -f tempkey` and save the private key to gitpod `gp env PRIVATE_SSH_KEY="$(<tempkey)"` (or by following https://www.gitpod.io/docs/environment-variables#using-the-account-settings)
       ## then follow https://doc.sagemath.org/html/en/developer/trac.html#linking-your-public-key-to-your-trac-account to register the public key with trac.
@@ -47,7 +38,7 @@ tasks:
         echo $PRIVATE_SSH_KEY | sed 's/\(-----\(BEGIN\|END\) OPENSSH PRIVATE KEY-----\)/\n\1\n/g' > ~/.ssh/id_rsa
         sed -i '/^$/d' ~/.ssh/id_rsa
         chmod 600 ~/.ssh/id_rsa
-        unset PRIVATE_SSH_KEY
+        echo "PubkeyAcceptedKeyTypes +ssh-rsa" > ~/.ssh/config
         ssh-keyscan -H trac.sagemath.org >> ~/.ssh/known_hosts
 
         # Setup trac repo
@@ -60,43 +51,7 @@ tasks:
         git remote add trac https://github.com/sagemath/sagetrac-mirror.git -t master -t develop
         git remote set-url --push trac pushing-needs-ssh-key
       fi
-      
-      ## No need for pyenv
-      pyenv shell --unset 2> /dev/null
-      pyenv global system 2> /dev/null
-      if [ -d local ]; then
-        mkdir -p logs && echo '### .gitpod.yml Setup.before: Prebuild init script has been run. Running "make" again in case the build had timed out.' >> logs/install.log
-        # The init script has already populated the SAGE_LOCAL,
-        # but only /workspace is preserved; and the $HOME/sage-local may contain a resurrected
-        # copy of sage-local.  Replace it again by a symlink.
-        rm -Rf $HOME/sage-local
-        ln -sf $(pwd)/local $HOME/sage-local
-        # Now run make. No timeout here.
-        MAKE='make -j24' make build V=0
-      else
-        # Prebuild init script has not been run
-        # Only /workspace is preserved during build.
-        # If the Docker image contains a built SAGE_LOCAL, use it to populate the SAGE_LOCAL in the workspace.
-        if [ -d $HOME/sage-local ]; then
-          mv $HOME/sage-local local
-        fi
-        rm -Rf $HOME/sage-local
-        ln -sf $(pwd)/local $HOME/sage-local
-        # Save the logs of the source tree used by the Docker build
-        if [ -d $HOME/sage/logs ]; then
-          mv $HOME/sage/logs logs
-        fi
-      fi
-      # Remove the source tree used by the Docker build and replace it by a symlink
-      rm -Rf $HOME/sage
-      ln -s $(pwd) $HOME/sage
-    init: |
-      # Start build
-      mkdir -p logs && echo '### .gitpod.yml Setup.init: Starting build' >> logs/install.log
-      ./bootstrap
-      ./configure --enable-editable --enable-download-from-upstream-url --prefix=$HOME/sage-local --with-sage-venv
-      ## Gitpod has a timeout of 1h, so make sure we are below this to ensure that the prebuild is always successful
-      MAKE='make -j24' timeout 51m make build V=0 || echo "(ignoring error)"
+
     env:
       SAGE_NUM_THREADS: 8
 
@@ -109,6 +64,8 @@ vscode:
     - lextudio.restructuredtext
     - streetsidesoftware.code-spell-checker
     - ms-toolsai.jupyter
+    - ms-toolsai.jupyter-keymap
+    - ms-toolsai.jupyter-renderers
 
 # https://www.gitpod.io/docs/prebuilds#github-specific-configuration
 github:

.vscode/settings.json

@@ -21,7 +21,10 @@
         "--doctest-modules",
     ],
     "python.testing.unittestEnabled": false,
+    "python.linting.pycodestyleEnabled": true,
+    "python.linting.enabled": true,
     "cSpell.words": [
+        "furo"
         "Conda",
         "Cython"
     ],

.zenodo.json

@@ -1,10 +1,10 @@
 {
     "description": "Mirror of the Sage https://sagemath.org/ source tree", 
     "license": "other-open", 
-    "title": "sagemath/sage: 9.7.beta1", 
-    "version": "9.7.beta1", 
+    "title": "sagemath/sage: 9.7.beta3", 
+    "version": "9.7.beta3", 
     "upload_type": "software", 
-    "publication_date": "2022-05-26", 
+    "publication_date": "2022-06-19", 
     "creators": [
         {
             "affiliation": "SageMath.org", 
@@ -15,7 +15,7 @@
     "related_identifiers": [
         {
             "scheme": "url", 
-            "identifier": "https://github.com/sagemath/sage/tree/9.7.beta1", 
+            "identifier": "https://github.com/sagemath/sage/tree/9.7.beta3", 
             "relation": "isSupplementTo"
         }, 
         {

Makefile

@@ -87,6 +87,8 @@ pypi-sdists: sage_setup
 	./sage --sh build/pkgs/sagelib/spkg-src
 	./sage --sh build/pkgs/sagemath_objects/spkg-src
 	./sage --sh build/pkgs/sagemath_categories/spkg-src
+	./sage --sh build/pkgs/sagemath_environment/spkg-src
+	./sage --sh build/pkgs/sagemath_repl/spkg-src
 	@echo "Built sdists are in upstream/"
 
 # ssl: build Sage, and also install pyOpenSSL. This is necessary for
@@ -168,7 +170,6 @@ bootstrap-clean:
 	rm -rf config configure build/make/Makefile-auto.in
 	rm -f src/doc/en/installation/*.txt
 	rm -rf src/doc/en/reference/spkg/*.rst
-	rm -f src/doc/en/reference/repl/*.txt
 	rm -f environment.yml
 	rm -f src/environment.yml
 	rm -f src/environment-dev.yml

VERSION.txt

@@ -1 +1 @@
-SageMath version 9.7.beta1, Release Date: 2022-05-26
+SageMath version 9.7.beta3, Release Date: 2022-06-19

bootstrap

@@ -85,12 +85,12 @@ bootstrap () {
     rm -f m4/sage_spkg_configures.m4
     spkg_configures=""
     # initialize SAGE_ENABLE... options for standard packages
-    for pkgname in $(./sage --package list :standard: | sort); do
+    for pkgname in $(sage-package list :standard: | sort); do
         spkg_configures="$spkg_configures
 AS_VAR_SET_IF([SAGE_ENABLE_$pkgname], [], [AS_VAR_SET([SAGE_ENABLE_$pkgname], [yes])])"
     done
     # --enable-SPKG options
-    for pkgname in $(./sage --package list :optional: :experimental: | sort); do
+    for pkgname in $(sage-package list :optional: :experimental: | sort); do
                 # Trac #29629: Temporary solution for Sage 9.1: Do not provide
                 # --enable-SPKG options for installing pip packages
                 if [ ! -f build/pkgs/$pkgname/requirements.txt ]; then
@@ -109,12 +109,12 @@ SAGE_SPKG_ENABLE([$pkgname], [$pkgtype], [$(grep -v ^= build/pkgs/$pkgname/SPKG.
                     esac
                 fi
     done
-    for pkgname in $(./sage --package list --has-file spkg-configure.m4 | sort); do
+    for pkgname in $(sage-package list --has-file spkg-configure.m4 | sort); do
         echo "m4_sinclude([build/pkgs/$pkgname/spkg-configure.m4])"
         spkg_configures="$spkg_configures
 SAGE_SPKG_CONFIGURE_$(echo ${pkgname} | tr '[a-z]' '[A-Z]')"
     done >> m4/sage_spkg_configures.m4
-    for pkgname in $(./sage --package list | sort); do
+    for pkgname in $(sage-package list | sort); do
         DIR=build/pkgs/$pkgname
         pkgtype="$(cat $DIR/type)"
         if test -f "$DIR/requirements.txt"; then
@@ -143,7 +143,7 @@ SAGE_SPKG_FINALIZE([$pkgname], [$pkgtype], [$SPKG_SOURCE], [$SPKG_TREE_VAR])"
     done
     echo "$spkg_configures" >> m4/sage_spkg_configures.m4
 
-    for pkgname in $(./sage --package list --has-file bootstrap); do
+    for pkgname in $(sage-package list --has-file bootstrap); do
         (cd build/pkgs/$pkgname && ./bootstrap) || exit 1
     done
 
@@ -247,7 +247,6 @@ save () {
 	build/make/Makefile-auto.in \
 	src/doc/en/installation/*.txt \
 	src/doc/en/reference/spkg/*.rst \
-	src/doc/en/reference/repl/*.txt \
 	environment.yml \
 	src/environment.yml \
 	environment-optional.yml \
@@ -262,10 +261,10 @@ save () {
 
     # Compute checksum
     if [ "${BOOTSTRAP_QUIET}" = "no" ]; then
-	./sage --package fix-checksum configure
+	sage-package fix-checksum configure
     else
 	# Hide the "Updating checksum..." message
-	./sage --package fix-checksum configure > /dev/null
+	sage-package fix-checksum configure > /dev/null
     fi
 }
 

bootstrap-conda

@@ -4,6 +4,8 @@
 # Generate auto-generated conda environment files
 #########################################################################
 
+export PATH="$(pwd)/build/bin:$PATH"
+
 STRIP_COMMENTS="sed s/#.*//;"
 RECOMMENDED_SPKG_PATTERN="@(_recommended$(for a in $(head -n 1 build/pkgs/_recommended/dependencies); do echo -n "|"$a; done))"
 
@@ -13,7 +15,7 @@ OPTIONAL_SYSTEM_PACKAGES=
 SAGELIB_SYSTEM_PACKAGES=
 SAGELIB_OPTIONAL_SYSTEM_PACKAGES=
 RECOMMENDED_SYSTEM_PACKAGES=
-for PKG_BASE in $(./sage --package list --has-file distros/conda.txt); do
+for PKG_BASE in $(sage-package list --has-file distros/conda.txt); do
     PKG_SCRIPTS=build/pkgs/$PKG_BASE
     SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/distros/conda.txt
     PKG_TYPE=$(cat $PKG_SCRIPTS/type)
@@ -65,6 +67,7 @@ echo "  # Additional dev tools" >> src/environment-dev.yml
 echo "  - openssh" >> src/environment-dev.yml
 echo "  - pycodestyle" >> src/environment-dev.yml
 echo "  - pytest" >> src/environment-dev.yml
+echo "  - esbonio" >> src/environment-dev.yml
 
 cp environment.yml environment-optional.yml
     echo "  # optional packages" >> environment-optional.yml

build/bin/sage-bootstrap-python

@@ -11,7 +11,17 @@ if [ -z "$SAGE_ORIG_PATH" ]; then
     # If not we're running from within sage-env just set the existing path
     SAGE_ORIG_PATH="$PATH"
 fi
-
+IFS=':'
+NEW_PATH=""
+for path in $SAGE_ORIG_PATH
+do
+    case "$path" in
+        */.pyenv/shims*);;
+        *) NEW_PATH="$NEW_PATH$path:";;
+    esac
+done
+unset IFS
+SAGE_ORIG_PATH=${NEW_PATH%%':'}
 # In particular, it is invoked by "bootstrap -d" for sage-download-file,
 # i.e., before a configure run, and by "sage-spkg", also for sage-download-file.
 # So it needs to find a python that has the urllib module.