Trac #33213: Replace SAGE_TMP by the system location in the sage library

Re: https://groups.google.com/g/sage-devel/c/zhjl_j6j_Qc

These days, using a `SAGE_TMP` that by default lives under `$HOME` is
overly complex and often inefficient. In this ticket we implement the
first phase of its removal, to be replaced by python's `tempfile`
module. Specifically,

  1. We replace all direct uses of `SAGE_TMP` within the sage library
and doctests.
  2. We update `tmp_dir()` and `tmp_filename()` to use the `tempfile`
defaults.
  3. We remove `SAGE_TMP`.

Afterward, the custom functions `tmp_dir()` and `tmp_filename()` can be
deprecated in favor of `tempfile.TemporaryDirectory()` and
`tempfile.NamedTemporaryFile()`.

Moreover when #8784 is done, we'll be able to remove sage-cleaner
entirely.

URL: https://trac.sagemath.org/33213
Reported by: mjo
Ticket author(s): Michael Orlitzky
Reviewer(s): Matthias Koeppe, Dima Pasechnik

Author: Release Manager

pkgs/sage-setup/tox.ini

@@ -4,11 +4,11 @@
 #
 # To build and test in the tox environment:
 #
-# ./sage -sh -c '(cd pkgs/sage-setup && tox)'
+# ./sage -sh -c '(cd pkgs/sage-setup && tox -e sagepython)'
 #
 # To test interactively:
 #
-# pkgs/sage-setup/.tox/python/bin/python
+# pkgs/sage-setup/.tox/sagepython/bin/python
 #
 [tox]
 
@@ -28,3 +28,9 @@ commands =
 
 # TODO: Test importing sage_setup.library_order -- when that can handle missing pkgconfig libraries...
 # TODO: Test more modules -- when the dependency on sage.env has been removed...
+
+[testenv:sagepython]
+passenv =
+    SAGE_VENV
+
+basepython = {env:SAGE_VENV}/bin/python3

pkgs/sagemath-categories/tox.ini

@@ -1,10 +1,10 @@
 # To build and test in the tox environment:
 #
-# ./sage -sh -c '(cd pkgs/sagemath-categories/src && tox -v -v -v)'
+# ./sage -sh -c '(cd pkgs/sagemath-categories/src && tox -v -v -v -e sagepython)'
 #
 # To test interactively:
 #
-# pkgs/sagemath-categories/.tox/python/bin/python
+# pkgs/sagemath-categories/.tox/sagepython/bin/python
 #
 [tox]
 
@@ -31,3 +31,9 @@ commands =
     python -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.categories.all import *; SimplicialComplexes(); FunctionFields()'
 
     bash -c 'cd bin && SAGE_SRC=$(python -c "from sage.env import SAGE_SRC; print(SAGE_SRC)") && sage-runtests --environment=sage.all__sagemath_categories --optional=sage $SAGE_SRC/sage/structure  || echo "(lots of doctest failures are expected)"'
+
+[testenv:sagepython]
+passenv =
+    SAGE_VENV
+
+basepython = {env:SAGE_VENV}/bin/python3

pkgs/sagemath-objects/tox.ini

@@ -1,10 +1,10 @@
 # To build and test in the tox environment:
 #
-# ./sage -sh -c '(cd pkgs/sagemath-objects && tox -v -v)'
+# ./sage -sh -c '(cd pkgs/sagemath-objects && tox -v -v -e sagepython)'
 #
 # To test interactively:
 #
-# pkgs/sagemath-objects/.tox/python/bin/python
+# pkgs/sagemath-objects/.tox/sagepython/bin/python
 #
 [tox]
 
@@ -29,3 +29,9 @@ commands =
     python -c 'import sys; "" in sys.path and sys.path.remove(""); from sage.all__sagemath_objects import *'
 
     bash -c 'cd bin && SAGE_SRC=$(python -c "from sage.env import SAGE_SRC; print(SAGE_SRC)") && sage-runtests --environment=sage.all__sagemath_objects --optional=sage $SAGE_SRC/sage/structure || echo "(lots of doctest failures are expected)"'
+
+[testenv:sagepython]
+passenv =
+    SAGE_VENV
+
+basepython = {env:SAGE_VENV}/bin/python3

pkgs/sagemath-standard/tox.ini

@@ -9,7 +9,8 @@
 #
 # To build and test in the tox environment using the concrete Python dependencies specified
 # by requirements.txt, using the wheels built and stored by the Sage distribution:
-# (Using 'sage -sh' ensures that we use the same Python as the one that we built the wheels
+# (Using 'sage -sh' in combination with 'sagepython-...' tox environments
+#  ensures that we use the same Python as the one that we built the wheels
 #  for. This can also be done ensured manually by using the tox environment py38-sagewheels etc.)
 #
 # Afterwards, to test interactively:
@@ -25,17 +26,17 @@ envlist =
     # Build dependencies according to requirements.txt (all versions fixed).
     # Use ONLY the wheels built and stored by the Sage distribution (no PyPI):
     #
-    #   ./sage -sh -c '(cd pkgs/sagemath-standard && tox -v -v -v -e python-sagewheels-nopypi)'
+    #   ./sage -sh -c '(cd pkgs/sagemath-standard && tox -v -v -v -e sagepython-sagewheels-nopypi)'
     #
-    python-sagewheels-nopypi,
+    sagepython-sagewheels-nopypi,
     #
     # Build and test without using the concrete dependencies specified by requirements.txt,
     # using the dependencies declared in pyproject.toml and setup.cfg (install-requires) only:
     # Still use ONLY the wheels built and stored by the Sage distribution (no PyPI).
     #
-    #   ./sage -sh -c '(cd pkgs/sagemath-standard && tox -v -v -v -e python-sagewheels-nopypi-norequirements)'
+    #   ./sage -sh -c '(cd pkgs/sagemath-standard && tox -v -v -v -e sagepython-sagewheels-nopypi-norequirements)'
     #
-    python-sagewheels-nopypi-norequirements,
+    sagepython-sagewheels-nopypi-norequirements,
     #
     # EXPERIMENTAL ENVIRONMENTS:
     #
@@ -44,14 +45,14 @@ envlist =
     # and additionally allow packages from PyPI.
     # Because all versions are fixed, we "should" end up using the prebuilt wheels.
     #
-    #   ./sage -sh -c '(cd pkgs/sagemath-standard && tox -v -v -v -e python-sagewheels)'
+    #   ./sage -sh -c '(cd pkgs/sagemath-standard && tox -v -v -v -e sagepython-sagewheels)'
     #
-    python-sagewheels,
+    sagepython-sagewheels,
     #
     # Likewise, but using pipenv using Pipfile-dist (= SAGE_ROOT/Pipfile).
     # This also fixes the concrete dependencies (at least for some packages).
     #
-    python-sagewheels-pipenv-dist,
+    sagepython-sagewheels-pipenv-dist,
     #
     # Build using the dependencies declared in pyproject.toml and setup.cfg (install-requires) only.
     # Use the wheels built and stored by the Sage distribution,
@@ -60,11 +61,11 @@ envlist =
     # Because the version ranges will allow for packages to come in from PyPI (in source or wheel form),
     # this is likely to fail because we do not have control over the configuration of these packages.
     #
-    python-sagewheels-norequirements,
+    sagepython-sagewheels-norequirements,
     #
     # Likewise, but using pipenv
     #
-    python-sagewheels-pipenv
+    sagepython-sagewheels-pipenv
 
 [testenv]
 deps =
@@ -129,3 +130,9 @@ commands =
     sage -c 'import sys; print("sys.path =", sys.path); import sage.all; print(sage.all.__file__)'
 
     sage -t -p --all
+
+[testenv:sagepython]
+passenv =
+    SAGE_VENV
+
+basepython = {env:SAGE_VENV}/bin/python3

src/doc/en/developer/packaging_sage_library.rst

@@ -584,19 +584,19 @@ Following the comments in the file
 ``SAGE_ROOT/pkgs/sagemath-standard/tox.ini``, we can try the following
 command::
 
-  $ ./bootstrap && ./sage -sh -c '(cd pkgs/sagemath-standard && SAGE_NUM_THREADS=16 tox -v -v -v -e py39-sagewheels-nopypi)'
+  $ ./bootstrap && ./sage -sh -c '(cd pkgs/sagemath-standard && SAGE_NUM_THREADS=16 tox -v -v -v -e sagepython-sagewheels-nopypi)'
 
 This command does not make any changes to the normal installation of
 Sage. The virtual environment is created in a subdirectory of
 ``SAGE_ROOT/pkgs/sagemath-standard-no-symbolics/.tox/``. After the command
 finishes, we can start the separate installation of the Sage library
 in its virtual environment::
 
-  $ pkgs/sagemath-standard/.tox/py39-sagewheels-nopypi/bin/sage
+  $ pkgs/sagemath-standard/.tox/sagepython-sagewheels-nopypi/bin/sage
 
 We can also run parts of the testsuite::
 
-  $ pkgs/sagemath-standard/.tox/py39-sagewheels-nopypi/bin/sage -tp 4 src/sage/graphs/
+  $ pkgs/sagemath-standard/.tox/sagepython-sagewheels-nopypi/bin/sage -tp 4 src/sage/graphs/
 
 The whole ``.tox`` directory can be safely deleted at any time.
 
@@ -609,15 +609,15 @@ without depending on optional packages, but without the packages
 
 Again we can run the test with ``tox`` in a separate virtual environment::
 
-  $ ./bootstrap && ./sage -sh -c '(cd pkgs/sagemath-standard-no-symbolics && SAGE_NUM_THREADS=16 tox -v -v -v -e py39-sagewheels-nopypi)'
+  $ ./bootstrap && ./sage -sh -c '(cd pkgs/sagemath-standard-no-symbolics && SAGE_NUM_THREADS=16 tox -v -v -v -e sagepython-sagewheels-nopypi)'
 
 Some small distributions, for example the ones providing the two
 lowest levels, `sagemath-objects <https://pypi.org/project/sagemath-objects/>`_
 and `sagemath-categories <https://pypi.org/project/sagemath-categories/>`_
 (from :trac:`29865`), can be installed and tested
 without relying on the wheels from the Sage build::
 
-  $ ./bootstrap && ./sage -sh -c '(cd pkgs/sagemath-objects && SAGE_NUM_THREADS=16 tox -v -v -v -e py39)'
+  $ ./bootstrap && ./sage -sh -c '(cd pkgs/sagemath-objects && SAGE_NUM_THREADS=16 tox -v -v -v -e sagepython)'
 
 This command finds the declared build-time and run-time dependencies
 on PyPI, either as source tarballs or as prebuilt wheels, and builds

src/sage/combinat/cluster_algebra_quiver/cluster_seed.py

@@ -1080,7 +1080,9 @@ def save_image(self, filename, circular=False, mark=None, save_pos=False):
         EXAMPLES::
 
             sage: S = ClusterSeed(['F',4,[1,2]])
-            sage: S.save_image(os.path.join(SAGE_TMP, 'sage.png'))
+            sage: import tempfile
+            sage: with tempfile.NamedTemporaryFile(suffix=".png") as f:
+            ....:     S.save_image(f.name)
         """
         graph_plot = self.plot( circular=circular, mark=mark, save_pos=save_pos)
         graph_plot.save( filename=filename )

src/sage/combinat/cluster_algebra_quiver/quiver.py

@@ -715,7 +715,9 @@ def save_image(self, filename, circular=False):
         EXAMPLES::
 
             sage: Q = ClusterQuiver(['F',4,[1,2]])
-            sage: Q.save_image(os.path.join(SAGE_TMP, 'sage.png'))
+            sage: import tempfile
+            sage: with tempfile.NamedTemporaryFile(suffix=".png") as f:
+            ....:     Q.save_image(f.name)
         """
         graph_plot = self.plot(circular=circular)
         graph_plot.save(filename=filename)
@@ -738,14 +740,18 @@ def qmu_save(self, filename=None):
         EXAMPLES::
 
             sage: Q = ClusterQuiver(['F',4,[1,2]])
-            sage: Q.qmu_save(os.path.join(SAGE_TMP, 'sage.qmu'))
+            sage: import tempfile
+            sage: with tempfile.NamedTemporaryFile(suffix=".qmu") as f:
+            ....:     Q.qmu_save(f.name)
 
         Make sure we can save quivers with `m != n` frozen variables, see :trac:`14851`::
 
             sage: S = ClusterSeed(['A',3])
             sage: T1 = S.principal_extension()
             sage: Q = T1.quiver()
-            sage: Q.qmu_save(os.path.join(SAGE_TMP, 'sage.qmu'))
+            sage: import tempfile
+            sage: with tempfile.NamedTemporaryFile(suffix=".qmu") as f:
+            ....:     Q.qmu_save(f.name)
         """
         M = self.b_matrix()
         if self.m():

src/sage/combinat/words/morphism.py

@@ -761,7 +761,9 @@ def __call__(self, w, order=1, datatype=None):
 
             sage: w == loads(dumps(w))
             True
-            sage: save(w, filename=os.path.join(SAGE_TMP, 'test.sobj'))
+            sage: import tempfile
+            sage: with tempfile.NamedTemporaryFile(suffix=".sobj") as f:
+            ....:     save(w, filename=f.name)
 
         The ``datatype`` argument is deprecated::
 

src/sage/interfaces/cleaner.py

@@ -17,24 +17,10 @@
 
 import os
 
-from sage.misc.misc import SAGE_TMP
-
-def cleaner(pid, cmd=''):
-    """
-    Write a line to the ``spawned_processes`` file with the given
-    ``pid`` and ``cmd``.
-    """
-    if cmd != '':
-        cmd = cmd.strip().split()[0]
-    # This is safe, since only this process writes to this file.
-    F = os.path.join(SAGE_TMP, 'spawned_processes')
-    try:
-        with open(F, 'a') as o:
-            o.write('%s %s\n'%(pid, cmd))
-    except IOError:
-        pass
-    else:
-        start_cleaner()
+import atexit, tempfile
+_spd = tempfile.TemporaryDirectory()
+SAGE_SPAWNED_PROCESS_FILE = os.path.join(_spd.name, "spawned_processes")
+atexit.register(lambda: _spd.cleanup())
 
 
 def start_cleaner():

src/sage/interfaces/expect.py

@@ -48,7 +48,6 @@
 import time
 import gc
 from . import quit
-from . import cleaner
 from random import randrange
 
 import pexpect
@@ -59,7 +58,6 @@
 
 from sage.structure.element import RingElement
 
-from sage.misc.misc import SAGE_TMP_INTERFACE
 from sage.env import SAGE_EXTCODE, LOCAL_IDENTIFIER
 from sage.misc.object_multiplexer import Multiplex
 from sage.misc.instancedoc import instancedoc
@@ -440,7 +438,11 @@ def _do_cleaner(self):
             return False
 
     def _start(self, alt_message=None, block_during_init=True):
-        self.quit()  # in case one is already running
+        if self.is_running():
+            # In case one is already running. We check first because
+            # quit() can reset the local temporary filename at an
+            # unexpected time as the process is started "on demand."
+            self.quit()
 
         self._session_number += 1
 
@@ -515,7 +517,7 @@ def _start(self, alt_message=None, block_during_init=True):
             os.chdir(currentdir)
 
         if self._do_cleaner():
-            cleaner.cleaner(self._expect.pid, cmd)
+            quit.register_spawned_process(self._expect.pid, cmd)
 
         try:
             self._expect.expect(self._prompt)
@@ -594,6 +596,14 @@ def _reset_expect(self):
         """
         self._session_number += 1
         try:
+            # Spaghetti alert: when running several computations in
+            # parallel, the pexpect interface is reset in each one,
+            # and this next line is needed to trigger the generation
+            # of a new temporary file when otherwise the existing
+            # member variable would be shared. That also means that
+            # you can't use this method to clean up an existing
+            # tmpfile, because you can delete the one that the parent
+            # is using.
             del self.__local_tmpfile
         except AttributeError:
             pass
@@ -732,8 +742,17 @@ def _local_tmpfile(self):
         try:
             return self.__local_tmpfile
         except AttributeError:
-            self.__local_tmpfile = os.path.join(SAGE_TMP_INTERFACE, 'tmp' + str(self.pid()))
-            return self.__local_tmpfile
+            pass
+
+        import atexit, os
+        from tempfile import NamedTemporaryFile
+        # FriCAS uses the ".input" suffix, and the other
+        # interfaces are suffix-agnostic, so using ".input" here
+        # lets us avoid a subclass override for FriCAS.
+        with NamedTemporaryFile(suffix=".input", delete=False) as f:
+            self.__local_tmpfile = f.name
+            atexit.register(lambda: os.remove(f.name))
+        return self.__local_tmpfile
 
     def _remote_tmpdir(self):
         return self.__remote_tmpdir

src/sage/interfaces/fricas.py

@@ -200,7 +200,6 @@
 import os
 from sage.interfaces.tab_completion import ExtraTabCompletion
 from sage.interfaces.expect import Expect, ExpectElement, FunctionElement, ExpectFunction
-from sage.misc.misc import SAGE_TMP_INTERFACE
 from sage.env import DOT_SAGE, LOCAL_IDENTIFIER
 from sage.misc.instancedoc import instancedoc
 from sage.rings.integer_ring import ZZ
@@ -475,17 +474,6 @@ def _read_in_file_command(self, filename):
 
         return ')read %s )quiet' % filename
 
-    def _local_tmpfile(self):
-        """
-        Return a local tmpfile ending with ".input" used to buffer long
-        command lines sent to FriCAS.
-
-        """
-        try:
-            return self.__local_tmpfile
-        except AttributeError:
-            self.__local_tmpfile = os.path.join(SAGE_TMP_INTERFACE, 'tmp' + str(self.pid()) + '.input')
-            return self.__local_tmpfile
 
     def _remote_tmpfile(self):
         """