Trac #31306: sage.repl: Replace use of SAGE_EXTCODE by importlib.resources

We eliminate direct reading of files from the package directories of
sagelib by using `importlib.resources` (available since python 3.7)
- `git grep 'SAGE_EXTCODE' src/sage`
- `git grep '__file__' src/sage`

This will help make sagelib `zip_safe` (https://setuptools.readthedocs.i
o/en/latest/userguide/miscellaneous.html#setting-the-zip-safe-flag)

In this ticket, we take care of `sage.repl`, moving data needed by its
doctests from `SAGE_EXTCODE` (= `src/sage/ext_data`) into the package
directory. (Thus, we avoid a dependency on support for resources in
namespace packages brought by Python 3.10(?) or the backport package
`importlib-resources`, see
python/importlib_resources#196)

Follow-up tickets will deal with other parts of the library. In the end,
the directory `src/sage/ext_data` will be eliminated.

References:
- https://docs.python.org/3/library/importlib.html#module-
importlib.resources
- https://importlib-resources.readthedocs.io/en/latest/migration.html
- https://importlib-resources.readthedocs.io/en/latest/using.html

URL: https://trac.sagemath.org/31306
Reported by: mkoeppe
Ticket author(s): Matthias Koeppe
Reviewer(s): Dima Pasechnik

Author: Release Manager

src/sage/repl/display/formatter.py

@@ -142,16 +142,16 @@ def format(self, obj, include=None, exclude=None):
         TESTS::
 
             sage: import os
-            sage: from sage.env import SAGE_EXTCODE
-            sage: example_png = os.path.join(SAGE_EXTCODE, 'doctest', 'rich_output', 'example.png')
+            sage: import importlib.resources
             sage: from sage.repl.rich_output.backend_ipython import BackendIPython
             sage: backend = BackendIPython()
             sage: shell = get_test_shell()
             sage: backend.install(shell=shell)
             sage: shell.run_cell('get_ipython().display_formatter')
             <sage.repl.display.formatter.SageDisplayFormatter object at 0x...>
             sage: shell.run_cell('from IPython.display import Image')
-            sage: shell.run_cell('ipython_image = Image("{0}")'.format(example_png))
+            sage: with importlib.resources.path(sage.repl.rich_output, 'example.png') as example_png:
+            ....:     shell.run_cell('ipython_image = Image("{0}")'.format(example_png))
             sage: shell.run_cell('ipython_image')
             <IPython.core.display.Image object>
             sage: shell.run_cell('get_ipython().display_formatter.format(ipython_image)')

src/sage/ext_data/doctest/rich_output/example.avi src/sage/repl/rich_output/example.avi

@@ -18,6 +18,7 @@
 
 import os
 import base64
+import importlib.resources
 
 from sage.cpython.string import bytes_to_str
 from sage.repl.rich_output.output_basic import OutputBase
@@ -74,10 +75,7 @@ def example(cls):
             sage: OutputImagePng.example().png.get().startswith(b'\x89PNG')
             True
         """
-        from sage.env import SAGE_EXTCODE
-        filename = os.path.join(SAGE_EXTCODE, 'doctest', 'rich_output', 'example.png')
-        with open(filename, 'rb') as f:
-            return cls(f.read())
+        return cls(importlib.resources.read_binary(__package__, 'example.png'))
 
 
 class OutputImageGif(OutputBase):
@@ -125,10 +123,7 @@ def example(cls):
             sage: OutputImageGif.example().gif.get().startswith(b'GIF89a')
             True
         """
-        from sage.env import SAGE_EXTCODE
-        filename = os.path.join(SAGE_EXTCODE, 'doctest', 'rich_output', 'example.gif')
-        with open(filename, 'rb') as f:
-            return cls(f.read())
+        return cls(importlib.resources.read_binary(__package__, 'example.gif'))
 
     def html_fragment(self):
         """
@@ -195,10 +190,7 @@ def example(cls):
             sage: OutputImageJpg.example().jpg.get().startswith(b'\xff\xd8\xff\xe0\x00\x10JFIF')
             True
         """
-        from sage.env import SAGE_EXTCODE
-        filename = os.path.join(SAGE_EXTCODE, 'doctest', 'rich_output', 'example.jpg')
-        with open(filename, 'rb') as f:
-            return cls(f.read())
+        return cls(importlib.resources.read_binary(__package__, 'example.jpg'))
 
 
 class OutputImageSvg(OutputBase):
@@ -246,10 +238,7 @@ def example(cls):
             sage: b'</svg>' in OutputImageSvg.example().svg.get()
             True
         """
-        from sage.env import SAGE_EXTCODE
-        filename = os.path.join(SAGE_EXTCODE, 'doctest', 'rich_output', 'example.svg')
-        with open(filename, 'rb') as f:
-            return cls(f.read())
+        return cls(importlib.resources.read_binary(__package__, 'example.svg'))
 
 
 class OutputImagePdf(OutputBase):
@@ -297,10 +286,7 @@ def example(cls):
             sage: OutputImagePdf.example().pdf.get().startswith(b'%PDF-1.4')
             True
         """
-        from sage.env import SAGE_EXTCODE
-        filename = os.path.join(SAGE_EXTCODE, 'doctest', 'rich_output', 'example.pdf')
-        with open(filename, 'rb') as f:
-            return cls(f.read())
+        return cls(importlib.resources.read_binary(__package__, 'example.pdf'))
 
 
 class OutputImageDvi(OutputBase):
@@ -348,7 +334,4 @@ def example(cls):
             sage: b'TeX output' in OutputImageDvi.example().dvi.get()
             True
         """
-        from sage.env import SAGE_EXTCODE
-        filename = os.path.join(SAGE_EXTCODE, 'doctest', 'rich_output', 'example.dvi')
-        with open(filename, 'rb') as f:
-            return cls(f.read())
+        return cls(importlib.resources.read_binary(__package__, 'example.dvi'))

src/sage/ext_data/doctest/rich_output/example.canvas3d src/sage/repl/rich_output/example.canvas3d

@@ -16,6 +16,7 @@
 
 
 import os
+import importlib.resources
 
 from sage.cpython.string import bytes_to_str, FS_ENCODING
 from sage.repl.rich_output.output_basic import OutputBase
@@ -110,15 +111,8 @@ def example(cls):
             sage: rich_output.preview_png.get().startswith(b'\x89PNG')
             True
         """
-        from sage.env import SAGE_EXTCODE
-        example_png_filename = os.path.join(
-            SAGE_EXTCODE, 'doctest', 'rich_output', 'example.png')
-        with open(example_png_filename, 'rb') as f:
-            example_png = f.read()
-        scene_zip_filename = os.path.join(
-            SAGE_EXTCODE, 'doctest', 'rich_output', 'example_jmol.spt.zip')
-        with open(scene_zip_filename, 'rb') as f:
-            scene_zip = f.read()
+        example_png = importlib.resources.read_binary(__package__, 'example.png')
+        scene_zip = importlib.resources.read_binary(__package__, 'example_jmol.spt.zip')
         return cls(scene_zip, example_png)
 
 
@@ -163,10 +157,8 @@ def example(cls):
             sage: rich_output.canvas3d.get_str()
             '[{"vertices":[{"x":1,"y":1,"z":1},...{"x":1,"y":-1,"z":-1}],"faces":[[0,1,2,3]],"color":"008000"}]'
         """
-        from sage.env import SAGE_EXTCODE
-        filename = os.path.join(
-            SAGE_EXTCODE, 'doctest', 'rich_output', 'example.canvas3d')
-        return cls(OutputBuffer.from_file(filename))
+        with importlib.resources.path(__package__, 'example.canvas3d') as filename:
+            return cls(OutputBuffer.from_file(filename))
 
 
 class OutputSceneThreejs(OutputBase):
@@ -358,10 +350,8 @@ def example(cls):
             sage: rich_output.mtl.get_str()
             'newmtl texture177\nKa 0.2 0.2 0.5\nKd 0.4 0.4 1.0\nKs 0.0 0.0 0.0\nillum 1\nNs 1\nd 1\n'
         """
-        from sage.env import SAGE_EXTCODE
-        with_path = lambda x: os.path.join(
-            SAGE_EXTCODE, 'doctest', 'rich_output', 'example_wavefront', x)
-        return cls(
-            OutputBuffer.from_file(with_path('scene.obj')),
-            OutputBuffer.from_file(with_path('scene.mtl')),
-        )
+        with importlib.resources.path(__package__, 'example_wavefront_scene.obj') as filename:
+            scene_obj = OutputBuffer.from_file(filename)
+        with importlib.resources.path(__package__, 'example_wavefront_scene.mtl') as filename:
+            scene_mtl = OutputBuffer.from_file(filename)
+        return cls(scene_obj, scene_mtl)

src/sage/ext_data/doctest/rich_output/example.dvi src/sage/repl/rich_output/example.dvi

@@ -16,6 +16,7 @@
 
 
 import os
+import importlib.resources
 
 from sage.repl.rich_output.output_basic import OutputBase
 from sage.repl.rich_output.buffer import OutputBuffer
@@ -70,11 +71,9 @@ def example(cls):
             sage: OutputVideoOgg.example().mimetype
             'video/ogg'
         """
-        from sage.env import SAGE_EXTCODE
-        filename = os.path.join(SAGE_EXTCODE, 'doctest', 'rich_output',
-                                'example' + cls.ext)
-        return cls(OutputBuffer.from_file(filename),
-                   {'controls': True, 'loop': False})
+        with importlib.resources.path(__package__, 'example' + cls.ext) as filename:
+            return cls(OutputBuffer.from_file(filename),
+                       {'controls': True, 'loop': False})
 
     def html_fragment(self, url, link_attrs=''):
         r"""

src/sage/ext_data/doctest/rich_output/example.flv src/sage/repl/rich_output/example.flv

@@ -133,6 +133,9 @@ sage.interfaces =
 sage.doctest =
     tests/*
 
+sage.repl.rich_output =
+    example*
+
 sage =
     ext_data/*
     ext_data/kenzo/*
@@ -141,8 +144,6 @@ sage =
     ext_data/images/*
     ext_data/doctest/*
     ext_data/doctest/invalid/*
-    ext_data/doctest/rich_output/*
-    ext_data/doctest/rich_output/example_wavefront/*
     ext_data/gap/*
     ext_data/gap/joyner/*
     ext_data/mwrank/*