sagemathgh-36964: Add `# sage_setup: distribution` directives to all files, remove remaining `# coding: utf-8`
    
<!-- ^^^^^
Please provide a concise, informative and self-explanatory title.
Don't put issue numbers in there, do this in the PR body below.
For example, instead of "Fixes sagemath#1234" use "Introduce new method to
calculate 1+1"
-->
<!-- Describe your changes here in detail -->

These directives at the top of the file inform developers about the
intended assignment of modules to pip-installable distributions.

As of this PR, there should be no change to the existing distributions
(sagemath-categories...) nor the monolithic build of the Sage library.

<!-- Why is this change required? What problem does it solve? -->
- Cherry-picked from sagemath#35095
<!-- If this PR resolves an open issue, please link to it here. For
example "Fixes sagemath#12345". -->
<!-- If your change requires a documentation PR, please link it
appropriately. -->

### 📝 Checklist

<!-- Put an `x` in all the boxes that apply. -->
<!-- If your change requires a documentation PR, please link it
appropriately -->
<!-- If you're unsure about any of these, don't hesitate to ask. We're
here to help! -->
<!-- Feel free to remove irrelevant items. -->

- [x] The title is concise, informative, and self-explanatory.
- [ ] The description explains in detail what this PR is about.
- [ ] I have linked a relevant issue or discussion.
- [ ] I have created tests covering the changes.
- [ ] I have updated the documentation accordingly.

### ⌛ Dependencies

<!-- List all open PRs that this PR logically depends on
- sagemath#12345: short description why this is a dependency
- sagemath#34567: ...
-->

<!-- If you're unsure about any of these, don't hesitate to ask. We're
here to help! -->
- Depends on sagemath#36951 (merged here)
- Depends on sagemath#36676 (merged here)
- Depends on sagemath#37667 (merged here)
    
URL: sagemath#36964
Reported by: Matthias Köppe
Reviewer(s): Kwankyu Lee, Matthias Köppe, Tobias Diez

Author: Release Manager

bootstrap

@@ -232,8 +232,7 @@ save () {
 	src/environment-optional-3.[89].yml src/environment-optional-3.1[0-9].yml \
 	src/Pipfile \
 	src/pyproject.toml \
-	src/requirements.txt \
-	src/setup.cfg
+	src/requirements.txt
 
     # Update version
     echo "$NEWCONFVERSION" >$PKG/package-version.txt

build/bin/write-dockerfile.sh

@@ -236,7 +236,7 @@ $ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap bootstrap-
 $ADD config/config.rpath config/config.rpath
 $ADD src/doc/bootstrap src/doc/bootstrap
 $ADD src/bin src/bin
-$ADD src/Pipfile.m4 src/pyproject.toml.m4 src/requirements.txt.m4 src/setup.cfg.m4 src/VERSION.txt src/
+$ADD src/Pipfile.m4 src/pyproject.toml.m4 src/requirements.txt.m4 src/VERSION.txt src/
 $ADD m4 ./m4
 $ADD pkgs pkgs
 $ADD build ./build

build/pkgs/typing_extensions/version_requirements.txt

@@ -1,3 +1,3 @@
 # According to https://github.com/python/typing_extensions/blob/main/CHANGELOG.md,
 # version 4.4.0 adds another Python 3.11 typing backport
-typing_extensions >= 4.4.0; python_version<'3.11'
+typing_extensions >= 4.4.0; python_version<"3.11"

pkgs/sagemath-categories/known-test-failures.json

@@ -26,7 +26,7 @@
         "ntests": 28
     },
     "sage.categories.affine_weyl_groups": {
-        "ntests": 14
+        "ntests": 15
     },
     "sage.categories.algebra_ideals": {
         "failed": true,
@@ -148,7 +148,7 @@
     },
     "sage.categories.coxeter_groups": {
         "failed": true,
-        "ntests": 362
+        "ntests": 363
     },
     "sage.categories.cw_complexes": {
         "failed": true,
@@ -301,7 +301,7 @@
     },
     "sage.categories.finite_complex_reflection_groups": {
         "failed": true,
-        "ntests": 178
+        "ntests": 199
     },
     "sage.categories.finite_coxeter_groups": {
         "ntests": 6
@@ -493,10 +493,6 @@
         "failed": true,
         "ntests": 13
     },
-    "sage.categories.inner_product_spaces": {
-        "failed": true,
-        "ntests": 9
-    },
     "sage.categories.integral_domains": {
         "failed": true,
         "ntests": 22
@@ -783,7 +779,7 @@
     },
     "sage.cpython.debug": {
         "failed": true,
-        "ntests": 14
+        "ntests": 13
     },
     "sage.cpython.dict_del_by_value": {
         "failed": true,
@@ -826,7 +822,6 @@
         "ntests": 321
     },
     "sage.doctest.reporting": {
-        "failed": true,
         "ntests": 124
     },
     "sage.doctest.sources": {
@@ -1091,7 +1086,7 @@
     },
     "sage.misc.sageinspect": {
         "failed": true,
-        "ntests": 329
+        "ntests": 332
     },
     "sage.misc.superseded": {
         "failed": true,
@@ -1124,6 +1119,7 @@
         "ntests": 128
     },
     "sage.repl.configuration": {
+        "failed": true,
         "ntests": 22
     },
     "sage.repl.display.fancy_repr": {
@@ -1149,6 +1145,7 @@
         "ntests": 42
     },
     "sage.repl.inputhook": {
+        "failed": true,
         "ntests": 7
     },
     "sage.repl.interface_magic": {
@@ -1157,7 +1154,7 @@
     },
     "sage.repl.interpreter": {
         "failed": true,
-        "ntests": 118
+        "ntests": 114
     },
     "sage.repl.ipython_extension": {
         "failed": true,
@@ -1188,7 +1185,7 @@
     },
     "sage.repl.load": {
         "failed": true,
-        "ntests": 42
+        "ntests": 38
     },
     "sage.repl.preparse": {
         "failed": true,
@@ -1315,9 +1312,6 @@
     "sage.structure.global_options": {
         "ntests": 145
     },
-    "sage.structure.graphics_file": {
-        "ntests": 8
-    },
     "sage.structure.indexed_generators": {
         "failed": true,
         "ntests": 90

pkgs/sagemath-standard/setup.cfg

@@ -69,25 +69,60 @@
     from sage_setup.autogen import autogen_all
     autogen_all()
 
-# TODO: This should be quiet by default
-print("Discovering Python/Cython source code....")
-t = time.time()
-from sage.misc.package import is_package_installed_and_updated
-distributions = ['']
-optional_packages_with_extensions = os.environ.get('SAGE_OPTIONAL_PACKAGES_WITH_EXTENSIONS', '').split(',')
-distributions += ['sagemath-{}'.format(pkg)
-                  for pkg in optional_packages_with_extensions
-                  if is_package_installed_and_updated(pkg)]
-log.warn('distributions = {0}'.format(distributions))
-from sage_setup.find import find_python_sources
-python_packages, python_modules, cython_modules = find_python_sources(
-    SAGE_SRC, ['sage'], distributions=distributions)
-
-log.debug('python_packages = {0}'.format(python_packages))
-log.debug('python_modules = {0}'.format(python_modules))
-log.debug('cython_modules = {0}'.format(cython_modules))
-
-print("Discovered Python/Cython sources, time: %.2f seconds." % (time.time() - t))
+    # TODO: This should be quiet by default
+    print("Discovering Python/Cython source code....")
+    t = time.time()
+    from sage.misc.package import is_package_installed_and_updated
+
+    distributions = [
+        '',
+        "sagemath-brial",
+        "sagemath-categories",
+        "sagemath-combinat",
+        "sagemath-eclib",
+        "sagemath-environment",
+        "sagemath-flint",
+        "sagemath-gap",
+        "sagemath-giac",
+        "sagemath-glpk",
+        "sagemath-graphs",
+        "sagemath-groups",
+        "sagemath-homfly",
+        "sagemath-lcalc",
+        "sagemath-libbraiding",
+        "sagemath-libecm",
+        "sagemath-linbox",
+        "sagemath-modules",
+        "sagemath-mpmath",
+        "sagemath-ntl",
+        "sagemath-objects",
+        "sagemath-pari",
+        "sagemath-plot",
+        "sagemath-polyhedra",
+        "sagemath-repl",
+        "sagemath-schemes",
+        "sagemath-singular",
+        "sagemath-symbolics",
+    ]
+
+    optional_packages_with_extensions = os.environ.get('SAGE_OPTIONAL_PACKAGES_WITH_EXTENSIONS', '').split(',')
+    distributions += ['sagemath-{}'.format(pkg)
+                      for pkg in optional_packages_with_extensions
+                      if is_package_installed_and_updated(pkg)]
+    log.warn('distributions = {0}'.format(distributions))
+    from sage_setup.find import find_python_sources
+    python_packages, python_modules, cython_modules = find_python_sources(
+        SAGE_SRC, ['sage'], distributions=distributions)
+
+    log.debug('python_packages = {0}'.format(python_packages))
+    log.debug('python_modules = {0}'.format(python_modules))
+    log.debug('cython_modules = {0}'.format(cython_modules))
+
+    print("Discovered Python/Cython sources, time: %.2f seconds." % (time.time() - t))
+else:
+    # sdist, egg_info, dist_info
+    python_packages = []
+    cython_modules = []
 
 #########################################################
 ### Distutils

pkgs/sagemath-standard/setup.cfg.m4

@@ -17,7 +17,7 @@
 [tox]
 envlist =
     # Build and test without using the concrete dependencies specified by requirements.txt,
-    # using the dependencies declared in pyproject.toml and setup.cfg (install-requires) only:
+    # using the dependencies declared in pyproject.toml (build-system requires, dependencies) only:
     # Still use ONLY the wheels built and stored by the Sage distribution (no PyPI).
     #
     #   ./sage -sh -c '(cd pkgs/sagemath-standard && tox -v -v -v -e sagepython-sagewheels-nopypi-norequirements)'
@@ -45,7 +45,7 @@ envlist =
     #
     #   ./sage -sh -c '(cd pkgs/sagemath-standard && tox -v -v -v -e sagepython-sagewheels-pipenv-dist)'
     #
-    # Build using the dependencies declared in pyproject.toml and setup.cfg (install-requires) only.
+    # Build using the dependencies declared in pyproject.toml (build-system requires, dependencies) only:
     # Use the wheels built and stored by the Sage distribution,
     # and additionally allow packages from PyPI.
     #

pkgs/sagemath-standard/setup.py

@@ -6,17 +6,17 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 
 [dev-packages]
-## We do not list packages that are already declared as install_requires
-## in setup.cfg
+## We do not list packages that are already declared as dependencies (install_requires)
+## in pyproject.toml
 pycodestyle = "*"
 tox = "*"
 pytest = "*"
 rope = "*"
 six = "*"
 
 [packages]
-## We do not list packages that are already declared as install_requires
-## in setup.cfg
+## We do not list packages that are already declared as dependencies (install_requires)
+## in pyproject.toml
 
 [packages.e1839a8]
 path = "."

pkgs/sagemath-standard/tox.ini

@@ -193,8 +193,8 @@ included in one of the following places:
   This practice is deprecated, see :issue:`33037`.
 
 In all cases, the files must be listed (explicitly or via wildcards) in
-the section ``options.package_data`` of the file
-:sage_root:`pkgs/sagemath-standard/setup.cfg.m4` (or the corresponding
+the section ``[tool.setuptools.package-data]`` of the file
+:sage_root:`SAGE_ROOT/pkgs/sagemath-standard/pyproject.toml.m4` (or the corresponding
 file of another distribution).
 
 Large data files should not be added to the Sage source tree. Instead, it

src/Pipfile.m4

@@ -14,8 +14,9 @@ Python language standard
 
 Sage library code needs to be compatible with all versions of Python
 that Sage supports.  The information regarding the supported versions
-can be found in the files ``build/pkgs/python3/spkg-configure.m4`` and
-``src/setup.cfg.m4``.
+can be found in the files ``build/pkgs/python3/spkg-configure.m4``
+(for the Sage distribution), ``m4/pyproject_toml_metadata.m4`` (for
+the Sage library and most other distribution packages in ``pkgs/``).
 
 Python 3.9 is the oldest supported version.  Hence,
 all language and library features that are available in Python 3.9 can

src/doc/en/developer/coding_basics.rst

@@ -174,8 +174,7 @@ The source directory of a distribution package, such as
 
     $ ./sage --fixdistributions --set all --from-egg-info
 
-- `pyproject.toml <https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/>`_,
-  `setup.cfg <https://setuptools.pypa.io/en/latest/userguide/declarative_config.html>`_,
+- `pyproject.toml <https://pip.pypa.io/en/stable/reference/build-system/pyproject-toml/>`_
   and `requirements.txt <https://pip.pypa.io/en/stable/user_guide/#requirements-files>`_ --
   standard Python packaging metadata, declaring the distribution name, dependencies,
   etc.
@@ -288,8 +287,9 @@ modules must be part of the distribution, or provided by another
 distribution -- which then must be declared as a run-time dependency.
 
 *Declaring run-time dependencies:* These dependencies are declared in
-``setup.cfg`` (generated from ``setup.cfg.m4``) as
-`install_requires <https://setuptools.pypa.io/en/latest/userguide/dependency_management.html#declaring-required-dependency>`_.
+``pyproject.toml`` (generated from ``pyproject.toml.m4``) as
+`[project] dependencies <https://setuptools.pypa.io/en/latest/userguide/dependency_management.html#declaring-required-dependency>`_ (in the older terminology of ``setup.cfg`` and ``setup.py``,
+these dependencies were known as ``install_requires``).
 
 *Reducing module-level run-time dependencies:*
 
@@ -467,14 +467,17 @@ features, which will only be working if the user also has installed
 **sagemath-symbolics**.
 
 *Declaring optional run-time dependencies:* It is possible to declare
-such optional dependencies as `extras_require <https://setuptools.pypa.io/en/latest/userguide/dependency_management.html#optional-dependencies>`_ in ``setup.cfg``
-(generated from ``setup.cfg.m4``).  This is a very limited mechanism
+such dependencies as `[project.optional-dependencies] <https://setuptools.pypa.io/en/latest/userguide/dependency_management.html#optional-dependencies>`_ in ``pyproject.toml``
+(generated from ``pyproject.toml.m4``).
+(In the older terminology of ``setup.cfg`` and ``setup.py``,
+these optional dependencies were known as ``extras_require``.)
+This is a very limited mechanism
 -- in particular it does not affect the build phase of the
 distribution in any way. It basically only provides a way to give a
 nickname to a distribution that can be installed as an add-on.
 
-In our example, we could declare an ``extras_require`` so that users
-could use ``pip install sagemath-coding[symbolics]``.
+In our example, we could declare an optional dependency so that users
+could use ``pip install "sagemath-coding[symbolics]"``.
 
 
 Doctest-only dependencies
@@ -516,7 +519,7 @@ The version information for dependencies comes from the files
 ``build/pkgs/*/package-version.txt``.  We use the
 `m4 <https://www.gnu.org/software/m4/manual/html_node/index.html>`_
 macro processor to insert the version information in the generated files
-``pyproject.toml``, ``setup.cfg``, ``requirements.txt``.
+``pyproject.toml`` and ``requirements.txt``.
 
 
 Hierarchy of distribution packages
@@ -546,8 +549,8 @@ Hierarchy of distribution packages
     sphinx_plot(g, figsize=(8, 4), axes=False)
 
 
-Solid arrows indicate ``install_requires``, i.e., a declared runtime dependency.
-Dashed arrows indicate ``extras_require``, i.e., a declared optional runtime dependency.
+Solid arrows indicate declared runtime dependencies (``install_requires``).
+Dashed arrows indicate declared optional runtime dependencies (``extras_require``).
 Not shown in the diagram are build dependencies and optional dependencies for testing.
 
 - `sage_conf <https://pypi.org/project/sage-conf/>`_ is a configuration

src/doc/en/developer/coding_in_python.rst

@@ -23,6 +23,7 @@ Using calculus
 
 - :doc:`More about symbolic variables and functions <sage/calculus/var>`
 - :doc:`Main operations on symbolic expressions <sage/symbolic/expression>`
+- :doc:`sage/calculus/expr`
 - :doc:`Assumptions about symbols and functions <sage/symbolic/assumptions>`
 - :doc:`sage/symbolic/relation`
 - :doc:`sage/symbolic/integration/integral`
@@ -65,6 +66,7 @@ Internal functionality supporting calculus
 
    sage/symbolic/expression
    sage/symbolic/callable
+   sage/calculus/expr
    sage/symbolic/assumptions
    sage/symbolic/relation
    sage/calculus/calculus