diff --git a/src/bin/sage b/src/bin/sage index 64d7986482b..ec6c356f994 100755 --- a/src/bin/sage +++ b/src/bin/sage @@ -482,10 +482,13 @@ usage_advanced() { echo " --fixdoctests file.py" echo " -- Run doctests and replace output of failing doctests" echo " with actual output." - echo " --fiximports " + echo " --fiximports " echo " -- Replace imports from sage.PAC.KAGE.all by specific" echo " imports when sage.PAC.KAGE is an implicit namespace" echo " package" + echo " --fixdistributions " + echo " -- Check or update '# sage_setup: distribution'" + echo " directives in source files" fi echo " --sh [...] -- run a shell with Sage environment variables" echo " as they are set in the runtime of Sage" @@ -986,6 +989,11 @@ if [ "$1" = '-fiximports' -o "$1" = '--fiximports' ]; then exec sage-python -m sage.misc.replace_dot_all "$@" fi +if [ "$1" = '-fixdistributions' -o "$1" = '--fixdistributions' ]; then + shift + exec sage-python -m sage.misc.package_dir "$@" +fi + if [ "$1" = '-tox' -o "$1" = '--tox' ]; then shift if [ -n "$SAGE_SRC" -a -f "$SAGE_SRC/tox.ini" ]; then diff --git a/src/doc/en/developer/packaging_sage_library.rst b/src/doc/en/developer/packaging_sage_library.rst index 0244d96e5ac..576356a9dff 100644 --- a/src/doc/en/developer/packaging_sage_library.rst +++ b/src/doc/en/developer/packaging_sage_library.rst @@ -146,6 +146,34 @@ The source directory of a distribution package, such as controls which files and directories of the monolithic Sage library source tree are included in the distribution + The manifest should be kept in sync with the directives of the form + ``# sage_setup: distribution = sagemath-polyhedra`` at the top of + source files. Sage provides a tool ``sage --fixdistributions`` + that assists with this task. For example:: + + $ ./sage --fixdistributions --set sagemath-polyhedra \ + src/sage/geometry/polyhedron/base*.py + + adds or updates the directives in the specified files; and:: + + $ ./sage --fixdistributions --add sagemath-polyhedra \ + src/sage/geometry/polyhedron + + adds the directive to all files in the given directory that do not + include a directive yet. + + After a distribution has been built (for example, by the command + ``make pypi-wheels``) or at least an sdist has been built (for + example, by the command ``make sagemath_polyhedra-sdist``), the + distribution directives in all files in the source distribution + can be updated using the switch ``--from--egg-info``:: + + $ ./sage --fixdistributions --set sagemath-polyhedra --from-egg-info + + To take care of all distributions, use:: + + $ ./sage --fixdistributions --set all --from-egg-info + - `pyproject.toml `_, `setup.cfg `_, and `requirements.txt `_ -- diff --git a/src/sage/misc/package_dir.py b/src/sage/misc/package_dir.py index 9f075b16c31..3749a4f2a91 100644 --- a/src/sage/misc/package_dir.py +++ b/src/sage/misc/package_dir.py @@ -14,7 +14,10 @@ import os import glob +import re import sys + +from collections import defaultdict from contextlib import contextmanager @@ -26,12 +29,12 @@ class SourceDistributionFilter: - ``include_distributions`` -- (default: ``None``) if not ``None``, should be a sequence or set of strings: include files whose - ``distribution`` (from a ``# sage_setup: distribution = PACKAGE`` + ``distribution`` (from a ``# sage_setup:`` ``distribution = PACKAGE`` directive in the source file) is an element of ``distributions``. - ``exclude_distributions`` -- (default: ``None``) if not ``None``, should be a sequence or set of strings: exclude files whose - ``distribution`` (from a ``# sage_setup: distribution = PACKAGE`` + ``distribution`` (from a ``# sage_setup:`` ``distribution = PACKAGE`` directive in the module source file) is in ``exclude_distributions``. EXAMPLES:: @@ -90,9 +93,12 @@ def __contains__(self, filename): return distribution not in self._exclude_distributions +distribution_directive = re.compile(r"(\s*#?\s*)(sage_setup:\s*distribution\s*=\s*([-_A-Za-z0-9]*))") + + def read_distribution(src_file): - """ - Parse ``src_file`` for a ``# sage_setup: distribution = PKG`` directive. + r""" + Parse ``src_file`` for a ``# sage_setup:`` ``distribution = PKG`` directive. INPUT: @@ -119,9 +125,12 @@ def read_distribution(src_file): line = line.lstrip() if not line: continue - if line[0] != '#': + if line.startswith('#') or line.startswith(';'): + line = line[1:].lstrip() + elif line.startswith('/*') or line.startswith('//') or line.startswith(';;'): + line = line[2:].lstrip() + else: break - line = line[1:].lstrip() kind = "sage_setup:" if line.startswith(kind): key, _, value = (s.strip() for s in line[len(kind):].partition('=')) @@ -130,6 +139,105 @@ def read_distribution(src_file): return '' +def update_distribution(src_file, distribution, *, verbose=False): + r""" + Add or update a ``# sage_setup:`` ``distribution = PKG`` directive in ``src_file``. + + For a Python or Cython file, if a ``distribution`` directive + is not already present, it is added. + + For any other file, if a ``distribution`` directive is not already + present, no action is taken. + + INPUT: + + - ``src_file`` -- file name of a source file + + EXAMPLES:: + + sage: from sage.misc.package_dir import read_distribution, update_distribution + sage: import tempfile + sage: def test(filename, file_contents): + ....: with tempfile.TemporaryDirectory() as d: + ....: fname = os.path.join(d, filename) + ....: with open(fname, 'w') as f: + ....: f.write(file_contents) + ....: with open(fname, 'r') as f: + ....: print(f.read() + "====") + ....: update_distribution(fname, 'sagemath-categories') + ....: with open(fname, 'r') as f: + ....: print(f.read() + "====") + ....: update_distribution(fname, '') + ....: with open(fname, 'r') as f: + ....: print(f.read(), end="") + sage: test('module.py', '# Python file\n') + # Python file + ==== + # sage_setup: distribution...= sagemath-categories + # Python file + ==== + # sage_setup: distribution...= + # Python file + sage: test('file.cpp', '// sage_setup: ' 'distribution=sagemath-modules\n' + ....: '// C++ file with existing directive\n') + // sage_setup: distribution...=sagemath-modules + // C++ file with existing directive + ==== + // sage_setup: distribution...= sagemath-categories + // C++ file with existing directive + ==== + // sage_setup: distribution...= + // C++ file with existing directive + sage: test('file.cpp', '// C++ file without existing directive\n') + // C++ file without existing directive + ==== + // C++ file without existing directive + ==== + // C++ file without existing directive + """ + if not distribution: + distribution = '' + directive = 'sage_setup: ' f'distribution = {distribution}'.rstrip() + try: + with open(src_file, 'r') as f: + src_lines = f.read().splitlines() + except UnicodeDecodeError: + # Silently skip binary files + return + any_found = False + any_change = False + for i, line in enumerate(src_lines): + if m := distribution_directive.search(line): + old_distribution = m.group(3) + if any_found: + # Found a second distribution directive; remove it. + if not (line := distribution_directive.sub(r'', line)): + line = None + else: + line = distribution_directive.sub(fr'\1{directive}', line) + if line != src_lines[i]: + src_lines[i] = line + any_change = True + if verbose: + print(f"{src_file}: changed 'sage_setup: " f"distribution' " + f"from {old_distribution!r} to {distribution!r}") + any_found = True + if not any_found: + if any(src_file.endswith(ext) + for ext in [".pxd", ".pxi", ".py", ".pyx", ".sage"]): + src_lines.insert(0, f'# {directive}') + any_change = True + if verbose: + print(f"{src_file}: added 'sage_setup: " + f"distribution = {distribution}' directive") + if not any_change: + return + with open(src_file, 'w') as f: + for line in src_lines: + if line is not None: + f.write(line + '\n') + + def is_package_or_sage_namespace_package_dir(path, *, distribution_filter=None): r""" Return whether ``path`` is a directory that contains a Python package. @@ -147,7 +255,7 @@ def is_package_or_sage_namespace_package_dir(path, *, distribution_filter=None): - ``distribution_filter`` -- (optional, default: ``None``) only consider ``all*.py`` files whose distribution (from a - ``# sage_setup: distribution = PACKAGE`` directive in the source file) + ``# sage_setup:`` ``distribution = PACKAGE`` directive in the source file) is an element of ``distribution_filter``. EXAMPLES: @@ -333,3 +441,181 @@ def seen(p, m={}): path = [p for p in path if not seen(p)] yield from walk_packages(path, info.name + '.', onerror) + + +def _all_filename(distribution): + if not distribution: + return 'all.py' + return f"all__{distribution.replace('-', '_')}.py" + + +def _distribution_from_all_filename(filename): + if m := re.match('all(__(.*?))?[.]py', filename): + if distribution_per_all_filename := m.group(2): + return distribution_per_all_filename.replace('_', '-') + return '' + return False + + +if __name__ == '__main__': + + from argparse import ArgumentParser + + parser = ArgumentParser(prog="sage --fixdistributions", + description="Maintenance tool for distribution packages of the Sage library", + epilog="By default, '%(prog)s' shows the distribution of each file.") + parser.add_argument('--add', metavar='DISTRIBUTION', type=str, default=None, + help=("add a 'sage_setup: DISTRIBUTION' directive to FILES; " + "do not change files that already have a nonempty directive")) + parser.add_argument('--set', metavar='DISTRIBUTION', type=str, default=None, + help="add or update the 'sage_setup: DISTRIBUTION' directive in FILES") + parser.add_argument('--from-egg-info', action="store_true", default=False, + help="take FILES from pkgs/DISTRIBUTION/DISTRIBUTION.egg-info/SOURCES.txt") + parser.add_argument("filename", metavar='FILES', nargs='*', type=str, + help=("source files or directories (default: all files from SAGE_SRC, " + "unless --from-egg-info, --add, or --set are used)")) + + args = parser.parse_args() + + distribution = args.set or args.add or '' + + if distribution == 'all': + distributions = ["sagemath-bliss", + "sagemath-coxeter3", + "sagemath-mcqd", + "sagemath-meataxe", + "sagemath-sirocco", + "sagemath-tdlib", + "sagemath-environment", + "sagemath-categories", + "sagemath-repl", + "sagemath-objects"] + else: + distributions = [distribution.replace('_', '-')] + + if args.from_egg_info: + if not distribution: + print("Switch '--from-egg-info' must be used with either " + "'--add DISTRIBUTION' or '--set DISTRIBUTION'") + sys.exit(1) + elif not args.filename: + if distribution: + print("Switches '--add' and '--set' require the switch '--from-egg-info' " + "or one or more file or directory names") + sys.exit(1) + from sage.env import SAGE_SRC + if (not SAGE_SRC + or not os.path.exists(os.path.join(SAGE_SRC, 'sage')) + or not os.path.exists(os.path.join(SAGE_SRC, 'conftest_test.py'))): + print(f'{SAGE_SRC=} does not seem to contain a copy of the Sage source tree') + sys.exit(1) + args.filename = [os.path.join(SAGE_SRC, 'sage')] + + ordinary_packages = set() + package_distributions_per_directives = defaultdict(set) # path -> set of strings (distributions) + package_distributions_per_all_files = defaultdict(set) # path -> set of strings (distributions) + + def handle_file(root, file): + path = os.path.join(root, file) + if args.set is not None: + update_distribution(path, distribution, verbose=True) + file_distribution = distribution + elif args.add is not None: + if not (file_distribution := read_distribution(path)): + update_distribution(path, distribution, verbose=True) + file_distribution = distribution + else: + file_distribution = read_distribution(path) + print(f'{path}: file in distribution {file_distribution!r}') + package_distributions_per_directives[root].add(file_distribution) + if file.startswith('__init__.'): + ordinary_packages.add(root) + elif (distribution_per_all_filename := _distribution_from_all_filename(file)) is False: + # Not an all*.py file. + pass + elif not distribution_per_all_filename: + # An all.py file. + if file_distribution: + # The all.py is declared to belong to a named distribution, that's OK + package_distributions_per_all_files[root].add(file_distribution) + else: + pass + else: + # An all__*.py file + if distribution_per_all_filename != file_distribution: + print(f'{path}: file should go in distribution {distribution_per_all_filename!r}, not {file_distribution!r}') + package_distributions_per_all_files[root].add(distribution_per_all_filename) + + for distribution in distributions: + + paths = list(args.filename) + + if args.from_egg_info: + from sage.env import SAGE_ROOT + if not distribution: + print("Switch '--from-egg-info' must be used with either " + "'--add DISTRIBUTION' or '--set DISTRIBUTION'") + sys.exit(1) + if not SAGE_ROOT: + print(f'{SAGE_ROOT=} does not seem to contain a copy of the Sage source root') + sys.exit(1) + distribution_dir = os.path.join(SAGE_ROOT, 'pkgs', distribution) + if not os.path.exists(distribution_dir): + print(f'{distribution_dir} does not exist') + sys.exit(1) + distribution_underscore = distribution.replace('-', '_') + try: + with open(os.path.join(distribution_dir, + f'{distribution_underscore}.egg-info', 'SOURCES.txt'), "r") as f: + paths.extend(os.path.join(SAGE_ROOT, 'src', line.strip()) + for line in f + if line.startswith('sage/')) + print(f"sage --fixdistributions: found egg-info of distribution {distribution!r}") + except FileNotFoundError: + if len(distributions) > 1: + print(f"sage --fixdistributions: distribution {distribution!r} does not have egg-info, skipping it; " + f"run 'make {distribution_underscore}-sdist' or 'make {distribution_underscore}' to create it") + continue + else: + print(f"sage --fixdistributions: distribution {distribution!r} does not have egg-info; " + f"run 'make {distribution_underscore}-sdist' or 'make {distribution_underscore}' to create it") + sys.exit(1) + + for path in paths: + path = os.path.relpath(path) + if os.path.isdir(path): + if not is_package_or_sage_namespace_package_dir(path): + print(f'{path}: non-package directory') + else: + for root, dirs, files in os.walk(path): + for dir in sorted(dirs): + path = os.path.join(root, dir) + if any(dir.startswith(prefix) for prefix in ['.', 'build', 'dist', '__pycache__', '_vendor', '.tox']): + # Silently skip + dirs.remove(dir) + elif not is_package_or_sage_namespace_package_dir(path): + print(f'{path}: non-package directory') + dirs.remove(dir) + for file in sorted(files): + if any(file.endswith(ext) for ext in [".pyc", ".pyo", ".bak", ".so", "~"]): + continue + handle_file(root, file) + else: + handle_file(*os.path.split(path)) + + print(f"sage --fixdistributions: checking consistency") + + for package in ordinary_packages: + if len(package_distributions_per_directives[package]) > 1: + print(f'{package}: ordinary packages (with __init__.py) cannot be split in several distributions (' + + ', '.join(f'{dist!r}' + for dist in sorted(package_distributions_per_directives[package])) + ')') + + for package, distributions_per_directives in package_distributions_per_directives.items(): + if package in ordinary_packages: + pass + elif ((missing_all_files := distributions_per_directives - package_distributions_per_all_files[package]) + and not(missing_all_files == set(['']) and len(distributions_per_directives) < 2)): + s = '' if len(missing_all_files) == 1 else 's' + print(f'{package}: missing file{s} ' + ', '.join(_all_filename(distribution) + for distribution in missing_all_files))