parallel invocation of tox environments #439 (#1102)

This solution is ``subprocess`` based. We're already heavily using subprocesses, so I consider this implementation safe. 

Resolves #439.

Author: gaborbernat

docs/changelog/439.feature.rst

@@ -0,0 +1 @@
+Parallel mode added (alternative to ``detox`` which is being deprecated), for more details see :ref:`parallel_mode` - by :user:`gaborbernat`.

docs/changelog/template.jinja2

@@ -13,7 +13,8 @@
 {% if definitions[category]['showcontent'] %}
 
 {% for text, values in sections[section][category].items() %}
-- {{ text }} ({{ values|join(', ') }})
+- {{ text }}
+  {{ values|join(',\n  ') }}
 {% endfor %}
 
 {% else %}

docs/config.rst

@@ -158,6 +158,26 @@ Global settings are defined under the ``tox`` section as:
     Name of the virtual environment used to create a source distribution from the
     source tree.
 
+.. conf:: parallel_show_output ^ bool ^ false
+
+    .. versionadded:: 3.7.0
+
+    If set to True the content of the output will always be shown when running in parallel mode.
+
+.. conf:: depends ^ comma separated values
+
+    .. versionadded:: 3.7.0
+
+    tox environments this depends on. tox will try to run all dependent environments before running this
+    environment. Format is same as :conf:`envlist` (allows factor usage).
+
+    .. warning::
+
+       ``depends`` does not pull in dependencies into the run target, for example if you select ``py27,py36,coverage``
+       via the ``-e`` tox will only run those three (even if ``coverage`` may specify as ``depends`` other targets too -
+       such as ``py27, py35, py36, py37``).
+
+
 Jenkins override
 ++++++++++++++++
 

docs/example/basic.rst

@@ -381,3 +381,57 @@ meant exactly for that purpose, by setting the ``alwayscopy`` directive in your
 
     [testenv]
     alwayscopy = True
+
+.. _`parallel_mode`:
+
+Parallel mode
+-------------
+``tox`` allows running environments in parallel:
+
+- Invoke by using the ``--parallel`` or ``-p`` flag. After the packaging phase completes tox will run in parallel
+  processes tox environments (spins a new instance of the tox interpreter, but passes through all host flags and
+  environment variables).
+- ``-p`` takes an argument specifying the degree of parallelization:
+
+  - ``all`` to run all invoked environments in parallel,
+  - ``auto`` to limit it to CPU count,
+  - or pass an integer to set that limit.
+- Parallel mode displays a progress spinner while running tox environments in parallel, and reports outcome of
+  these as soon as completed with a human readable duration timing attached.
+- Parallel mode by default shows output only of failed environments and ones marked as :conf:`parallel_show_output`
+  ``=True``.
+- There's now a concept of dependency between environments (specified via :conf:`depends`), tox will re-order the
+  environment list to be run to satisfy these dependencies (in sequential run too). Furthermore, in parallel mode,
+  will only schedule a tox environment to run once all of its dependencies finished (independent of their outcome).
+
+  .. warning::
+
+    ``depends`` does not pull in dependencies into the run target, for example if you select ``py27,py36,coverage``
+    via the ``-e`` tox will only run those three (even if ``coverage`` may specify as ``depends`` other targets too -
+    such as ``py27, py35, py36, py37``).
+
+- ``--parallel-live``/``-o`` allows showing the live output of the standard output and error, also turns off reporting
+  described above.
+- Note: parallel evaluation disables standard input. Use non parallel invocation if you need standard input.
+
+Example final output:
+
+.. code-block:: bash
+
+    $ tox -e py27,py36,coverage -p all
+    ✔ OK py36 in 9.533 seconds
+    ✔ OK py27 in 9.96 seconds
+    ✔ OK coverage in 2.0 seconds
+    ___________________________ summary ______________________________________________________
+      py27: commands succeeded
+      py36: commands succeeded
+      coverage: commands succeeded
+      congratulations :)
+
+
+Example progress bar, showing a rotating spinner, the number of environments running and their list (limited up to \
+120 characters):
+
+.. code-block:: bash
+
+    ⠹ [2] py27 | py36

docs/plugins.rst

@@ -58,7 +58,6 @@ and locations of all installed plugins::
     3.0.0 imported from /home/ob/.virtualenvs/tmp/lib/python3.6/site-packages/tox/__init__.py
     registered plugins:
         tox-travis-0.10 at /home/ob/.virtualenvs/tmp/lib/python3.6/site-packages/tox_travis/hooks.py
-        detox-0.12 at /home/ob/.virtualenvs/tmp/lib/python3.6/site-packages/detox/tox_proclimit.py
 
 
 Creating a plugin

setup.py

@@ -40,6 +40,7 @@
     setup_requires=["setuptools-scm>2, <4"],  # readthedocs needs it
     extras_require={
         "testing": [
+            "freezegun >= 0.3.11",
             "pytest >= 3.0.0, <4",
             "pytest-cov >= 2.5.1, <3",
             "pytest-mock >= 1.10.0, <2",

src/tox/config.py src/tox/config/__init__.py

@@ -21,6 +21,7 @@
 import tox
 from tox.constants import INFO
 from tox.interpreters import Interpreters, NoInterpreterInfo
+from .parallel import add_parallel_flags, ENV_VAR_KEY as PARALLEL_ENV_VAR_KEY, add_parallel_config
 
 hookimpl = tox.hookimpl
 """DEPRECATED - REMOVE - this is left for compatibility with plugins importing this from here.
@@ -59,7 +60,13 @@ class Parser:
     """Command line and ini-parser control object."""
 
     def __init__(self):
-        self.argparser = argparse.ArgumentParser(description="tox options", add_help=False)
+        class HelpFormatter(argparse.ArgumentDefaultsHelpFormatter):
+            def __init__(self, prog):
+                super(HelpFormatter, self).__init__(prog, max_help_position=35, width=190)
+
+        self.argparser = argparse.ArgumentParser(
+            description="tox options", add_help=False, prog="tox", formatter_class=HelpFormatter
+        )
         self._testenv_attr = []
 
     def add_argument(self, *args, **kwargs):
@@ -274,7 +281,9 @@ def parse_cli(args, pm):
         print(get_version_info(pm))
         raise SystemExit(0)
     interpreters = Interpreters(hook=pm.hook)
-    config = Config(pluginmanager=pm, option=option, interpreters=interpreters, parser=parser)
+    config = Config(
+        pluginmanager=pm, option=option, interpreters=interpreters, parser=parser, args=args
+    )
     return config, option
 
 
@@ -413,6 +422,7 @@ def tox_addoption(parser):
         dest="sdistonly",
         help="only perform the sdist packaging activity.",
     )
+    add_parallel_flags(parser)
     parser.add_argument(
         "--parallel--safe-build",
         action="store_true",
@@ -799,6 +809,8 @@ def develop(testenv_config, value):
         help="list of extras to install with the source distribution or develop install",
     )
 
+    add_parallel_config(parser)
+
 
 def cli_skip_missing_interpreter(parser):
     class SkipMissingInterpreterAction(argparse.Action):
@@ -822,7 +834,7 @@ def __call__(self, parser, namespace, values, option_string=None):
 class Config(object):
     """Global Tox config object."""
 
-    def __init__(self, pluginmanager, option, interpreters, parser):
+    def __init__(self, pluginmanager, option, interpreters, parser, args):
         self.envconfigs = OrderedDict()
         """Mapping envname -> envconfig"""
         self.invocationcwd = py.path.local()
@@ -831,6 +843,7 @@ def __init__(self, pluginmanager, option, interpreters, parser):
         self.option = option
         self._parser = parser
         self._testenv_attr = parser._testenv_attr
+        self.args = args
 
         """option namespace containing all parsed command line options"""
 
@@ -1040,7 +1053,7 @@ def __init__(self, config, ini_path, ini_data):  # noqa
         # factors stated in config envlist
         stated_envlist = reader.getstring("envlist", replace=False)
         if stated_envlist:
-            for env in _split_env(stated_envlist):
+            for env in config.envlist:
                 known_factors.update(env.split("-"))
 
         # configure testenvs
@@ -1119,6 +1132,9 @@ def make_envconfig(self, name, section, subs, config, replace=True):
                     res = reader.getlist(env_attr.name, sep=" ")
                 elif atype == "line-list":
                     res = reader.getlist(env_attr.name, sep="\n")
+                elif atype == "env-list":
+                    res = reader.getstring(env_attr.name, replace=False)
+                    res = tuple(_split_env(res))
                 else:
                     raise ValueError("unknown type {!r}".format(atype))
                 if env_attr.postprocess:
@@ -1133,6 +1149,7 @@ def make_envconfig(self, name, section, subs, config, replace=True):
 
     def _getenvdata(self, reader, config):
         candidates = (
+            os.environ.get(PARALLEL_ENV_VAR_KEY),
             self.config.option.env,
             os.environ.get("TOXENV"),
             reader.getstring("envlist", replace=False),
@@ -1167,6 +1184,8 @@ def _getenvdata(self, reader, config):
 
 def _split_env(env):
     """if handed a list, action="append" was used for -e """
+    if env is None:
+        return []
     if not isinstance(env, list):
         env = [e.split("#", 1)[0].strip() for e in env.split("\n")]
         env = ",".join([e for e in env if e])

src/tox/config/parallel.py

@@ -0,0 +1,77 @@
+from __future__ import absolute_import, unicode_literals
+
+from argparse import ArgumentTypeError
+
+ENV_VAR_KEY = "TOX_PARALLEL_ENV"
+OFF_VALUE = 0
+DEFAULT_PARALLEL = OFF_VALUE
+
+
+def auto_detect_cpus():
+    try:
+        from os import sched_getaffinity  # python 3 only
+
+        def cpu_count():
+            return len(sched_getaffinity(0))
+
+    except ImportError:
+        # python 2 options
+        try:
+            from os import cpu_count
+        except ImportError:
+            from multiprocessing import cpu_count
+
+    try:
+        n = cpu_count()
+    except NotImplementedError:  # pragma: no cov
+        n = None  # pragma: no cov
+    return n if n else 1
+
+
+def parse_num_processes(s):
+    if s == "all":
+        return None
+    if s == "auto":
+        return auto_detect_cpus()
+    else:
+        value = int(s)
+        if value < 0:
+            raise ArgumentTypeError("value must be positive")
+        return value
+
+
+def add_parallel_flags(parser):
+    parser.add_argument(
+        "-p",
+        "--parallel",
+        dest="parallel",
+        help="run tox environments in parallel, the argument controls limit: all,"
+        " auto - cpu count, some positive number, zero is turn off",
+        action="store",
+        type=parse_num_processes,
+        default=DEFAULT_PARALLEL,
+        metavar="VAL",
+    )
+    parser.add_argument(
+        "-o",
+        "--parallel-live",
+        action="store_true",
+        dest="parallel_live",
+        help="connect to stdout while running environments",
+    )
+
+
+def add_parallel_config(parser):
+    parser.add_testenv_attribute(
+        "depends",
+        type="env-list",
+        help="tox environments that this environment depends on (must be run after those)",
+    )
+
+    parser.add_testenv_attribute(
+        "parallel_show_output",
+        type="bool",
+        default=False,
+        help="if set to True the content of the output will always be shown "
+        "when running in parallel mode",
+    )