Add support for TOX_LIMITED_SHEBANG (#2226)

* Add support for TOX_LIMITED_SHEBANG

Signed-off-by: Bernát Gábor <[email protected]>

* Fix Windows

Signed-off-by: Bernát Gábor <[email protected]>

Author: gaborbernat

docs/changelog/2208.feature.rst

@@ -0,0 +1,2 @@
+Add support for rewriting script invocations that have valid shebang lines when the ``TOX_LIMITED_SHEBANG`` environment
+variable is set and not empty - by :user:`gaborbernat`.

src/tox/execute/local_sub_process/__init__.py

@@ -13,6 +13,7 @@
 from ..api import Execute, ExecuteInstance, ExecuteOptions, ExecuteStatus
 from ..request import ExecuteRequest, StdinSource
 from ..stream import SyncWrite
+from ..util import shebang
 
 if sys.platform == "win32":  # explicit check for mypy # pragma: win32 cover
     # needs stdin/stdout handlers backed by overlapped IO
@@ -171,8 +172,12 @@ def cmd(self) -> Sequence[str]:
                     else:
                         msg = f"{base} (resolves to {executable})" if base == executable else base
                         raise Fail(f"{msg} is not allowed, use allowlist_externals to allow it")
-                # else use expanded format
-                cmd = [executable, *self.request.cmd[1:]]
+                cmd = [executable]
+                if sys.platform != "win32" and self.request.env.get("TOX_LIMITED_SHEBANG", "").strip():
+                    shebang_line = shebang(executable)
+                    if shebang_line:
+                        cmd = [*shebang_line, executable]
+                cmd.extend(self.request.cmd[1:])
             self._cmd = cmd
         return self._cmd
 

src/tox/execute/util.py

@@ -0,0 +1,30 @@
+from typing import List, Optional
+
+
+def shebang(exe: str) -> Optional[List[str]]:
+    """
+    :param exe: the executable
+    :return: the shebang interpreter arguments
+    """
+    # When invoking a command using a shebang line that exceeds the OS shebang limit (e.g. Linux has a limit of 128;
+    # BINPRM_BUF_SIZE) the invocation will fail. In this case you'd want to replace the shebang invocation with an
+    # explicit invocation.
+    # see https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/fs/binfmt_script.c#n34
+    try:
+        with open(exe, "rb") as file_handler:
+            marker = file_handler.read(2)
+            if marker != b"#!":
+                return None
+            shebang_line = file_handler.readline()
+    except OSError:
+        return None
+    try:
+        decoded = shebang_line.decode("UTF-8")
+    except UnicodeDecodeError:
+        return None
+    return [i.strip() for i in decoded.strip().split() if i.strip()]
+
+
+__all__ = [
+    "shebang",
+]

tests/execute/local_subprocess/test_execute_util.py

@@ -0,0 +1,26 @@
+from pathlib import Path
+
+from tox.execute.util import shebang
+
+
+def test_shebang_found(tmp_path: Path) -> None:
+    script_path = tmp_path / "a"
+    script_path.write_text("#!  /bin/python \t-c\t")
+    assert shebang(str(script_path)) == ["/bin/python", "-c"]
+
+
+def test_shebang_file_missing(tmp_path: Path) -> None:
+    script_path = tmp_path / "a"
+    assert shebang(str(script_path)) is None
+
+
+def test_shebang_no_shebang(tmp_path: Path) -> None:
+    script_path = tmp_path / "a"
+    script_path.write_bytes(b"magic")
+    assert shebang(str(script_path)) is None
+
+
+def test_shebang_non_utf8_file(tmp_path: Path) -> None:
+    script_path, content = tmp_path / "a", b"#!" + bytearray.fromhex("c0")
+    script_path.write_bytes(content)
+    assert shebang(str(script_path)) is None

tests/execute/local_subprocess/test_local_subprocess.py

@@ -1,12 +1,14 @@
 import json
 import logging
 import os
+import shutil
+import stat
 import subprocess
 import sys
 from io import TextIOWrapper
 from pathlib import Path
 from typing import Dict, List, Tuple
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, create_autospec
 
 import psutil
 import pytest
@@ -15,7 +17,7 @@
 from psutil import AccessDenied
 from pytest_mock import MockerFixture
 
-from tox.execute.api import Outcome
+from tox.execute.api import ExecuteOptions, Outcome
 from tox.execute.local_sub_process import SIG_INTERRUPT, LocalSubProcessExecuteInstance, LocalSubProcessExecutor
 from tox.execute.request import ExecuteRequest, StdinSource
 from tox.execute.stream import SyncWrite
@@ -298,3 +300,36 @@ def test_allow_list_external_ok(fake_exe_on_path: Path, mode: str) -> None:
     inst = LocalSubProcessExecuteInstance(request, MagicMock(), out=SyncWrite("out", None), err=SyncWrite("err", None))
 
     assert inst.cmd == [exe]
+
+
+def test_shebang_limited_on(tmp_path: Path) -> None:
+    exe, script, instance = _create_shebang_test(tmp_path, env={"TOX_LIMITED_SHEBANG": "1"})
+    if sys.platform == "win32":  # pragma: win32 cover
+        assert instance.cmd == [str(script), "--magic"]
+    else:
+        assert instance.cmd == [exe, "-s", str(script), "--magic"]
+
+
+@pytest.mark.parametrize("env", [{}, {"TOX_LIMITED_SHEBANG": ""}])
+def test_shebang_limited_off(tmp_path: Path, env: Dict[str, str]) -> None:
+    _, script, instance = _create_shebang_test(tmp_path, env=env)
+    assert instance.cmd == [str(script), "--magic"]
+
+
+def test_shebang_failed_to_parse(tmp_path: Path) -> None:
+    _, script, instance = _create_shebang_test(tmp_path, env={"TOX_LIMITED_SHEBANG": "yes"})
+    script.write_text("")
+    assert instance.cmd == [str(script), "--magic"]
+
+
+def _create_shebang_test(tmp_path: Path, env: Dict[str, str]) -> Tuple[str, Path, LocalSubProcessExecuteInstance]:
+    exe = shutil.which("python")
+    assert exe is not None
+    script = tmp_path / f"s{'.EXE' if sys.platform == 'win32' else ''}"
+    script.write_text(f"#!{exe} -s")
+    script.chmod(script.stat().st_mode | stat.S_IEXEC)  # mark it executable
+    env["PATH"] = str(script.parent)
+    request = create_autospec(ExecuteRequest, cmd=["s", "--magic"], env=env, allow=None)
+    writer = create_autospec(SyncWrite)
+    instance = LocalSubProcessExecuteInstance(request, create_autospec(ExecuteOptions), writer, writer)
+    return exe, script, instance

whitelist.txt

@@ -13,6 +13,8 @@ autoclass
 autodoc
 autosectionlabel
 autouse
+binprm
+buf
 bufsize
 byref
 cachetools
@@ -81,6 +83,7 @@ fixup
 fmt
 formatter
 fromdocname
+fromhex
 fromkeys
 fs
 fullmatch
@@ -101,6 +104,7 @@ hookimpl
 hookspec
 hookspecs
 ident
+iexec
 ign
 ignorecase
 impl