runs changed examples again (#1111)

* incomplete WIP

* move secret into step that needs it, use script instead of action

* not .py, as module

* commit a python file to test diff workflow

* remove unneeded dummy python file

* adjust event name handling

* add test event with no changed files

Author: charlesfrye

Diff for: .github/workflows/run-examples.yml

@@ -0,0 +1,70 @@
+name: Run
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "**.py"
+  push:
+    branches:
+      - main
+    paths:
+      - "**.py"
+  workflow_dispatch:
+
+# Cancel previous runs of the same PR but do not cancel previous runs on main
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+env:
+  TERM: linux
+  TERMINFO: /etc/terminfo
+  MODAL_ENVIRONMENT: examples
+
+jobs:
+  # Output all changed files in a JSON format compatible with GitHub Actions job matrices
+  diff-matrix:
+    name: Generate matrix of changed examples
+    runs-on: ubuntu-24.04
+    outputs:
+      matrix: ${{ steps.diff.outputs.all_changed_files }}
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Find changed examples
+        id: diff
+        run: python3 -m internal.generate_diff_matrix
+
+  # Run each changed example, using the output of the previous step as a job matrix
+  run-changed:
+    name: Run changed example
+    needs: [diff-matrix]
+    if:
+      ${{ needs.diff-matrix.outputs.matrix != '[]' &&
+      needs.diff-matrix.outputs.matrix != '' }}
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        file: ${{ fromJson(needs.diff-matrix.outputs.matrix) }}
+      fail-fast: false
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - uses: ./.github/actions/setup
+
+      - name: Run example
+        run: |
+          echo "Running ${{ matrix.file }}"
+          stem=$(basename "${{ matrix.file }}" .py)
+          python3 -m internal.run_example $stem || exit $?
+        env:
+          MODAL_TOKEN_ID: ${{ secrets.MODAL_MODAL_LABS_TOKEN_ID }}
+          MODAL_TOKEN_SECRET: ${{ secrets.MODAL_MODAL_LABS_TOKEN_SECRET }}

Diff for: internal/generate_diff_matrix.py

@@ -0,0 +1,86 @@
+import json
+import os
+import subprocess
+import sys
+
+
+def load_event():
+    event_path = os.environ.get("GITHUB_EVENT_PATH")
+    if not event_path:
+        print("GITHUB_EVENT_PATH not set", file=sys.stderr)
+        sys.exit(1)
+    try:
+        with open(event_path, "r") as f:
+            return json.load(f)
+    except Exception as e:
+        print(f"Error loading event JSON: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def determine_diff_range(event, event_name):
+    if event_name == "pull_request":
+        try:
+            base = event["pull_request"]["base"]["sha"]
+            head = event["pull_request"]["head"]["sha"]
+        except KeyError as e:
+            print(f"Missing key in pull_request event: {e}", file=sys.stderr)
+            sys.exit(1)
+    elif event_name == "push":
+        base = event.get("before")
+        head = event.get("after")
+    else:
+        print(f"Unsupported event type: {event_name}", file=sys.stderr)
+        sys.exit(1)
+
+    if not base or not head:
+        print("Could not determine base and head commits", file=sys.stderr)
+        sys.exit(1)
+    return base, head
+
+
+def get_changed_files(base, head):
+    try:
+        result = subprocess.run(
+            ["git", "diff", "--name-only", base, head],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        return result.stdout.splitlines()
+    except subprocess.CalledProcessError as e:
+        print(f"Error running git diff: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+def filter_files(files):
+    return [
+        f
+        for f in files
+        if f.endswith(".py")
+        and not (f.startswith("internal/") or f.startswith("misc/"))
+    ]
+
+
+def write_output(key, value):
+    github_output = os.environ.get("GITHUB_OUTPUT")
+    if github_output:
+        try:
+            with open(github_output, "a") as out:
+                out.write(f"{key}={value}\n")
+        except Exception as e:
+            print(f"Error writing to GITHUB_OUTPUT: {e}", file=sys.stderr)
+
+
+def main():
+    event = load_event()
+    event_name = event.get("event_name") or os.environ.get("GITHUB_EVENT_NAME")
+    base, head = determine_diff_range(event, event_name)
+    changed_files = get_changed_files(base, head)
+    filtered_files = filter_files(changed_files)
+    json_output = json.dumps(filtered_files)
+    write_output("all_changed_files", json_output)
+    print(json_output)
+
+
+if __name__ == "__main__":
+    main()

Diff for: internal/test-event.json

@@ -0,0 +1,18 @@
+{
+  "event_name": "pull_request",
+  "pull_request": {
+    "base": {
+      "ref": "main",
+      "sha": "3e3cba16881e73a80887c2f09477e86f0522b072"
+    },
+    "head": {
+      "ref": "charlesfrye/run-examples-again",
+      "sha": "b639aa6e806d2db555cbf4cfc29f2b93c4d50fcb"
+    }
+  },
+  "repository": {
+    "full_name": "modal-labs/modal-examples"
+  },
+  "ref": "refs/pull/1/merge"
+}
+

Diff for: internal/test_generate_diff_matrix.py

@@ -0,0 +1,127 @@
+import json
+import subprocess
+
+import generate_diff_matrix as gdm
+import pytest
+
+
+def test_determine_diff_range_push():
+    event = {"before": "commit1", "after": "commit2"}
+    base, head = gdm.determine_diff_range(event, "push")
+    assert base == "commit1"
+    assert head == "commit2"
+
+
+def test_determine_diff_range_pull():
+    event = {
+        "pull_request": {
+            "base": {"sha": "base_sha"},
+            "head": {"sha": "head_sha"},
+        }
+    }
+    base, head = gdm.determine_diff_range(event, "pull_request")
+    assert base == "base_sha"
+    assert head == "head_sha"
+
+
+def test_determine_diff_range_invalid_event():
+    event = {}
+    with pytest.raises(SystemExit):
+        gdm.determine_diff_range(event, "unsupported_event")
+
+
+def test_filter_files():
+    files = [
+        "example.py",
+        "internal/test.py",
+        "misc/skip.py",
+        "script.js",
+        "dir/another.py",
+    ]
+    filtered = gdm.filter_files(files)
+    assert filtered == ["example.py", "dir/another.py"]
+
+
+def test_get_changed_files(monkeypatch):
+    class DummyCompletedProcess:
+        def __init__(self, stdout):
+            self.stdout = stdout
+
+    def fake_run(args, capture_output, text, check):
+        return DummyCompletedProcess("file1.py\nfile2.py\n")
+
+    monkeypatch.setattr(subprocess, "run", fake_run)
+    files = gdm.get_changed_files("base", "head")
+    assert files == ["file1.py", "file2.py"]
+
+
+def test_write_output(tmp_path, monkeypatch):
+    temp_output = tmp_path / "github_output.txt"
+    monkeypatch.setenv("GITHUB_OUTPUT", str(temp_output))
+
+    gdm.write_output("test_key", "test_value")
+
+    with open(temp_output, "r") as f:
+        content = f.read()
+    assert "test_key=test_value" in content
+
+
+def test_main_push(monkeypatch, tmp_path):
+    # simulate a push event by creating a temporary event JSON
+    event_data = {"before": "commit1", "after": "commit2"}
+    event_file = tmp_path / "event.json"
+    event_file.write_text(json.dumps(event_data))
+
+    monkeypatch.setenv("GITHUB_EVENT_PATH", str(event_file))
+    monkeypatch.setenv("GITHUB_EVENT_NAME", "push")
+
+    output_file = tmp_path / "output.txt"
+    monkeypatch.setenv("GITHUB_OUTPUT", str(output_file))
+
+    # override get_changed_files to simulate a git diff call
+    def fake_get_changed_files(base, head):
+        return ["file1.py", "internal/ignore.py", "misc/skip.py", "dir/keep.py"]
+
+    monkeypatch.setattr(gdm, "get_changed_files", fake_get_changed_files)
+
+    gdm.main()
+
+    with open(output_file, "r") as f:
+        output_content = f.read().strip()
+    expected = json.dumps(["file1.py", "dir/keep.py"])
+    assert f"all_changed_files={expected}" in output_content
+
+
+def test_main_pull(monkeypatch, tmp_path):
+    # simulate a pull_request event
+    event_data = {
+        "pull_request": {
+            "base": {"sha": "base_commit"},
+            "head": {"sha": "head_commit"},
+        }
+    }
+    event_file = tmp_path / "event.json"
+    event_file.write_text(json.dumps(event_data))
+
+    monkeypatch.setenv("GITHUB_EVENT_PATH", str(event_file))
+    monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request")
+
+    output_file = tmp_path / "output.txt"
+    monkeypatch.setenv("GITHUB_OUTPUT", str(output_file))
+
+    def fake_get_changed_files(base, head):
+        return [
+            "pull_file.py",
+            "internal/not_this.py",
+            "misc/also_not.py",
+            "folder/keep_this.py",
+        ]
+
+    monkeypatch.setattr(gdm, "get_changed_files", fake_get_changed_files)
+
+    gdm.main()
+
+    with open(output_file, "r") as f:
+        output_content = f.read().strip()
+    expected = json.dumps(["pull_file.py", "folder/keep_this.py"])
+    assert f"all_changed_files={expected}" in output_content