From d41e8379cad5e3b10aa924db1c72d84f62b0e843 Mon Sep 17 00:00:00 2001
From: Arpad Borsos <arpad.borsos@sentry.io>
Date: Tue, 18 Mar 2025 11:25:00 +0100
Subject: [PATCH] Remove usage of codecov ATS feature
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
We are in the process of deprecating the ATS feature of codecov, and I saw you are one of the few (open source) users of it.
This PR removes the usage of ATS, just calling out to the whole testsuite all at once.
I did keep some of the `--collect` logic for the `codemod-tests` CI job, as I believe this is doing some kind of testsuite splitting.
As I saw you have modified the CI related workflows fairly recently, I’m curious whether the ATS feature as it exists is providing any value to you?
---
.github/actions/run-ats/action.yml | 61 ------------------------------
.github/actions/run-ats/ats.sh | 57 ----------------------------
.github/actions/run-ats/collect.sh | 4 --
.github/codecov.yml | 4 --
.github/workflows/test.yml | 44 +++++++++++++--------
5 files changed, 28 insertions(+), 142 deletions(-)
delete mode 100644 .github/actions/run-ats/action.yml
delete mode 100755 .github/actions/run-ats/ats.sh
delete mode 100644 .github/actions/run-ats/collect.sh
diff --git a/.github/actions/run-ats/action.yml b/.github/actions/run-ats/action.yml
deleted file mode 100644
index eaf13a032..000000000
--- a/.github/actions/run-ats/action.yml
+++ /dev/null
@@ -1,61 +0,0 @@
-name: 'Run ATS'
-description: 'Run Automated Test Selection to determine which tests to run'
-
-inputs:
- default_tests:
- description: 'Default test path to run'
- required: true
- codecov_static_token:
- description: 'Codecov static token'
- required: true
- codecov_token:
- description: 'Codecov token'
- required: true
- collect_args:
- description: 'Additional arguments for test collection'
- required: false
- default: ''
- ats_collect_args:
- description: 'Additional arguments for ATS collection'
- required: false
- default: ''
- codecov_flags:
- description: 'Flags for codecov upload'
- required: true
-
-runs:
- using: "composite"
- steps:
- - name: Run ATS
- shell: bash
- env:
- BASE_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
- DEFAULT_TESTS: ${{ inputs.default_tests }}
- CODECOV_STATIC_TOKEN: ${{ inputs.codecov_static_token }}
- CODECOV_TOKEN: ${{ inputs.codecov_token }}
- COLLECT_ARGS: ${{ inputs.collect_args }}
- ATS_COLLECT_ARGS: ${{ inputs.ats_collect_args }}
- run: |
- uv run codecovcli create-commit -t ${{ inputs.codecov_token }}
- uv run codecovcli create-report -t ${{ inputs.codecov_token }}
- bash .github/actions/run-ats/ats.sh
-
- - name: Run tests
- shell: bash
- run: |
- TESTS_TO_RUN=$(cat codecov_ats/tests_to_run.txt)
- if [ -z "$TESTS_TO_RUN" ]; then
- echo "No tests to run, skipping..."
- exit 0
- fi
- echo $TESTS_TO_RUN | xargs uv run --frozen pytest --cov \
- -o junit_suite_name="${{ github.job }}" \
- -n auto \
- -vv \
- --cov-append \
- ${{ inputs.collect_args }}
-
- - uses: ./.github/actions/report
- with:
- flag: ${{ inputs.codecov_flags }}
- codecov_token: ${{ inputs.codecov_token }}
diff --git a/.github/actions/run-ats/ats.sh b/.github/actions/run-ats/ats.sh
deleted file mode 100755
index bd107c16f..000000000
--- a/.github/actions/run-ats/ats.sh
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/usr/bin/env bash
-if ! command -v jq &> /dev/null; then
- apt update
- apt install jq -y
-fi
-export PATH=$PATH:$HOME/.local/bin
-echo "Git fetch"
-git fetch
-echo "Creating commit"
-uv run --frozen codecovcli create-commit -t ${CODECOV_TOKEN}
-echo "Creating report"
-uv run --frozen codecovcli create-report -t ${CODECOV_TOKEN}
-echo "Running static analysis"
-uv run --frozen codecovcli static-analysis --token ${CODECOV_STATIC_TOKEN} --folders-to-exclude .venv
-ATS_COLLECT_ARGS="${ATS_COLLECT_ARGS}${DEFAULT_TESTS},"
-echo "Label analysis with base sha: ${BASE_SHA} and default tests: ${ATS_COLLECT_ARGS}"
-runner_param="collect_tests_options=${ATS_COLLECT_ARGS}"
-
-# Always get last 10 commits as fallback options, starting with BASE_SHA if provided
-base_commit_candidates=($(git log --format=%H | sed -n "1,10p"))
-if [[ -n ${BASE_SHA} ]]; then
- # Prepend BASE_SHA to the candidates list if it's provided
- base_commit_candidates=("${BASE_SHA}" "${base_commit_candidates[@]}")
-fi
-
-for base_commit in ${base_commit_candidates[@]}
-do
- echo "Attempting label analysis with base commit: $base_commit"
- response=$(uv run --frozen codecovcli label-analysis --token ${CODECOV_STATIC_TOKEN} --base-sha=$base_commit --dry-run --dry-run-format="json" --runner-param "$runner_param" --max-wait-time=30 || true)
- if [[ -n $response ]]; then
- break
- else
- echo "-> Attempt failed"
- fi
-done
-
-if [[ -z $response ]]; then
- echo "Failed to run label analysis with any base commit."
-fi
-
-mkdir codecov_ats
-jq <<< "$response" '.runner_options + .ats_tests_to_run | .[1:] | map(gsub("\""; "")) | join(" ")' --raw-output > codecov_ats/tests_to_run.txt
-jq <<< "$response" '.runner_options + .ats_tests_to_skip | .[1:] | map(gsub("\""; "")) | join(" ")' --raw-output > codecov_ats/tests_to_skip.txt
-testcount() { jq <<< "$response" ".$1 | length - 1"; }
-run_count=$(testcount ats_tests_to_run)
-skip_count=$(testcount ats_tests_to_skip)
-echo "Run count: $run_count"
-echo "Skip count: $skip_count"
-if [ ! -s codecov_ats/tests_to_run.txt ]; then
- echo "No tests to run, collecting from default tests"
- PYTEST_ARGS="${COLLECT_ARGS} ${DEFAULT_TESTS}"
- echo "Using args: ${PYTEST_ARGS}"
- TESTS_TO_RUN=$(PYTEST_ARGS=${PYTEST_ARGS} ./.github/actions/run-ats/collect.sh)
- echo "${TESTS_TO_RUN}" > codecov_ats/tests_to_run.txt
- run_count=1
- echo "Added ${TESTS_TO_RUN} as fallback. New run count: $run_count"
-fi
diff --git a/.github/actions/run-ats/collect.sh b/.github/actions/run-ats/collect.sh
deleted file mode 100644
index ba34e1e6a..000000000
--- a/.github/actions/run-ats/collect.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-TESTS_TO_RUN=$(uv run --frozen pytest --collect-only ${PYTEST_ARGS} -q --disable-warnings --no-summary --no-header)
-TESTS_TO_RUN=$(echo "${TESTS_TO_RUN}" | head -n -2)
-echo $TESTS_TO_RUN
diff --git a/.github/codecov.yml b/.github/codecov.yml
index aabc7ab14..1b2a4b91a 100644
--- a/.github/codecov.yml
+++ b/.github/codecov.yml
@@ -39,22 +39,18 @@ component_management:
flag_management:
default_rules:
carryforward: true
- carryforward_mode: 'labels'
statuses:
- type: project
individual_flags:
- name: unit-tests
carryforward: true
- carryforward_mode: 'labels'
statuses:
- type: 'project'
- type: 'patch'
- name: codemod-tests
carryforward: true
- carryforward_mode: 'labels'
- name: integration-tests
carryforward: true
- carryforward_mode: 'labels'
comment:
layout: "condensed_header, condensed_files"
hide_project_coverage: true
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 9b923bf53..a6973cfdf 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -32,15 +32,21 @@ jobs:
- name: Setup environment
uses: ./.github/actions/setup-environment
- - name: Run ATS and Tests
- uses: ./.github/actions/run-ats
- timeout-minutes: 15
+ - name: Run tests
+ shell: bash
+ run: |
+ uv run --frozen pytest --cov \
+ -o junit_suite_name="${{ github.job }}" \
+ -n auto \
+ -vv \
+ --cov-append \
+ --timeout 15 \
+ "tests/unit"
+
+ - uses: ./.github/actions/report
with:
- default_tests: "tests/unit"
- codecov_static_token: ${{ secrets.CODECOV_STATIC_TOKEN }}
+ flag: unit-tests
codecov_token: ${{ secrets.CODECOV_TOKEN }}
- collect_args: "--timeout 15"
- codecov_flags: unit-tests
codemod-tests:
needs: access-check
@@ -71,18 +77,24 @@ jobs:
- name: Cache oss-repos
uses: ./.github/actions/setup-oss-repos
- - name: Run ATS and Tests
- uses: ./.github/actions/run-ats
+ - name: Run tests
timeout-minutes: 15
+ shell: bash
+ run: |
+ TESTS_TO_RUN=$(uv run --frozen pytest --collect-only --size=${{matrix.size}} --sync-graph=${{matrix.sync_graph}} -q --disable-warnings --no-summary --no-header)
+ TESTS_TO_RUN=$(echo "${TESTS_TO_RUN}" | head -n -2)
+ echo $TESTS_TO_RUN | xargs uv run --frozen pytest --cov \
+ -o junit_suite_name="${{ github.job }}" \
+ -n auto \
+ -vv \
+ --cov-append \
+ --timeout 15 \
+ "tests/integration/codemod/test_codemods.py"
+
+ - uses: ./.github/actions/report
with:
- default_tests: "tests/integration/codemod/test_codemods.py"
- codecov_static_token: ${{ secrets.CODECOV_STATIC_TOKEN }}
+ flag: codemod-tests-${{matrix.size}}-${{matrix.sync_graph}}
codecov_token: ${{ secrets.CODECOV_TOKEN }}
- collect_args: "--size=${{matrix.size}} --sync-graph=${{matrix.sync_graph}}"
- ats_collect_args: "--size=${{matrix.size}},--sync-graph=${{matrix.sync_graph}},"
- codecov_flags: codemod-tests-${{matrix.size}}-${{matrix.sync_graph}}
- env:
- GITHUB_WORKSPACE: $GITHUB_WORKSPACE
parse-tests:
needs: access-check