diff --git a/.github/scripts/tests_matrix.sh b/.github/scripts/tests_matrix.sh new file mode 100644 index 00000000000..ca0b6eb8684 --- /dev/null +++ b/.github/scripts/tests_matrix.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +build_types="'validation'" +hw_types="'validation'" +wokwi_types="'validation'" +qemu_types="'validation'" + +if [[ $IS_PR != 'true' ]] || [[ $PERFORMANCE_ENABLED == 'true' ]]; then + build_types+=",'performance'" + hw_types+=",'performance'" + #wokwi_types+=",'performance'" + #qemu_types+=",'performance'" +fi + +targets="'esp32','esp32s2','esp32s3','esp32c3','esp32c6','esp32h2'" + +mkdir -p info + +echo "[$wokwi_types]" > info/wokwi_types.txt +echo "[$targets]" > info/targets.txt + +echo "build-types=[$build_types]" >> $GITHUB_OUTPUT +echo "hw-types=[$hw_types]" >> $GITHUB_OUTPUT +echo "wokwi-types=[$wokwi_types]" >> $GITHUB_OUTPUT +echo "qemu-types=[$qemu_types]" >> $GITHUB_OUTPUT +echo "targets=[$targets]" >> $GITHUB_OUTPUT diff --git a/.github/workflows/dangerjs.yml b/.github/workflows/dangerjs.yml index 9f7360bc34f..75c046731f3 100644 --- a/.github/workflows/dangerjs.yml +++ b/.github/workflows/dangerjs.yml @@ -19,4 +19,6 @@ jobs: - name: DangerJS pull request linter uses: espressif/shared-github-dangerjs@v1 env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + rule-max-commits: 'false' + commit-messages-min-summary-length: '10' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 26de19d8f10..ab8baa6d14c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -57,29 +57,25 @@ jobs: hw-types: ${{ steps.set-matrix.outputs.hw-types }} wokwi-types: ${{ steps.set-matrix.outputs.wokwi-types }} qemu-types: ${{ steps.set-matrix.outputs.qemu-types }} + targets: ${{ steps.set-matrix.outputs.targets }} + env: + IS_PR: ${{ github.event.pull_request.number != null }} + PERFORMANCE_ENABLED: ${{ contains(github.event.pull_request.labels.*.name, 'perf_test') }} steps: + - name: Checkout + uses: actions/checkout@v4 + with: + sparse-checkout: .github/scripts/tests_matrix.sh + - name: Set matrix id: set-matrix - run: | - build_types='["validation"' - hw_types='["validation"' - wokwi_types='["validation"' - qemu_types='["validation"' - - is_pr=${{ github.event.pull_request.number != null }} - is_performance_enabled=${{ contains(github.event.pull_request.labels.*.name, 'perf_test') }} - - if [[ $is_pr != 'true' ]] || [[ $is_performance_enabled == 'true' ]]; then - build_types+=',"performance"' - hw_types+=',"performance"' - #wokwi_types+=',"performance"' - #qemu_types+=',"performance"' - fi + run: bash .github/scripts/tests_matrix.sh - echo "build-types=$build_types]" >> $GITHUB_OUTPUT - echo "hw-types=$hw_types]" >> $GITHUB_OUTPUT - echo "wokwi-types=$wokwi_types]" >> $GITHUB_OUTPUT - echo "qemu-types=$qemu_types]" >> $GITHUB_OUTPUT + - name: Upload + uses: actions/upload-artifact@v4 + with: + name: matrix_info + path: info/* call-build-tests: name: Build @@ -88,7 +84,7 @@ jobs: strategy: matrix: type: ${{ fromJson(needs.gen-matrix.outputs.build-types) }} - chip: ['esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32c6', 'esp32h2'] + chip: ${{ fromJson(needs.gen-matrix.outputs.targets) }} with: type: ${{ matrix.type }} chip: ${{ matrix.chip }} @@ -105,7 +101,7 @@ jobs: fail-fast: false matrix: type: ${{ fromJson(needs.gen-matrix.outputs.hw-types) }} - chip: ['esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32c6', 'esp32h2'] + chip: ${{ fromJson(needs.gen-matrix.outputs.targets) }} with: type: ${{ matrix.type }} chip: ${{ matrix.chip }} diff --git a/.github/workflows/tests_results.yml b/.github/workflows/tests_results.yml index a255016c413..f9c572bf546 100644 --- a/.github/workflows/tests_results.yml +++ b/.github/workflows/tests_results.yml @@ -79,6 +79,7 @@ jobs: event_name: ${{ env.original_event }} files: ./artifacts/**/*.xml action_fail: true + compare_to_earlier_commit: false - name: Fail if tests failed if: ${{ env.original_conclusion == 'failure' || env.original_conclusion == 'timed_out' || github.event.workflow_run.conclusion == 'failure' || github.event.workflow_run.conclusion == 'timed_out' }} diff --git a/.github/workflows/tests_wokwi.yml b/.github/workflows/tests_wokwi.yml index f016cad25e0..a891ca89dfd 100644 --- a/.github/workflows/tests_wokwi.yml +++ b/.github/workflows/tests_wokwi.yml @@ -22,6 +22,9 @@ jobs: outputs: pr_num: ${{ steps.set-ref.outputs.pr_num }} ref: ${{ steps.set-ref.outputs.ref }} + base: ${{ steps.set-ref.outputs.base }} + targets: ${{ steps.set-ref.outputs.targets }} + types: ${{ steps.set-ref.outputs.types }} steps: - name: Report pending uses: actions/github-script@v7 @@ -51,10 +54,18 @@ jobs: name: event_file path: artifacts/event_file + - name: Download and extract matrix info + uses: actions/download-artifact@v4 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} + name: matrix_info + path: artifacts/matrix_info + - name: Try to read PR number id: set-ref run: | - pr_num=$(jq -r '.pull_request.number' artifacts/event_file/event.json) + pr_num=$(jq -r '.pull_request.number' artifacts/event_file/event.json | tr -cd "[:digit:]") if [ -z "$pr_num" ] || [ "$pr_num" == "null" ]; then pr_num="" fi @@ -64,11 +75,22 @@ jobs: ref=${{ github.ref }} fi - action=$(jq -r '.action' artifacts/event_file/event.json) + action=$(jq -r '.action' artifacts/event_file/event.json | tr -cd "[:alpha:]_") if [ "$action" == "null" ]; then action="" fi + base=$(jq -r '.pull_request.base.ref' artifacts/event_file/event.json | tr -cd "[:alnum:]/_.-") + if [ -z "$base" ] || [ "$base" == "null" ]; then + base=${{ github.ref }} + fi + + types=$(cat artifacts/matrix_info/wokwi_types.txt | tr -cd "[:alpha:],[]'") + targets=$(cat artifacts/matrix_info/targets.txt | tr -cd "[:alnum:],[]'") + + echo "base = $base" + echo "targets = $targets" + echo "types = $types" echo "pr_num = $pr_num" printf "$ref" >> artifacts/ref.txt @@ -98,6 +120,9 @@ jobs: cat artifacts/conclusion.txt echo "pr_num=$pr_num" >> $GITHUB_OUTPUT + echo "base=$base" >> $GITHUB_OUTPUT + echo "targets=$targets" >> $GITHUB_OUTPUT + echo "types=$types" >> $GITHUB_OUTPUT echo "ref=$ref" >> $GITHUB_OUTPUT - name: Download and extract parent hardware results @@ -164,8 +189,8 @@ jobs: strategy: fail-fast: false matrix: - type: ['validation'] - chip: ['esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32c6', 'esp32h2'] + type: ${{ fromJson(needs.get-artifacts.outputs.types) }} + chip: ${{ fromJson(needs.get-artifacts.outputs.targets) }} steps: - name: Report pending uses: actions/github-script@v7 @@ -211,9 +236,12 @@ jobs: echo "enabled=$enabled" >> $GITHUB_OUTPUT # Note that changes to the workflows and tests will only be picked up after the PR is merged + # DO NOT CHECKOUT THE USER'S REPOSITORY IN THIS WORKFLOW. IT HAS HIGH SECURITY RISKS. - name: Checkout repository if: ${{ steps.check-tests.outputs.enabled == 'true' }} uses: actions/checkout@v4 + with: + ref: ${{ needs.get-artifacts.outputs.base || github.ref }} - uses: actions/setup-python@v5 if: ${{ steps.check-tests.outputs.enabled == 'true' }} diff --git a/tests/performance/coremark/coremark.ino b/tests/performance/coremark/coremark.ino index 776db7874db..872b53050f0 100644 --- a/tests/performance/coremark/coremark.ino +++ b/tests/performance/coremark/coremark.ino @@ -40,7 +40,7 @@ void setup() { Serial.printf("Cores: %d\n", CONFIG_SOC_CPU_CORES_NUM); Serial.flush(); for (int i = 0; i < N_RUNS; i++) { - Serial.printf("Run %d", i); + Serial.printf("Run %d\n", i); coremark_main(); Serial.flush(); } diff --git a/tests/performance/fibonacci/fibonacci.ino b/tests/performance/fibonacci/fibonacci.ino index 01fd6f7bee2..c82fd6b70d8 100644 --- a/tests/performance/fibonacci/fibonacci.ino +++ b/tests/performance/fibonacci/fibonacci.ino @@ -31,7 +31,7 @@ void setup() { Serial.printf("N: %d\n", FIB_N); Serial.flush(); for (int i = 0; i < N_RUNS; i++) { - Serial.printf("Run %d", i); + Serial.printf("Run %d\n", i); unsigned long start = millis(); fibonacci = fib(FIB_N); unsigned long elapsed = millis() - start; diff --git a/tests/performance/fibonacci/test_fibonacci.py b/tests/performance/fibonacci/test_fibonacci.py index 622ea77ee4b..ced9368184e 100644 --- a/tests/performance/fibonacci/test_fibonacci.py +++ b/tests/performance/fibonacci/test_fibonacci.py @@ -2,24 +2,21 @@ import logging import os +fib_results = {} + +def fib(n): + if n < 2: + return n + elif str(n) in fib_results: + return fib_results[str(n)] + else: + fib_results[str(n)] = fib(n - 1) + fib(n - 2) + return fib_results[str(n)] + def test_fibonacci(dut, request): LOGGER = logging.getLogger(__name__) - # Fibonacci results starting from fib(35) to fib(45) - fib_results = [ - 9227465, - 14930352, - 24157817, - 39088169, - 63245986, - 102334155, - 165580141, - 267914296, - 433494437, - 701408733, - ] - # Match "Runs: %d" res = dut.expect(r"Runs: (\d+)", timeout=60) runs = int(res.group(0).decode("utf-8").split(" ")[1]) @@ -30,7 +27,11 @@ def test_fibonacci(dut, request): res = dut.expect(r"N: (\d+)", timeout=300) fib_n = int(res.group(0).decode("utf-8").split(" ")[1]) LOGGER.info("Calculating Fibonacci({})".format(fib_n)) - assert fib_n > 30 and fib_n < 50, "Invalid Fibonacci number" + assert fib_n > 0, "Invalid Fibonacci number" + + # Calculate Fibonacci results + expected_result = fib(fib_n) + LOGGER.info("Expected Fibonacci result: {}".format(expected_result)) list_time = [] @@ -48,7 +49,7 @@ def test_fibonacci(dut, request): assert fib_result > 0, "Invalid Fibonacci result" # Check if the result is correct - assert fib_result == fib_results[fib_n - 35] + assert fib_result == expected_result # Match "Time: %lu.%03lu s" res = dut.expect(r"Time: (\d+)\.(\d+) s", timeout=300) diff --git a/tests/performance/psramspeed/ci.json b/tests/performance/psramspeed/ci.json index 8d58dbf5250..341df103671 100644 --- a/tests/performance/psramspeed/ci.json +++ b/tests/performance/psramspeed/ci.json @@ -3,9 +3,7 @@ "qemu": false, "wokwi": false }, - "targets": { - "esp32c3": false, - "esp32c6": false, - "esp32h2": false - } + "requires": [ + "CONFIG_SPIRAM=y" + ] } diff --git a/tests/performance/psramspeed/psramspeed.ino b/tests/performance/psramspeed/psramspeed.ino index 81175e6b3a5..be91733abfc 100644 --- a/tests/performance/psramspeed/psramspeed.ino +++ b/tests/performance/psramspeed/psramspeed.ino @@ -252,7 +252,7 @@ void setup() { Serial.printf("Max test size: %d\n", MAX_TEST_SIZE); Serial.flush(); for (int i = 0; i < N_RUNS; i++) { - Serial.printf("Run %d", i); + Serial.printf("Run %d\n", i); memcpy_speed_test(dest, src, MAX_TEST_SIZE, N_COPIES); Serial.flush(); memset_speed_test(dest, FILL_VALUE, MAX_TEST_SIZE, N_COPIES); diff --git a/tests/performance/ramspeed/ramspeed.ino b/tests/performance/ramspeed/ramspeed.ino index e0ab0db4c5f..776f6540679 100644 --- a/tests/performance/ramspeed/ramspeed.ino +++ b/tests/performance/ramspeed/ramspeed.ino @@ -248,7 +248,7 @@ void setup() { Serial.printf("Max test size: %d\n", MAX_TEST_SIZE); Serial.flush(); for (int i = 0; i < N_RUNS; i++) { - Serial.printf("Run %d", i); + Serial.printf("Run %d\n", i); memcpy_speed_test(dest, src, MAX_TEST_SIZE, N_COPIES); Serial.flush(); memset_speed_test(dest, FILL_VALUE, MAX_TEST_SIZE, N_COPIES); diff --git a/tests/performance/superpi/superpi.ino b/tests/performance/superpi/superpi.ino index ffa6c932b35..7ac4b2f13d7 100644 --- a/tests/performance/superpi/superpi.ino +++ b/tests/performance/superpi/superpi.ino @@ -25,7 +25,7 @@ void setup() { Serial.printf("Digits: %d\n", DIGITS); Serial.flush(); for (int i = 0; i < N_RUNS; i++) { - Serial.printf("Run %d", i); + Serial.printf("Run %d\n", i); unsigned long start = millis(); pi_calc(DIGITS); unsigned long elapsed = millis() - start; diff --git a/tests/requirements.txt b/tests/requirements.txt index 1b43a6104f2..63d204a96b0 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,7 +1,7 @@ cryptography==43.0.1 --only-binary cryptography pytest-cov==5.0.0 -pytest-embedded-serial-esp==1.11.5 -pytest-embedded-arduino==1.11.5 -pytest-embedded-wokwi==1.11.5 -pytest-embedded-qemu==1.11.5 +pytest-embedded-serial-esp==1.11.6 +pytest-embedded-arduino==1.11.6 +pytest-embedded-wokwi==1.11.6 +pytest-embedded-qemu==1.11.6 diff --git a/tests/validation/gpio/test_gpio.py b/tests/validation/gpio/test_gpio.py index e36282561b5..f11b9fd99cc 100644 --- a/tests/validation/gpio/test_gpio.py +++ b/tests/validation/gpio/test_gpio.py @@ -1,5 +1,15 @@ +import logging + def test_gpio(dut): + LOGGER = logging.getLogger(__name__) + dut.expect_exact("Button test") + + LOGGER.info("Expecting button press 1") dut.expect_exact("Button pressed 1 times") + + LOGGER.info("Expecting button press 2") dut.expect_exact("Button pressed 2 times") + + LOGGER.info("Expecting button press 3") dut.expect_exact("Button pressed 3 times") diff --git a/tests/validation/nvs/test_nvs.py b/tests/validation/nvs/test_nvs.py index 364df56de1c..a2b4842fa91 100644 --- a/tests/validation/nvs/test_nvs.py +++ b/tests/validation/nvs/test_nvs.py @@ -1,4 +1,13 @@ +import logging + def test_nvs(dut): + LOGGER = logging.getLogger(__name__) + + LOGGER.info("Expecting counter value 0") dut.expect_exact("Current counter value: 0") + + LOGGER.info("Expecting counter value 1") dut.expect_exact("Current counter value: 1") + + LOGGER.info("Expecting counter value 2") dut.expect_exact("Current counter value: 2") diff --git a/tests/validation/periman/test_periman.py b/tests/validation/periman/test_periman.py index d8dc4b8eeb5..a2d25f5ba09 100644 --- a/tests/validation/periman/test_periman.py +++ b/tests/validation/periman/test_periman.py @@ -1,4 +1,7 @@ +import logging + def test_periman(dut): + LOGGER = logging.getLogger(__name__) peripherals = [ "GPIO", "SigmaDelta", @@ -29,9 +32,10 @@ def test_periman(dut): if peripheral in peripherals: if "not" in console_output: - assert False, f"Peripheral {peripheral} printed when it should not" + assert False, f"Output printed when it should not after peripheral {peripheral}" + LOGGER.info(f"Correct output after peripheral: {peripheral}") peripherals.remove(peripheral) else: assert False, f"Unknown peripheral: {peripheral}" - assert peripherals == [], f"Missing peripherals output: {peripherals}" + assert peripherals == [], f"Missing output after peripherals: {peripherals}" diff --git a/tests/validation/psram/ci.json b/tests/validation/psram/ci.json index fc34574cf37..341df103671 100644 --- a/tests/validation/psram/ci.json +++ b/tests/validation/psram/ci.json @@ -5,10 +5,5 @@ }, "requires": [ "CONFIG_SPIRAM=y" - ], - "targets": { - "esp32c3": false, - "esp32c6": false, - "esp32h2": false - } + ] } diff --git a/tests/validation/wifi/test_wifi.py b/tests/validation/wifi/test_wifi.py index 49dd22797d2..769283b06bd 100644 --- a/tests/validation/wifi/test_wifi.py +++ b/tests/validation/wifi/test_wifi.py @@ -1,6 +1,15 @@ +import logging + def test_wifi(dut): + LOGGER = logging.getLogger(__name__) + + LOGGER.info("Starting WiFi Scan") dut.expect_exact("Scan start") dut.expect_exact("Scan done") dut.expect_exact("Wokwi-GUEST") + LOGGER.info("WiFi Scan done") + + LOGGER.info("Connecting to WiFi") dut.expect_exact("WiFi connected") dut.expect_exact("IP address:") + LOGGER.info("WiFi connected")