Merge branch 'main' into amammad-js-SQLI

Author: erik-krogh

.github/dependabot.yml

@@ -17,3 +17,26 @@ updates:
     ignore:
       - dependency-name: '*'
         update-types: ['version-update:semver-patch', 'version-update:semver-minor']
+
+  - package-ecosystem: "gomod"
+    directory: "go/extractor"
+    schedule:
+      interval: "daily"
+    allow:
+      - dependency-name: "golang.org/x/mod"
+      - dependency-name: "golang.org/x/tools"
+    groups:
+      extractor-dependencies:
+        patterns:
+          - "golang.org/x/*"
+    reviewers:
+      - "github/codeql-go"
+
+  - package-ecosystem: "gomod"
+    directory: "go/ql/test"
+    schedule:
+      interval: "monthly"
+    ignore:
+      - dependency-name: "*"
+    reviewers:
+      - "github/codeql-go"

.github/labeler.yml

@@ -45,11 +45,7 @@ documentation:
 
 # Since these are all shared files that need to be synced, just pick _one_ copy of each.
 "DataFlow Library":
-  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImpl.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplCommon.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/tainttracking1/TaintTrackingImpl.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/DataFlowImplConsistency.qll"
-  - "java/ql/lib/semmle/code/java/dataflow/internal/FlowSummaryImpl.qll"
+  - "shared/dataflow/**/*"
 
 "ATM":
   - javascript/ql/experimental/adaptivethreatmodeling/**/*

.github/workflows/check-change-note.yml

@@ -9,26 +9,42 @@ on:
       - "*/ql/lib/**/*.ql"
       - "*/ql/lib/**/*.qll"
       - "*/ql/lib/**/*.yml"
+      - "shared/**/*.ql"
+      - "shared/**/*.qll"
       - "!**/experimental/**"
       - "!ql/**"
       - ".github/workflows/check-change-note.yml"
 
 jobs:
   check-change-note:
+    env: 
+      REPO: ${{ github.repository }}
+      PULL_REQUEST_NUMBER: ${{ github.event.number }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     runs-on: ubuntu-latest
     steps:
+
       - name: Fail if no change note found. To fix, either add one, or add the `no-change-note-required` label.
         if: |
           github.event.pull_request.draft == false &&
           !contains(github.event.pull_request.labels.*.name, 'no-change-note-required')
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq 'any(.[].filename ; test("/change-notes/.*[.]md$"))' |
-          grep true -c
+          change_note_files=$(gh api "repos/$REPO/pulls/$PULL_REQUEST_NUMBER/files" --paginate --jq '.[].filename | select(test("/change-notes/.*[.]md$"))')
+          
+          if [ -z "$change_note_files" ]; then
+            echo "No change note found. Either add one, or add the 'no-change-note-required' label."
+            exit 1
+          fi
+
+          echo "Change notes found:"
+          echo "$change_note_files"
+
       - name: Fail if the change note filename doesn't match the expected format. The file name must be of the form 'YYYY-MM-DD.md', 'YYYY-MM-DD-{title}.md', where '{title}' is arbitrary text, or released/x.y.z.md for released change-notes
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          gh api 'repos/${{github.repository}}/pulls/${{github.event.number}}/files' --paginate --jq '[.[].filename | select(test("/change-notes/.*[.]md$"))] | all(test("/change-notes/[0-9]{4}-[0-9]{2}-[0-9]{2}.*[.]md$") or test("/change-notes/released/[0-9]*[.][0-9]*[.][0-9]*[.]md$"))' |
-          grep true -c
+          bad_change_note_file_names=$(gh api "repos/$REPO/pulls/$PULL_REQUEST_NUMBER/files" --paginate --jq '[.[].filename | select(test("/change-notes/.*[.]md$"))][] | select((test("/change-notes/[0-9]{4}-[0-9]{2}-[0-9]{2}.*[.]md$") or test("/change-notes/released/[0-9]*[.][0-9]*[.][0-9]*[.]md$")) | not)')
+
+          if [ -n "$bad_change_note_file_names" ]; then
+            echo "The following change note file names are invalid:"
+            echo "$bad_change_note_file_names"
+            exit 1
+          fi

.github/workflows/check-implicit-this.yml

@@ -13,7 +13,7 @@ jobs:
   check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Check that implicit this warnings is enabled for all packs
         shell: bash
         run: |

.github/workflows/check-qldoc.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 2
 

.github/workflows/check-query-ids.yml

@@ -16,6 +16,6 @@ jobs:
     name: Check query IDs
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Check for duplicate query IDs
         run: python3 misc/scripts/check-query-ids.py

.github/workflows/codeql-analysis.yml

@@ -33,7 +33,7 @@ jobs:
         dotnet-version: 7.0.102
 
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/compile-queries.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest-xl
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup CodeQL
         uses: ./.github/actions/fetch-codeql
         with:
@@ -29,9 +29,9 @@ jobs:
         # run with --check-only if running in a PR (github.sha != main)
         if : ${{ github.event_name == 'pull_request' }}
         shell: bash
-        run: codeql query compile -q -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+        run: codeql query compile -q -j0 */ql/{src,examples} --keep-going --warnings=error --check-only --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" --compilation-cache-size=500
       - name: compile queries - full
         # do full compile if running on main - this populates the cache
         if : ${{ github.event_name != 'pull_request' }}
         shell: bash
-        run: codeql query compile -q -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+        run: codeql query compile -q -j0 */ql/{src,examples} --keep-going --warnings=error --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" --compilation-cache-size=500

.github/workflows/csharp-qltest.yml

@@ -29,7 +29,7 @@ jobs:
   qlupgrade:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ./.github/actions/fetch-codeql
       - name: Check DB upgrade scripts
         run: |
@@ -52,8 +52,7 @@ jobs:
       matrix:
         slice: ["1/2", "2/2"]
     steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/fetch-codeql
+      - uses: actions/checkout@v4
       - uses: ./csharp/actions/create-extractor-pack
       - name: Cache compilation cache
         id: query-cache
@@ -62,25 +61,41 @@ jobs:
           key: csharp-qltest-${{ matrix.slice }}
       - name: Run QL tests
         run: |
-          CODEQL_PATH=$(gh codeql version --format=json | jq -r .unpackedLocation)
-          # The legacy ASP extractor is not in this repo, so take the one from the nightly build
-          mv "$CODEQL_PATH/csharp/tools/extractor-asp.jar" "${{ github.workspace }}/csharp/extractor-pack/tools"
-          # Safe guard against using the bundled extractor
-          rm -rf "$CODEQL_PATH/csharp"
-          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}/csharp/extractor-pack" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
         env:
           GITHUB_TOKEN: ${{ github.token }}
   unit-tests:
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-2019]
+    runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Setup dotnet
         uses: actions/setup-dotnet@v3
         with:
           dotnet-version: 7.0.102
       - name: Extractor unit tests
         run: |
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/csharp/extractor/Semmle.Util.Tests"
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/csharp/extractor/Semmle.Extraction.Tests"
-          dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/csharp/autobuilder/Semmle.Autobuild.CSharp.Tests"
+          dotnet test -p:RuntimeFrameworkVersion=7.0.2 extractor/Semmle.Util.Tests
+          dotnet test -p:RuntimeFrameworkVersion=7.0.2 extractor/Semmle.Extraction.Tests
+          dotnet test -p:RuntimeFrameworkVersion=7.0.2 autobuilder/Semmle.Autobuild.CSharp.Tests
           dotnet test -p:RuntimeFrameworkVersion=7.0.2 "${{ github.workspace }}/cpp/autobuilder/Semmle.Autobuild.Cpp.Tests"
+        shell: bash
+  stubgentest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./csharp/actions/create-extractor-pack
+      - name: Run stub generator tests
+        run: |
+          # Generate (Asp)NetCore stubs
+          STUBS_PATH=stubs_output
+          python3 scripts/stubs/make_stubs_nuget.py webapp Swashbuckle.AspNetCore.Swagger 6.5.0 "$STUBS_PATH"
+          rm -rf ql/test/resources/stubs/_frameworks
+          # Update existing stubs in the repo with the freshly generated ones
+          mv "$STUBS_PATH/output/stubs/_frameworks" ql/test/resources/stubs/
+          git status
+          codeql test run --threads=0 --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries -- ql/test/library-tests/dataflow/flowsources/aspremote
+        env:
+          GITHUB_TOKEN: ${{ github.token }}

.github/workflows/csv-coverage-metrics.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup CodeQL
         uses: ./.github/actions/fetch-codeql
       - name: Create empty database
@@ -47,7 +47,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup CodeQL
         uses: ./.github/actions/fetch-codeql
       - name: Create empty database

.github/workflows/csv-coverage-pr-artifacts.yml

@@ -31,11 +31,11 @@ jobs:
           GITHUB_CONTEXT: ${{ toJSON(github.event) }}
         run: echo "$GITHUB_CONTEXT"
       - name: Clone self (github/codeql) - MERGE
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: merge
       - name: Clone self (github/codeql) - BASE
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 2
           path: base
@@ -89,9 +89,32 @@ jobs:
       - name: Save PR number
         run: |
           mkdir -p pr
-          echo ${{ github.event.pull_request.number }} > pr/NR
+          echo ${PR_NUMBER} > pr/NR
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
       - name: Upload PR number
         uses: actions/upload-artifact@v3
         with:
           name: pr
           path: pr/
+      - name: Save comment ID (if it exists)
+        run: |
+          # Find the latest comment starting with COMMENT_PREFIX
+          COMMENT_PREFIX=":warning: The head of this PR and the base branch were compared for differences in the framework coverage reports."
+          COMMENT_ID=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate | jq --arg prefix "${COMMENT_PREFIX}" 'map(select(.body|startswith($prefix)) | .id) | max // empty')
+          if [[ -z ${COMMENT_ID} ]]
+          then
+            echo "Comment not found. Not uploading 'comment/ID' artifact."
+          else
+            mkdir -p comment
+            echo ${COMMENT_ID} > comment/ID
+          fi
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+      - name: Upload comment ID (if it exists)
+        uses: actions/upload-artifact@v3
+        with:
+          name: comment
+          path: comment/
+          if-no-files-found: ignore

.github/workflows/csv-coverage-pr-comment.yml

@@ -20,7 +20,7 @@ jobs:
         GITHUB_CONTEXT: ${{ toJSON(github.event) }}
       run: echo "$GITHUB_CONTEXT"
     - name: Clone self (github/codeql)
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
     - name: Set up Python 3.8
       uses: actions/setup-python@v4
       with:

.github/workflows/csv-coverage-timeseries.yml

@@ -9,11 +9,11 @@ jobs:
 
     steps:
       - name: Clone self (github/codeql)
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: script
       - name: Clone self (github/codeql) for analysis
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: codeqlModels
           fetch-depth: 0

.github/workflows/csv-coverage-update.yml

@@ -17,7 +17,7 @@ jobs:
           GITHUB_CONTEXT: ${{ toJSON(github.event) }}
         run: echo "$GITHUB_CONTEXT"
       - name: Clone self (github/codeql)
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: ql
           fetch-depth: 0

.github/workflows/csv-coverage.yml

@@ -13,11 +13,11 @@ jobs:
 
     steps:
       - name: Clone self (github/codeql)
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: script
       - name: Clone self (github/codeql) for analysis
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           path: codeqlModels
           ref: ${{ github.event.inputs.qlModelShaOverride || github.ref }}

.github/workflows/fast-forward.yml

@@ -25,7 +25,7 @@ jobs:
           exit 1
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Git config
         shell: bash

.github/workflows/go-tests-other-os.yml

@@ -21,7 +21,7 @@ jobs:
         id: go
 
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Set up CodeQL CLI
         uses: ./.github/actions/fetch-codeql
@@ -56,7 +56,7 @@ jobs:
         id: go
 
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Set up CodeQL CLI
         uses: ./.github/actions/fetch-codeql

.github/workflows/go-tests.yml

@@ -29,7 +29,7 @@ jobs:
         id: go
 
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Set up CodeQL CLI
         uses: ./.github/actions/fetch-codeql