collect-statistics.yml

name: "[M] UTBot Java: collect statistics"

permissions: read-all

on:
  workflow_call:
    inputs:
      runners:
        description: 'Runners number'
        required: false
        default: '1'
        type: string
      run_number:
        description: 'Number of run tries per runner (values greater than 1 are not supported with grafana)'
        required: false
        default: '1'
        type: string
      message_prefix:
        description: 'Commit message prefix'
        required: false
        default: manual-run
        type: string
      push_results:
        description: 'Push metrics into github'
        required: false
        default: false
        type: boolean
      send_to_grafana:
        description: 'Send metrics to grafana'
        required: false
        default: false
        type: boolean

  workflow_dispatch:
    inputs:
      runners:
        description: 'Runners number'
        required: false
        default: '1'
        type: string
      run_number:
        description: 'Number of run tries per runner (values greater than 1 are not supported with grafana)'
        required: false
        default: '1'
        type: string
      message_prefix:
        description: 'Commit message prefix'
        required: false
        default: manual-run
        type: string
      push_results:
        description: 'Push metrics into github'
        required: false
        default: false
        type: boolean
      send_to_grafana:
        description: 'Send metrics to grafana'
        required: false
        default: false
        type: boolean

env:
  data_branch: monitoring-data
  data_path: monitoring/data
  monitoring_projects: monitoring/projects/
  push_script: monitoring/push_with_rebase.sh
  PUSHGATEWAY_HOSTNAME: monitoring.utbot.org
  PUSHGATEWAY_ADDITIONAL_PATH: /pushgateway-custom
  PROM_ADDITIONAL_LABELS: /service/github

jobs:
  setup_matrix:
    runs-on: ubuntu-latest
    outputs:
      projects: ${{ steps.set-matrix.outputs.projects }}
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3

      - name: Create matrix
        id: set-matrix
        shell: bash
        run: |
          read -r -a projects <<< $(ls --format=horizontal --indicator-style=none $monitoring_projects)
          projects=(${projects[@]/#/\"})
          projects=(${projects[@]/%/\"})
          printf -v projects '%s,' "${projects[@]}"
          projects=$(echo [${projects%,}])
          echo "projects=$projects" >> $GITHUB_OUTPUT
          echo $projects
          
          arr=$(echo [$(seq -s , ${{ inputs.runners }})])
          echo "matrix=$arr" >> $GITHUB_OUTPUT
          echo $arr

  build_and_collect_statistics:
    needs: setup_matrix
    continue-on-error: true
    strategy:
      # temporary commented, remove completely after 10.23 if all pipelines worked well
      #max-parallel: 3
      matrix:
        project: ${{ fromJson(needs.setup_matrix.outputs.projects) }}
        value: ${{ fromJson(needs.setup_matrix.outputs.matrix) }}
    runs-on: ubuntu-20.04
    container: unittestbot/java-env:java17-zulu-jdk-gradle7.6.1-kotlinc1.8.0
    steps:
      - name: Install git
        run: |
          apt-get update -y
          apt-get install git -y
          git config --global --add safe.directory $(pwd)

      - name: Checkout repository
        uses: actions/checkout@v3

      - name: Checkout monitoring data
        uses: actions/checkout@v3
        with:
          ref: ${{ env.data_branch }}
          path: ${{ env.data_path }}

      - name: Expand system swap
        shell: bash
        # trying to configure swap on host from running container
        run: |
          docker run -d --rm --name busybox --privileged --net=host --pid=host --ipc=host --volume /:/host busybox sleep infinity
          docker exec busybox /bin/sh -c 'chroot /host /bin/bash -c "swapoff /mnt/swapfile"'
          docker exec busybox /bin/sh -c 'chroot /host /bin/bash -c "dd if=/dev/zero of=/mnt/swapfile bs=1M count=8192 oflag=append conv=notrunc"'
          docker exec busybox /bin/sh -c 'chroot /host /bin/bash -c "mkswap /mnt/swapfile"'
          docker exec busybox /bin/sh -c 'chroot /host /bin/bash -c "swapon /mnt/swapfile"'

      - name: Run system monitoring
        # secret uploaded using base64 encoding to have one-line output:
        # cat file | base64 -w 0
        continue-on-error: true
        run: |
          chmod +x ./scripts/project/monitoring.sh
          ./scripts/project/monitoring.sh "${PUSHGATEWAY_HOSTNAME}" "${{ secrets.PUSHGATEWAY_USER }}" "${{ secrets.PUSHGATEWAY_PASSWORD }}"
          echo "Please visit Grafana to check metrics: https://${PUSHGATEWAY_HOSTNAME}/d/rYdddlPWk/node-exporter-full?orgId=1&from=now-1h&to=now&var-service=github&var-instance=${GITHUB_RUN_ID}-${HOSTNAME}&refresh=1m"

      - uses: actions/setup-python@v4
        with:
          python-version: '3.9'

      - name: Build and run monitoring UTBot Java
        run: |
          gradle :utbot-junit-contest:monitoringJar
          for i in $(seq ${{ inputs.run_number }})
          do
            java -jar \
              --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED \
              --add-opens java.base/java.lang.invoke=ALL-UNNAMED \
              --add-opens java.base/java.util.concurrent=ALL-UNNAMED \
              --add-opens java.base/java.util.concurrent.locks=ALL-UNNAMED \
              --add-opens java.base/java.text=ALL-UNNAMED \
              --add-opens java.base/java.time=ALL-UNNAMED \
              --add-opens java.base/java.io=ALL-UNNAMED \
              --add-opens java.base/java.nio=ALL-UNNAMED \
              --add-opens java.base/java.nio.file=ALL-UNNAMED \
              --add-opens java.base/java.net=ALL-UNNAMED \
              --add-opens java.base/sun.security.util=ALL-UNNAMED \
              --add-opens java.base/sun.reflect.generics.repository=ALL-UNNAMED \
              --add-opens java.base/sun.net.util=ALL-UNNAMED \
              --add-opens java.base/sun.net.fs=ALL-UNNAMED \
              --add-opens java.base/java.security=ALL-UNNAMED \
              --add-opens java.base/java.lang.ref=ALL-UNNAMED \
              --add-opens java.base/java.math=ALL-UNNAMED \
              --add-opens java.base/java.util.stream=ALL-UNNAMED \
              --add-opens java.base/java.util=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.misc=ALL-UNNAMED \
              --add-opens java.base/java.lang=ALL-UNNAMED \
              --add-opens java.base/java.lang.reflect=ALL-UNNAMED \
              --add-opens java.base/sun.security.provider=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.event=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.jimage=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.jimage.decompressor=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.jmod=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.jtrfs=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.loader=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.logger=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.math=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.misc=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.module=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.org.objectweb.asm.commons=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.org.objectweb.asm.signature=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.org.objectweb.asm.tree=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.org.objectweb.asm.tree.analysis=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.org.objectweb.asm.util=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.org.xml.sax=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.org.xml.sax.helpers=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.perf=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.platform=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.ref=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.util=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.util.xml=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.util.xml.impl=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.vm=ALL-UNNAMED \
              --add-opens java.base/jdk.internal.vm.annotation=ALL-UNNAMED \
              -Dutbot.monitoring.settings.path=$monitoring_projects/${{ matrix.project }}/monitoring.properties \
              utbot-junit-contest/build/libs/monitoring.jar \
              stats-$i.json
            mv logs/utbot.log logs/utbot-$i.log
          done

      - name: Get current date
        id: date
        run: |
          echo "date=$(date +'%Y-%m-%d-%H-%M-%S')" >> $GITHUB_OUTPUT
          echo "timestamp=$(date +%s)" >> $GITHUB_OUTPUT
          echo "last_month=$(date --date='last month' +%s)" >> $GITHUB_OUTPUT

      - name: Get metadata
        id: metadata
        run: |
          echo "commit=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
          echo "short_commit=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
          echo "branch=$(git name-rev --name-only HEAD)" >> $GITHUB_OUTPUT
          echo "build=$(date +'%Y.%-m')" >> $GITHUB_OUTPUT

      - name: Insert metadata
        id: insert
        shell: bash
        run: |
          OUT_FILE="$data_path/$date-$short_commit-${{ matrix.project }}-${{ matrix.value }}.json"
          echo "output=$OUT_FILE" >> $GITHUB_OUTPUT

          INPUTS=($(seq ${{ inputs.run_number }}))
          INPUTS=(${INPUTS[@]/#/stats-})
          INPUTS=(${INPUTS[@]/%/.json})
          INPUTS=${INPUTS[@]}
          echo $INPUTS

          python monitoring/insert_metadata.py \
            --stats_file $INPUTS \
            --output_file "$OUT_FILE" \
            --commit $commit \
            --branch $branch \
            --build "$build" \
            --timestamp $timestamp \
            --source_type "github-action" \
            --source_id $run_id
        env:
          date: ${{ steps.date.outputs.date }}
          timestamp: ${{ steps.date.outputs.timestamp }}
          commit: ${{ steps.metadata.outputs.commit }}
          short_commit: ${{ steps.metadata.outputs.short_commit }}
          branch: ${{ steps.metadata.outputs.branch }}
          build: ${{ steps.metadata.outputs.build }}
          run_id: ${{ github.run_id }}-${{ matrix.value }}

      - name: Upload statistics
        uses: actions/upload-artifact@v3
        with:
          name: statistics-${{ matrix.value }}-${{ matrix.project }}
          path: ${{ steps.insert.outputs.output }}

      - name: Commit and push statistics
        if: ${{ inputs.push_results }}
        run: |
          chmod +x $push_script
          ./$push_script
        env:
          target_branch: ${{ env.data_branch }}
          target_directory: ${{ env.data_path }}
          message: ${{ inputs.message_prefix }}-${{ steps.date.outputs.date }}
          github_token: ${{ secrets.GITHUB_TOKEN }}

      - name: Send metrics to grafana
        if: ${{ inputs.send_to_grafana }}
        run: |
          python monitoring/prepare_metrics.py --stats_file $stats_file --output_file grafana_metrics.json
          chmod +x scripts/project/json_to_prometheus.py
          python3 scripts/project/json_to_prometheus.py grafana_metrics.json | curl -u "${{ secrets.PUSHGATEWAY_USER }}:${{ secrets.PUSHGATEWAY_PASSWORD }}" --data-binary @- "https://${PUSHGATEWAY_HOSTNAME}${PUSHGATEWAY_ADDITIONAL_PATH}/metrics/job/pushgateway-custom/instance/run-${{ matrix.value }}-${{ matrix.project }}${PROM_ADDITIONAL_LABELS}"
          echo "Please visit Grafana to check metrics: https://monitoring.utbot.org/d/m6bagaD4z/utbot-nightly-statistic"
        env:
          stats_file: ${{ steps.insert.outputs.output }}

      - name: Upload logs
        if: ${{ always() }}
        uses: actions/upload-artifact@v3
        with:
          name: logs-${{ matrix.value }}-${{ matrix.project }}
          path: logs/

      - name: Upload artifacts
        if: ${{ always() }}
        uses: actions/upload-artifact@v3
        with:
          name: generated-${{ matrix.value }}-${{ matrix.project }}
          path: |
            /tmp/UTBot/generated*/*