Update CI

Try to keep the workflows a bit closer to how we setup things in the
plugin repositories.

Author: mdellweg

.ci/scripts/validate_commit_message.py

@@ -1,46 +1,54 @@
-# WARNING: DO NOT EDIT!
-#
-# This file was generated by plugin_template, and is managed by it. Please use
-# './plugin-template --github pulp_file' to update this file.
-#
-# For more info visit https://github.com/pulp/plugin_template
-
+import os
 import re
-import requests
 import subprocess
 import sys
+import tomllib
 from pathlib import Path
 
-KEYWORDS = ["fixes", "closes", "re", "ref"]
-NO_ISSUE = "[noissue]"
-STATUSES = ["NEW", "ASSIGNED", "POST", "MODIFIED"]
-REDMINE_URL = "https://pulp.plan.io"
-CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"]
+from github import Github
+
+with open("pyproject.toml", "rb") as fp:
+    PYPROJECT_TOML = tomllib.load(fp)
+KEYWORDS = ["fixes", "closes"]
+BLOCKING_REGEX = [
+    r"^DRAFT",
+    r"^WIP",
+    r"^NOMERGE",
+    r"^DO\s*NOT\s*MERGE",
+    r"^EXPERIMENT",
+    r"^FIXUP",
+    r"Apply suggestions from code review",
+]
+try:
+    CHANGELOG_EXTS = [
+        f".{item['directory']}" for item in PYPROJECT_TOML["tool"]["towncrier"]["type"]
+    ]
+except KeyError:
+    CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc"]
+NOISSUE_MARKER = "[noissue]"
 
 sha = sys.argv[1]
-project = ""
 message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8")
 
+if NOISSUE_MARKER in message:
+    sys.exit(f"Do not add '{NOISSUE_MARKER}' in the commit message.")
+
+if any((re.match(pattern, message, re.IGNORECASE) for pattern in BLOCKING_REGEX)):
+    sys.exit("This PR is not ready for consumption.")
 
-def __check_status(issue):
-    response = requests.get(f"{REDMINE_URL}/issues/{issue}.json")
-    response.raise_for_status()
-    bug_json = response.json()
-    status = bug_json["issue"]["status"]["name"]
-    if status not in STATUSES:
-        sys.exit(
-            "Error: issue #{issue} has invalid status of {status}. Status must be one of "
-            "{statuses}.".format(issue=issue, status=status, statuses=", ".join(STATUSES))
-        )
+g = Github(os.environ.get("GITHUB_TOKEN"))
+repo = g.get_repo("pulp/pulp-openapi-generator")
 
-    if project:
-        project_id = bug_json["issue"]["project"]["id"]
-        project_json = requests.get(f"{REDMINE_URL}/projects/{project_id}.json").json()
-        if project_json["project"]["identifier"] != project:
-            sys.exit(f"Error: issue {issue} is not in the {project} project.")
 
+def check_status(issue):
+    gi = repo.get_issue(int(issue))
+    if gi.pull_request:
+        sys.exit(f"Error: issue #{issue} is a pull request.")
+    if gi.closed_at:
+        sys.exit(f"Error: issue #{issue} is closed.")
 
-def __check_changelog(issue):
+
+def check_changelog(issue):
     matches = list(Path("CHANGES").rglob(f"{issue}.*"))
 
     if len(matches) < 1:
@@ -53,22 +61,15 @@ def __check_changelog(issue):
 print("Checking commit message for {sha}.".format(sha=sha[0:7]))
 
 # validate the issue attached to the commit
-regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS))
-pattern = re.compile(regex, re.IGNORECASE)
-
-issues = pattern.findall(message)
+issue_regex = r"(?:{keywords})[\s:]+#(\d+)".format(keywords=("|").join(KEYWORDS))
+issues = re.findall(issue_regex, message, re.IGNORECASE)
+cherry_pick_regex = r"^\s*\(cherry picked from commit [0-9a-f]*\)\s*$"
+cherry_pick = re.search(cherry_pick_regex, message, re.MULTILINE)
 
 if issues:
-    for issue in pattern.findall(message):
-        __check_status(issue)
-        # __check_changelog(issue)
-else:
-    if NO_ISSUE in message:
-        print("Commit {sha} has no issues but is tagged {tag}.".format(sha=sha[0:7], tag=NO_ISSUE))
-    else:
-        sys.exit(
-            "Error: no attached issues found for {sha}. If this was intentional, add "
-            " '{tag}' to the commit message.".format(sha=sha[0:7], tag=NO_ISSUE)
-        )
+    for issue in issues:
+        if not cherry_pick:
+            check_status(issue)
+            check_changelog(issue)
 
 print("Commit message for {sha} passed.".format(sha=sha[0:7]))

.github/workflows/ci.yml

@@ -1,56 +1,63 @@
 ---
-name: pulp-openapi-generator PR CI
-on: pull_request
+name: "pulp-openapi-generator PR CI"
+on:
+  pull_request:
 
-jobs:
+concurrency:
+  group: ${{ github.ref_name }}-${{ github.workflow }}
+  cancel-in-progress: true
 
-  lint:
-    runs-on: ubuntu-latest
+defaults:
+  run:
+    working-directory: "pulp-openapi-generator"
 
+jobs:
+  check-commits:
+    runs-on: "ubuntu-latest"
     steps:
-      - uses: actions/checkout@v2
+      - uses: "actions/checkout@v4"
         with:
-          # by default, it uses a depth of 1
-          # this fetches all history so that we can read each commit
           fetch-depth: 0
-
-      - uses: actions/setup-python@v2
+          path: "pulp-openapi-generator"
+      - uses: "actions/setup-python@v5"
         with:
-          python-version: "3.8"
-
-      - name: Check commit message
+          python-version: "3.11"
+      - name: "Install python dependencies"
+        run: |
+          pip install requests pygithub
+      - name: "Check commit message"
         if: github.event_name == 'pull_request'
         env:
-          GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }}
+          PY_COLORS: "1"
+          ANSIBLE_FORCE_COLOR: "1"
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}"
         run: |
-          pip install requests
-          sh .github/workflows/scripts/check_commit.sh
+          .github/workflows/scripts/check_commit.sh
 
   docs:
-    uses: "./.github/workflows/docs-ci.yml"
+    uses: "./.github/workflows/docs.yml"
 
-  pulp:
-    runs-on: ubuntu-latest
+  test:
+    runs-on: "ubuntu-latest"
     # run only after lint and docs finishes
     needs:
-      - lint
-      - docs
+      - "docs"
     strategy:
       fail-fast: false
       matrix:
         env:
-          - TEST: pulp
+          - TEST: "pulp"
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: "actions/checkout@v4"
         with:
-          # by default, it uses a depth of 1
-          # this fetches all history so that we can read each commit
           fetch-depth: 0
+          path: "pulp-openapi-generator"
 
-      - uses: actions/setup-python@v2
+      - uses: "actions/setup-python@v5"
         with:
-          python-version: "3.8"
+          python-version: "3.11"
 
       - uses: ruby/setup-ruby@v1
         with:
@@ -93,3 +100,20 @@ jobs:
           docker logs pulp || true
           docker exec pulp ls -latr /etc/yum.repos.d/ || true
           docker exec pulp cat /etc/yum.repos.d/* || true
+
+  ready-to-ship:
+    # This is a dummy dependent task to have a single entry for the branch protection rules.
+    runs-on: "ubuntu-latest"
+    needs:
+      - "check-commits"
+      - "test"
+      - "docs"
+    if: "always()"
+    steps:
+      - name: "Collect needed jobs results"
+        working-directory: "."
+        run: |
+          echo '${{toJson(needs)}}' | jq -r 'to_entries[]|select(.value.result!="success")|.key + ": " + .value.result'
+          echo '${{toJson(needs)}}' | jq -e 'to_entries|map(select(.value.result!="success"))|length == 0'
+          echo "CI says: Looks good!"
+...

.github/workflows/docs-ci.yml renamed to .github/workflows/docs.yml

@@ -15,5 +15,6 @@ jobs:
       - name: "Install Test Dependencies"
         run: |
           pip install -r doc_requirements.txt
-      - name: Build docs
-        run: make docs
+      - name: "Build docs"
+        run: |
+          make docs

.github/workflows/scripts/before_install.sh

@@ -7,9 +7,20 @@
 #
 # For more info visit https://github.com/pulp/plugin_template
 
+# make sure this script runs at the repo root
+cd "$(dirname "$(realpath -e "$0")")"/../../..
+
 set -mveuo pipefail
 
 # Intall requirements for ansible playbooks
 pip install docker netaddr boto3 ansible
 
-ansible-galaxy collection install amazon.aws
+for i in {1..3}
+do
+  ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3
+done
+if [[ $s -gt 0 ]]
+then
+  echo "Failed to install amazon.aws"
+  exit $s
+fi

.github/workflows/scripts/check_commit.sh

@@ -1,23 +1,11 @@
 #!/bin/bash
 
-# WARNING: DO NOT EDIT!
-#
-# This file was generated by plugin_template, and is managed by it. Please use
-# './plugin-template --github pulp_file' to update this file.
-#
-# For more info visit https://github.com/pulp/plugin_template
+# make sure this script runs at the repo root
+cd "$(dirname "$(realpath -e "$0")")/../../.."
 
 set -euv
 
-echo ::group::REQUESTS
-pip3 install requests
-echo ::endgroup::
-
-for sha in $(curl $GITHUB_CONTEXT | jq '.[].sha' | sed 's/"//g')
+for SHA in $(curl -H "Authorization: token $GITHUB_TOKEN" "$GITHUB_CONTEXT" | jq -r '.[].sha')
 do
-  python3 .ci/scripts/validate_commit_message.py $sha
-  VALUE=$?
-  if [ "$VALUE" -gt 0 ]; then
-    exit $VALUE
-  fi
+  python3 .ci/scripts/validate_commit_message.py "$SHA"
 done

.github/workflows/scripts/install.sh

@@ -7,11 +7,11 @@
 #
 # For more info visit https://github.com/pulp/plugin_template
 
-set -euv
-
 # make sure this script runs at the repo root
 cd "$(dirname "$(realpath -e "$0")")"/../../..
 
+set -euv
+
 TAG="${TAG:-latest}"
 
 mkdir -p .ci/ansible/vars

.github/workflows/scripts/test_bindings.py

@@ -152,8 +152,7 @@ def upload_file_in_chunks(file_path):
 pprint(repository_version_1)
 
 # Create an artifact from a local file
-file_path = os.path.join(os.environ["GITHUB_WORKSPACE"], ".github/workflows/scripts/test_bindings.py")
-artifact = artifacts.create(file=file_path)
+artifact = artifacts.create(file=__file__)
 pprint(artifact)
 
 # Create a FileContent from the artifact

.github/workflows/scripts/test_bindings.rb

@@ -40,7 +40,6 @@ def monitor_task(task_href)
     #
     # Returns:
     #     list[str]: List of hrefs that identify resource created by the task
-    completed = []
     task = @tasks_api.read(task_href)
     until ["completed", "failed", "canceled"].include? task.state
       sleep(2)
@@ -86,8 +85,8 @@ def upload_file_in_chunks(file_path)
           filechunk.write(chunk)
           filechunk.flush()
           actual_chunk_size = File.size(filechunk)
-          response = @uploads_api.update(content_range(offset, offset + actual_chunk_size -1, total_size), upload_href, filechunk)
-          offset += actual_chunk_size -1
+          response = @uploads_api.update(content_range(offset, offset + actual_chunk_size - 1, total_size), upload_href, filechunk)
+          offset += actual_chunk_size - 1
         ensure
           filechunk.close
           filechunk.unlink
@@ -100,7 +99,7 @@ def upload_file_in_chunks(file_path)
 end
 
 
-artifact = upload_file_in_chunks(File.join(ENV['GITHUB_WORKSPACE'], 'README.rst'))
+artifact = upload_file_in_chunks(File.new(File.expand_path(__FILE__)))
 
 # Create a File Remote
 remote_url = 'https://fixtures.pulpproject.org/file/PULP_MANIFEST'
@@ -120,12 +119,8 @@ def upload_file_in_chunks(file_path)
 
 repository_version_1 = @repoversions_api.read(created_resources[0])
 
-# Create an artifact from a local file
-file_path = File.join(ENV['GITHUB_WORKSPACE'], '.github/workflows/scripts/test_bindings.rb')
-artifact = @artifacts_api.create(File.new(file_path))
-
-# Create a FileContent from the artifact
-filecontent_response = @filecontent_api.create('foo.tar.gz', {artifact: artifact.pulp_href})
+# Create a FileContent from a local file
+filecontent_response = @filecontent_api.create('foo.tar.gz', {file: File.new(File.expand_path(__FILE__))})
 
 created_resources = monitor_task(filecontent_response.task)