diff --git a/helper.fish b/helper.fish index fd17f726..991a7097 100755 --- a/helper.fish +++ b/helper.fish @@ -365,6 +365,18 @@ function debDockerImage; set -gx DOCKER_DISTRO deb ; end if test -z "$DOCKER_DISTRO"; alpineDockerImage else ; set -gx DOCKER_DISTRO $DOCKER_DISTRO ; end +function enableDockerCveCheck ; set -gx RUN_CVE_CHECKS_FOR_DOCKER_IMAGE 1 ; end +function disableDockerCveCheck ; set -gx RUN_CVE_CHECKS_FOR_DOCKER_IMAGE 0 ; end +function enableCveReport ; set -gx CREATE_CVE_REPORT_FOR_DOCKER_IMAGE 1 ; end +function disableCveReport ; set -gx CREATE_CVE_REPORT_FOR_DOCKER_IMAGE 0 ; end +function cveBlockPublishing ; set -gx PUBLISH_DOCKER_IMAGE_ONLY_IF_CVE_CHECKS_PASS 1 ; end +function cveNoBlockPublishing ; set -gx PUBLISH_DOCKER_IMAGE_ONLY_IF_CVE_CHECKS_PASS 0 ; end +function cveToleranceNegligible ; set -gx CVE_SEVERITY_THRESHOLD negligible ; end +function cveToleranceLow ; set -gx CVE_SEVERITY_THRESHOLD low ; end +function cveToleranceMedium ; set -gx CVE_SEVERITY_THRESHOLD medium ; end +function cveToleranceHigh ; set -gx CVE_SEVERITY_THRESHOLD high ; end +function cveToleranceCritical ; set -gx CVE_SEVERITY_THRESHOLD critical ; end + function skipNondeterministic ; set -gx SKIPNONDETERMINISTIC true ; end function includeNondeterministic ; set -gx SKIPNONDETERMINISTIC false ; end if test -z "$SKIPNONDETERMINISTIC"; skipNondeterministic @@ -2424,6 +2436,11 @@ function moveResultsToWorkspace echo "mv JUnit XMLs ($WORKDIR/work/ArangoDB/testrunXml)" mv $WORKDIR/work/ArangoDB/testrunXml $WORKSPACE/testrunXml end + + if test -d $WORKDIR/work/grype_reports + echo "mv grype reports ($WORKDIR/work/grype_reports)" + mv $WORKDIR/work/grype_reports $WORKSPACE/grype_reports + end end end diff --git a/helper.linux.fish b/helper.linux.fish index eb2bb1b8..0ade6e46 100755 --- a/helper.linux.fish +++ b/helper.linux.fish @@ -1225,6 +1225,7 @@ function buildDockerAny and if test "$IMAGE_NAME1" != "$IMAGE_NAME2" docker tag $IMAGE_NAME1 $IMAGE_NAME2 end + and validateDockerImageIfNeeded $IMAGE_NAME2 and pushDockerImage $IMAGE_NAME2 and if test "$GCR_REG" = "On" docker tag $IMAGE_NAME1 $GCR_REG_PREFIX$IMAGE_NAME2 @@ -1245,6 +1246,36 @@ function buildDockerAny end end +function validateDockerImageIfNeeded + if test (count $argv) -eq 0 + echo Must give docker image name as argument + return 1 + end + set -l image_name $argv[1] + echo "going to scan docker image for CVEs: $image_name" + set -l filesafe_image_name (string replace "/" "-" -- $image_name) + if test "$RUN_CVE_CHECKS_FOR_DOCKER_IMAGE" = "1"; or test "$RUN_CVE_CHECKS_FOR_DOCKER_IMAGE" = "On" + if test "$CREATE_CVE_REPORT_FOR_DOCKER_IMAGE" = "1"; or test "$CREATE_CVE_REPORT_FOR_DOCKER_IMAGE" = "On" + set -l grype_report_dir $WORKDIR/work/grype_reports + if ! test -d $grype_report_dir + mkdir -p $grype_report_dir + end + set -l CVE_REPORT_FILE $grype_report_dir/grype-cve-report-$filesafe_image_name.txt + checkDockerImageForCves $image_name $CVE_REPORT_FILE + else + checkDockerImageForCves $image_name + end + end + if test $status -ne 0 + echo "Grype CVE check failed for $image_name" + if test "$PUBLISH_DOCKER_IMAGE_ONLY_IF_CVE_CHECKS_PASS" = "1"; or test "$PUBLISH_DOCKER_IMAGE_ONLY_IF_CVE_CHECKS_PASS" = "On" + return 1 + else + return 0 + end + end +end + function buildDockerArgs if test (count $argv) -eq 0 echo Must give image distro as argument @@ -1732,6 +1763,10 @@ function runInContainer -e PROMTOOL_PATH="$PROMTOOL_PATH" \ -e BUILD_REPO_INFO="$BUILD_REPO_INFO" \ -e ARANGODB_BUILD_DATE="$ARANGODB_BUILD_DATE" \ + -e RUN_CVE_CHECKS_FOR_DOCKER_IMAGE="$RUN_CVE_CHECKS_FOR_DOCKER_IMAGE" \ + -e CREATE_CVE_REPORT_FOR_DOCKER_IMAGE="$CREATE_CVE_REPORT_FOR_DOCKER_IMAGE" \ + -e PUBLISH_DOCKER_IMAGE_ONLY_IF_CVE_CHECKS_PASS="$PUBLISH_DOCKER_IMAGE_ONLY_IF_CVE_CHECKS_PASS" \ + -e CVE_SEVERITY_THRESHOLD="$CVE_SEVERITY_THRESHOLD" \ $argv) function termhandler --on-signal TERM --inherit-variable c if test -n "$c" @@ -1857,6 +1892,10 @@ function interactiveContainer -e PROMTOOL_PATH="$PROMTOOL_PATH" \ -e BUILD_REPO_INFO="$BUILD_REPO_INFO" \ -e ARANGODB_BUILD_DATE="$ARANGODB_BUILD_DATE" \ + -e RUN_CVE_CHECKS_FOR_DOCKER_IMAGE="$RUN_CVE_CHECKS_FOR_DOCKER_IMAGE" \ + -e CREATE_CVE_REPORT_FOR_DOCKER_IMAGE="$CREATE_CVE_REPORT_FOR_DOCKER_IMAGE" \ + -e PUBLISH_DOCKER_IMAGE_ONLY_IF_CVE_CHECKS_PASS="$PUBLISH_DOCKER_IMAGE_ONLY_IF_CVE_CHECKS_PASS" \ + -e CVE_SEVERITY_THRESHOLD="$CVE_SEVERITY_THRESHOLD" \ $argv if test -n "$agentstarted" @@ -2047,6 +2086,73 @@ function unpackBuildFiles runInContainer (eval echo \$UBUNTUBUILDIMAGE_$ARANGODB_VERSION_MAJOR$ARANGODB_VERSION_MINOR) $SCRIPTSDIR/unpackBuildFiles.fish "$argv[1]" end +function installGrype + if not set -q GRYPE_DIR[1] + set -gx GRYPE_DIR "$WORKDIR/work/tools/grype" + end + echo "Installing grype to $GRYPE_DIR" + if test ! -d "$GRYPE_DIR" + mkdir -p "$GRYPE_DIR" + end + curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b $GRYPE_DIR + set -l GRYPE_BIN "$GRYPE_DIR/grype" + $GRYPE_BIN version 2>&1 > /dev/null + if test $status -ne 0 + echo "Failed to install grype" + return 1 + end + set -gx GRYPE_BIN $GRYPE_BIN + echo "Grype is installed successfully" +end + +function downloadOrUpdateGrype + # if grype location is not predefined, set it to default value + if not set -q GRYPE_DIR[1] + set -gx GRYPE_DIR "$WORKDIR/work/tools/grype" + end + if not set -q GRYPE_BIN[1] + set -gx GRYPE_BIN "$GRYPE_DIR/grype" + end + if test -f "$GRYPE_BIN" + $GRYPE_BIN version + if test $status -eq 0 + echo "Grype is already installed. Updating the CVE database" + $GRYPE_BIN db update + if test $status -eq 0 + echo "Grype CVE database is updated successfully" + return 0 + end + echo "Failed to update the Grype CVE database" + return 1 + end + end + + # grype is not installed + installGrype +end + +function checkDockerImageForCves + if not set -q GRYPE_BIN[1] + downloadOrUpdateGrype + if test $status -ne 0 + return 1 + end + end + set -l image $argv[1] + set -l report_file $argv[2] + if not set -q CVE_SEVERITY_THRESHOLD[1] + set CVE_SEVERITY_THRESHOLD "high" + end + echo "scanning image for CVEs: $image" + if set -q report_file[1] + $GRYPE_BIN -f $CVE_SEVERITY_THRESHOLD -s all-layers --file $report_file docker:$image + or return $status + else + $GRYPE_BIN -f $CVE_SEVERITY_THRESHOLD -s all-layers docker:$image + or return $status + end +end + ## ############################################################################# ## set PARALLELISM in a sensible way ## ############################################################################# diff --git a/jenkins/forTestDocker.fish b/jenkins/forTestDocker.fish index a55fd732..f82884f2 100755 --- a/jenkins/forTestDocker.fish +++ b/jenkins/forTestDocker.fish @@ -38,6 +38,7 @@ and downloadStarter and setArchSuffix and set -xg HUB_COMMUNITY "arangodb/arangodb-test:$DOCKER_TAG_JENKINS$archSuffix" and buildDockerImage $HUB_COMMUNITY +and validateDockerImageIfNeeded $HUB_COMMUNITY and docker push $HUB_COMMUNITY and docker tag $HUB_COMMUNITY $GCR_REG_PREFIX$HUB_COMMUNITY and docker push $GCR_REG_PREFIX$HUB_COMMUNITY diff --git a/jenkins/forTestDockerCommunity.fish b/jenkins/forTestDockerCommunity.fish index 13ff6015..a3719c81 100755 --- a/jenkins/forTestDockerCommunity.fish +++ b/jenkins/forTestDockerCommunity.fish @@ -38,6 +38,7 @@ and downloadStarter and setArchSuffix and set -xg HUB_COMMUNITY "arangodb/arangodb-test:$DOCKER_TAG_JENKINS$archSuffix" and buildDockerImage $HUB_COMMUNITY +and validateDockerImageIfNeeded $HUB_COMMUNITY and docker push $HUB_COMMUNITY and docker tag $HUB_COMMUNITY $GCR_REG_PREFIX$HUB_COMMUNITY and docker push $GCR_REG_PREFIX$HUB_COMMUNITY diff --git a/jenkins/forTestDockerEnterprise.fish b/jenkins/forTestDockerEnterprise.fish index 6c6cebb7..f8986e29 100755 --- a/jenkins/forTestDockerEnterprise.fish +++ b/jenkins/forTestDockerEnterprise.fish @@ -40,6 +40,7 @@ and copyRclone "linux" and setArchSuffix and set -xg HUB_ENTERPRISE "arangodb/enterprise-test:$DOCKER_TAG_JENKINS$archSuffix" and buildDockerImage $HUB_ENTERPRISE +and validateDockerImageIfNeeded $HUB_COMMUNITY and docker push $HUB_ENTERPRISE and docker tag $HUB_ENTERPRISE $GCR_REG_PREFIX$HUB_ENTERPRISE and docker push $GCR_REG_PREFIX$HUB_ENTERPRISE diff --git a/jenkins/helper/generate_cve_report.py b/jenkins/helper/generate_cve_report.py new file mode 100644 index 00000000..e90b38c7 --- /dev/null +++ b/jenkins/helper/generate_cve_report.py @@ -0,0 +1,222 @@ +import json +import os +from jinja2 import Environment, BaseLoader +from datetime import datetime +import sys + +REPORT_HTML_TEMPLATE = """ + + + + + + + grype scan report + + + + + + + + + + + + + + + + + +
Scan date{{ scan_date }}
Grype version{{ grype_version }}
CVE database build timestamp{{ db_date }}
+ + + + + + + + + + + + + + {% for scan in scans %} + {% for vulnerability in scan.vulnerabilities %} + {% if vulnerability.severity == "Critical" or vulnerability.severity == "High" %} + + {% if loop.index == 1 %} + + {% endif %} + + + + + + + + + {% endif %} + {% endfor %} + {% endfor %} + +
Image tagsSeverityCVE IDDescriptionArtifact nameArtifact typeArtifact versionFixed version(s)
{{ scan.image_tags }}{{ vulnerability.severity }}{{ vulnerability.id }}{{ vulnerability.description }}{{ vulnerability.artifact_name }}{{ vulnerability.artifact_type }}{{ vulnerability.artifact_version }}{{ vulnerability.fixed_versions }}
+ + + + + + + + + + + + {% for scan in scans %} + {% for vulnerability in scan.vulnerabilities %} + + {% if loop.index == 1 %} + + {% endif %} + {% if vulnerability.severity == "Critical" or vulnerability.severity == "High" %} + + {% else %} + + {% endif %} + + + + + + + + {% endfor %} + {% endfor %} + + + + + + + + + +""" + +if len(sys.argv) != 3: + print("This script generates an HTML report from the JSON files output by the Grype scanner. Files must be named grypeResult*.json.\nUsage: python generate_cve_report.py [path to directory containing JSON files] [name of the HTML report file]") + sys.exit(1) + +results_path = sys.argv[1] +report_filename = sys.argv[2] + +json_results = [] +filenames = [f for f in os.listdir(results_path) if f.startswith("grypeResult") and f.endswith(".json")] +if len(filenames) == 0: + print("No grypeResult*.json files found in the specified directory.") + sys.exit(1) +for filename in filenames: + file_path = os.path.join(results_path, filename) + if os.path.isfile(file_path): + with open(file_path, "r") as file: + json_results.append(json.load(file)) + +report_data = {} +report_data["scan_date"] = json_results[0].get("descriptor", {}).get("timestamp", "") +report_data["grype_version"] = json_results[0].get("descriptor", {}).get("version", "") +report_data["db_date"] = json_results[0].get("descriptor", {}).get("descriptor", {}).get("db", {}).get("status", {}).get("from", {}).get("built", "") +report_data["scans"] = [] + +severity_order = { + "Critical": 0, + "High": 1, + "Medium": 2, + "Low": 3, + "Negligible": 4, + "Unknown": 5, +} + +for result in json_results: + table_entry = {} + table_entry["image_tags"] = "
".join(result["source"]["target"]["tags"]) + table_entry["vulnerabilities"] = [] + for match in result["matches"]: + vulnerability = match["vulnerability"] + artifact = match.get("artifact", {}) + vulnerability_entry = {} + vulnerability_entry["id"] = vulnerability.get("id", "") + vulnerability_entry["dataSource"] = vulnerability.get("dataSource", "") + vulnerability_entry["description"] = vulnerability.get("description", "") + vulnerability_entry["severity"] = vulnerability.get("severity", "") + vulnerability_entry["artifact_name"] = artifact.get("name", "") + vulnerability_entry["artifact_type"] = artifact.get("type", "") + vulnerability_entry["artifact_version"] = artifact.get("version", "") + vulnerability_entry["fixed_versions"] = "
".join( + vulnerability.get("fix", {}).get("versions", []) + ) + table_entry["vulnerabilities"].append(vulnerability_entry) + table_entry["vulnerabilities"] = sorted( + table_entry["vulnerabilities"], + key=lambda x: severity_order.get(x["severity"], 5), + ) + table_entry["row_count_total"] = len(table_entry["vulnerabilities"]) + table_entry["row_count_high_critical"] = len( + [ + v + for v in table_entry["vulnerabilities"] + if v["severity"] in ["Critical", "High"] + ] + ) + report_data["scans"].append(table_entry) + +template = Environment(loader=BaseLoader).from_string(REPORT_HTML_TEMPLATE) +html_output = template.render(report_data) + +with open(report_filename, "w", encoding="utf-8") as file: + file.write(html_output)