-
Notifications
You must be signed in to change notification settings - Fork 249
/
Copy pathrun-ib-pr-matrix.sh
executable file
·138 lines (133 loc) · 6.35 KB
/
run-ib-pr-matrix.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
#!/bin/bash -ex
TEST_FLAVOR=$1
CMS_BOT_DIR=$(cd $(dirname $0) >/dev/null 2>&1; pwd -P)
readarray -t ALL_GPU_TYPES < ${CMS_BOT_DIR}/gpu_flavors.txt
ARTIFACT_DIR="ib-baseline-tests/${RELEASE_FORMAT}/${ARCHITECTURE}/${REAL_ARCH}/matrix${TEST_FLAVOR}-results"
source $CMS_BOT_DIR/jenkins-artifacts
source $CMS_BOT_DIR/pr_testing/_helper_functions.sh
#Run on any machine to see which workflows should be run
if [ "${CHECK_WORKFLOWS}" = "true" ] ; then
echo "${WORKFLOWS}" > ${WORKSPACE}/workflows-${BUILD_ID}.log
send_jenkins_artifacts ${WORKSPACE}/workflows-${BUILD_ID}.log ${ARTIFACT_DIR}/workflows-${BUILD_ID}.log
OPTS=""
case "${TEST_FLAVOR}" in
high_stats ) ;;
nano ) OPTS="-w nano" ;;
* ) if is_in_array "${TEST_FLAVOR}" "${ALL_GPU_TYPES[@]}" ; then
OPTS=$(get_gpu_matrix_args)
fi
;;
esac
REL_WFS=""
if has_jenkins_artifacts ${ARTIFACT_DIR} -d ; then
REL_WFS=$(cmd_jenkins_artifacts ${ARTIFACT_DIR} "cat runall-report-step123*.log 2>/dev/null" | grep '_' | sed 's|_.*||' | tr '\n' ' ')
fi
if [ $(echo "${WORKFLOWS}" | sed 's|.*-l ||;s| .*||' | tr ',' '\n' | grep '^all$' | wc -l) -gt 0 ] ; then
ALL_WFS=$(runTheMatrix.py -n ${OPTS} ${MATRIX_ARGS} | grep -v ' workflows ' | grep '^[1-9][0-9]*\(.[0-9][0-9]*\|\)\s' | sed 's| .*||' | tr '\n' ',' | sed 's|,$||')
WORKFLOWS=$(echo "${WORKFLOWS}" | sed "s|all|${ALL_WFS}|")
fi
runTheMatrix.py -n ${OPTS} ${MATRIX_ARGS} ${WORKFLOWS} | grep -v ' workflows ' | grep '^[1-9][0-9]*\(.[0-9][0-9]*\|\)\s' | sed 's| .*||' > $WORKSPACE/req.wfs
for wf in $(cat $WORKSPACE/req.wfs) ; do
[ $(echo " $REL_WFS " | grep " $wf " | wc -l) -eq 0 ] || continue
WFS="${wf},${WFS}"
done
WFS=$(echo ${WFS} | sed 's|,$||')
if [ "${WFS}" = "" ] ; then
mv ${WORKSPACE}/workflows-${BUILD_ID}.log ${WORKSPACE}/workflows-${BUILD_ID}.done
send_jenkins_artifacts ${WORKSPACE}/workflows-${BUILD_ID}.done ${ARTIFACT_DIR}/workflows-${BUILD_ID}.done
echo "ARTIFACT_DIR=${ARTIFACT_DIR}" > $WORKSPACE/cvmfs-deploy-baseline
echo "CVMFS_SERVER=cms-ci" >> $WORKSPACE/cvmfs-deploy-baseline
exit 0
fi
echo "CHECK_WORKFLOWS=false" > ${WORKSPACE}/rerun.txt
echo "MATRIX_ARGS=${MATRIX_ARGS} -l ${WFS}" >> ${WORKSPACE}/rerun.txt
exit 0
fi
#Actually run runTheMatrix.py for the selected workflows
mkdir -p "$WORKSPACE/matrix-results"
UC_TEST_FLAVOR=$(echo ${TEST_FLAVOR} | tr '[a-z]' '[A-Z]')
[ "${PRODUCTION_RELEASE}" != "true" ] && PRODUCTION_RELEASE="false"
pushd "$WORKSPACE/matrix-results"
NJOBS=$(nproc)
CMD_OPTS=""
if ${PRODUCTION_RELEASE} && cmsDriver.py --help | grep -q '\-\-maxmem_profile' ; then
if [ "${TEST_FLAVOR}" != "rocm" ]; then
CMD_OPTS="--maxmem_profile"
fi
fi
case "${TEST_FLAVOR}" in
high_stats ) CMD_OPTS="-n 500" ; MATRIX_ARGS="-i all ${MATRIX_ARGS}" ;;
threading ) MATRIX_ARGS="-i all -t 4 ${MATRIX_ARGS}" ; let NJOBS=(${NJOBS}/4)+1 ;;
nano ) MATRIX_ARGS="-w nano -i all ${MATRIX_ARGS}" ;;
input ) MATRIX_ARGS="-i all --maxSteps=2 ${MATRIX_ARGS}" ; CMD_OPTS="-n 1 --prefix ${CMS_BOT_DIR}/pr_testing/retry-command.sh" ; export CMS_BOT_RETRY_COUNT=3 ;;
* ) if is_in_array "${TEST_FLAVOR}" "${ALL_GPU_TYPES[@]}" ; then
MATRIX_ARGS="$(get_gpu_matrix_args) ${MATRIX_ARGS}"
fi
;;
esac
[ $(runTheMatrix.py --help | grep 'job-reports' | wc -l) -gt 0 ] && MATRIX_ARGS="--job-reports $MATRIX_ARGS"
[ -f ${CMSSW_RELEASE_BASE}/src/Validation/Performance/python/TimeMemoryJobReport.py ] && CMD_OPTS="${CMD_OPTS} --customise Validation/Performance/TimeMemoryJobReport.customiseWithTimeMemoryJobReport"
if [ "${TEST_FLAVOR}" != "" ] ; then
eval "a=\${EXTRA_MATRIX_COMMAND_ARGS_${UC_TEST_FLAVOR}}"
CMD_OPTS="${CMD_OPTS} ${a}"
else
CMD_OPTS="${CMD_OPTS} ${EXTRA_MATRIX_COMMAND_ARGS}"
fi
[ "${CMD_OPTS}" != "" ] && MATRIX_ARGS="${MATRIX_ARGS} --command ' ${CMD_OPTS}'"
if [ "X$CMS_SITE_OVERRIDE" == "X" ]; then
CMS_SITE_OVERRIDE="local"
fi
eval CMS_PATH=/cvmfs/cms-ib.cern.ch SITECONFIG_PATH=/cvmfs/cms-ib.cern.ch/SITECONF/$CMS_SITE_OVERRIDE runTheMatrix.py -j ${NJOBS} ${MATRIX_ARGS} 2>&1 | tee -a matrixTests.${BUILD_ID}.log
mv runall-report-step123-.log runall-report-step123-.${BUILD_ID}.log
find . -name DQM*.root | sort | sed 's|^./||' > wf_mapping.${BUILD_ID}.txt
ERRORS_FILE=wf_errors.${BUILD_ID}.txt
touch $ERRORS_FILE
set +x
grep "ERROR executing.*" matrixTests.${BUILD_ID}.log | while read line ; do
WF_STEP=$(echo "$line" | sed 's/.* cd //g' | sed 's/_.*step/;/g' | sed 's/_.*$//g')
if ! grep $WF_STEP $ERRORS_FILE; then
echo $WF_STEP >> $ERRORS_FILE
fi
done
set -x
if [[ $CMD_OPTS =~ maxmem_profile ]] ; then
LOG=runall-report-step123-.${BUILD_ID}.log
for WF in $(grep -a '^[1-9][0-9]*' ${LOG} | grep ' Step[0-9]' | sed 's| .*||' | sort | uniq ) ; do
memFile=maxmem_profile_$(echo $WF | cut -d_ -f1).txt
pushd $WF
for log in $(ls step*.log);do
echo ${log} | cut -d_ -f1 >> ${memFile}
grep "Memory Report: " $log | tail -5 >> ${memFile}
done
popd
done
fi
popd
if [ "${UPLOAD_ARTIFACTS}" = "true" ] ; then
[ -f ${LOCALRT}/used-ibeos-sort ] && mv ${LOCALRT}/used-ibeos-sort $WORKSPACE/matrix-results/
echo "${WORKFLOWS}" > ${WORKSPACE}/matrix-results/workflows-${BUILD_ID}.done
set +x
NUM_PROC=$(nproc)
for r in $(find ${WORKSPACE}/matrix-results -name 'step*.root' -type f | grep -v 'inDQM.root$') ; do
while [ $(jobs -p | wc -l) -gt ${NUM_PROC} ] ; do sleep 0.1 ; done
echo "Running edmEventSize and edmProvDump: $r"
(edmEventSize -v $r > $r.edmEventSize || true) &
(edmProvDump $r > $r.edmProvDump || true) &
done
set -x
jobs
wait
send_jenkins_artifacts $WORKSPACE/matrix-results/ ${ARTIFACT_DIR}
echo "ARTIFACT_DIR=${ARTIFACT_DIR}" > $WORKSPACE/cvmfs-deploy-baseline
echo "CVMFS_SERVER=cms-ci" >> $WORKSPACE/cvmfs-deploy-baseline
REL_QUEUE=$(echo ${RELEASE_FORMAT} | sed 's|_X_.*|_X|')
if python3 -V ; then
DEV_QUEUE=$(cd ${CMS_BOT_DIR}; python3 -c 'from releases import CMSSW_DEVEL_BRANCH; print (CMSSW_DEVEL_BRANCH)')
else
DEV_QUEUE=$(cd ${CMS_BOT_DIR}; python -c 'from releases import CMSSW_DEVEL_BRANCH; print (CMSSW_DEVEL_BRANCH)')
fi
if [ "X${REL_QUEUE}" = "X${DEV_QUEUE}" ] ; then
echo "${REL_QUEUE}" > $WORKSPACE/BaselineDevRelease
send_jenkins_artifacts $WORKSPACE/BaselineDevRelease ib-baseline-tests/BaselineDevRelease
fi
fi