Skip to content

Commit c514a81

Browse files
jjhenkeljjhenkel
committed
New results dir.
1 parent 3e5ec15 commit c514a81

7 files changed

+223
-0
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
*.tsv
22
datasets/backups
33
trained-models/
4+
final-models/
45
ig-results/
56
debug/
67
*log*.txt

‎final-results/.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
**/predicted_target
2+
**/true_target
3+
**/attacked.txt

‎final-results/extract-single-attacks.sh

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/bin/bash
2+
3+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
4+
5+
echo "attackName,randTargFOneDrop,gradTargFOneDrop,randTargFOneDropCS,gradTargFOneDropCS"
6+
7+
THE_PATH="${DIR}/seq2seq/c2s/java-small/normal-model/depth-1-random-attack/log-normal.txt"
8+
F1_BASE=$(grep -Po 'f1: \d+.\d+' "${THE_PATH}" | awk '{ print $2 }')
9+
10+
THE_PATH_C2S="${DIR}/code2seq/c2s/java-small/normal-model/no-attack/log-normal.txt"
11+
F1_BASE_C2S=$(grep -Po 'F1: \d+.\d+' "${THE_PATH_C2S}" | awk '{ print 100.0*$2 }')
12+
13+
for attack in AddDeadCode InsertPrintStatements RenameFields RenameLocalVariables RenameParameters ReplaceTrueFalse UnrollWhiles WrapTryCatch; do
14+
CLEAN_NAME=${attack%"s"}
15+
RAND_PATH="${DIR}/individual-attacks/c2s-java/normal-model/${attack}/random/log.txt"
16+
GRAD_PATH="${DIR}/individual-attacks/c2s-java/normal-model/${attack}/gradient/log.txt"
17+
F1_RAND=$(grep -Po 'f1: \d+.\d+' "${RAND_PATH}" | awk "{ print $F1_BASE - \$2 }" | sed -e "s/^-0\./0./")
18+
F1_GRAD=$(grep -Po 'f1: \d+.\d+' "${GRAD_PATH}" | awk "{ print $F1_BASE - \$2 }" | sed -e "s/^-0\./0./")
19+
20+
RAND_PATH_C2S="${DIR}/individual-attacks/code2seq/c2s-java/normal-model/${attack}/random/log-normal.txt"
21+
GRAD_PATH_C2S="${DIR}/individual-attacks/code2seq/c2s-java/normal-model/${attack}/gradient/log-normal.txt"
22+
F1_RAND_C2S=$(grep -Po 'F1: \d+.\d+' "${RAND_PATH_C2S}" | awk "{ print $F1_BASE_C2S - 100.0*\$2 }" | sed -e "s/^-0\./0./")
23+
F1_GRAD_C2S=$(grep -Po 'F1: \d+.\d+' "${GRAD_PATH_C2S}" | awk "{ print $F1_BASE_C2S - 100.0*\$2 }" | sed -e "s/^-0\./0./")
24+
printf "%s,%.1f,%.1f,%.1f,%.1f\n" ${CLEAN_NAME} "${F1_RAND}" "${F1_GRAD}" "${F1_RAND_C2S}" "${F1_GRAD_C2S}"
25+
done

‎final-results/gen-graphs.py

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
import sys
2+
3+
4+
if __name__ == '__main__':
5+
first_line = True
6+
CSV_AS_MATRIX = []
7+
for line in sys.stdin:
8+
if first_line:
9+
first_line = False
10+
continue
11+
12+
CSV_AS_MATRIX.append(
13+
[ float(x) if '[' not in x else float(x.split(' ')[0]) for x in line.split(',')[2:] ]
14+
)
15+
16+
print('x,c2sJavaNormal,c2sJavaAug,c2sJavaAdvRand,c2sJavaAdvGrad,csnJavaNormal,csnJavaAug,csnJavaAdvRand,csnJavaAdvGrad,csnPythonNormal,csnPythonAug,csnPythonAdvRand,csnPythonAdvGrad,sriPythonNormal,sriPythonAug,sriPythonAdvRand,sriPythonAdvGrad')
17+
18+
idx = 1
19+
for i in [0, 1, 3, 2, 4]:
20+
print("{},{}".format(
21+
idx,
22+
",".join([ '{:.1f}'.format(x) for x in [
23+
CSV_AS_MATRIX[0][0+i],
24+
CSV_AS_MATRIX[1][0+i],
25+
CSV_AS_MATRIX[2][0+i],
26+
CSV_AS_MATRIX[3][0+i],
27+
CSV_AS_MATRIX[0][5+i],
28+
CSV_AS_MATRIX[1][5+i],
29+
CSV_AS_MATRIX[2][5+i],
30+
CSV_AS_MATRIX[3][5+i],
31+
CSV_AS_MATRIX[0][10+i],
32+
CSV_AS_MATRIX[1][10+i],
33+
CSV_AS_MATRIX[2][10+i],
34+
CSV_AS_MATRIX[3][10+i],
35+
CSV_AS_MATRIX[0][15+i],
36+
CSV_AS_MATRIX[1][15+i],
37+
CSV_AS_MATRIX[2][15+i],
38+
CSV_AS_MATRIX[3][15+i]
39+
]])
40+
))
41+
idx += 1
42+
43+
# print("{},{}".format(
44+
# i+1,
45+
# ",".join([ '{:.1f}'.format(x) for x in [
46+
# CSV_AS_MATRIX[0][0+i],
47+
# (CSV_AS_MATRIX[1][0+i] - CSV_AS_MATRIX[0][0+i]) / CSV_AS_MATRIX[0][0+i] * 100.0,
48+
# (CSV_AS_MATRIX[2][0+i] - CSV_AS_MATRIX[0][0+i]) / CSV_AS_MATRIX[0][0+i] * 100.0,
49+
# (CSV_AS_MATRIX[3][0+i] - CSV_AS_MATRIX[0][0+i]) / CSV_AS_MATRIX[0][0+i] * 100.0,
50+
# CSV_AS_MATRIX[0][5+i],
51+
# (CSV_AS_MATRIX[1][5+i] - CSV_AS_MATRIX[0][5+i]) / CSV_AS_MATRIX[0][5+i] * 100.0,
52+
# (CSV_AS_MATRIX[2][5+i] - CSV_AS_MATRIX[0][5+i]) / CSV_AS_MATRIX[0][5+i] * 100.0,
53+
# (CSV_AS_MATRIX[3][5+i] - CSV_AS_MATRIX[0][5+i]) / CSV_AS_MATRIX[0][5+i] * 100.0,
54+
# CSV_AS_MATRIX[0][10+i],
55+
# (CSV_AS_MATRIX[1][10+i] - CSV_AS_MATRIX[0][10+i]) / CSV_AS_MATRIX[0][10+i] * 100.0,
56+
# (CSV_AS_MATRIX[2][10+i] - CSV_AS_MATRIX[0][10+i]) / CSV_AS_MATRIX[0][10+i] * 100.0,
57+
# (CSV_AS_MATRIX[3][10+i] - CSV_AS_MATRIX[0][10+i]) / CSV_AS_MATRIX[0][10+i] * 100.0,
58+
# CSV_AS_MATRIX[0][15+i],
59+
# (CSV_AS_MATRIX[1][15+i] - CSV_AS_MATRIX[0][15+i]) / CSV_AS_MATRIX[0][15+i] * 100.0,
60+
# (CSV_AS_MATRIX[2][15+i] - CSV_AS_MATRIX[0][15+i]) / CSV_AS_MATRIX[0][15+i] * 100.0,
61+
# (CSV_AS_MATRIX[3][15+i] - CSV_AS_MATRIX[0][15+i]) / CSV_AS_MATRIX[0][15+i] * 100.0
62+
# ]])
63+
# ))

‎final-results/make-csvs-code2seq.sh

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#!/bin/bash
2+
3+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
4+
5+
echo -n "modelName,trainingType"
6+
echo -n ",ctsJavaSmallId,ctsJavaSmallQOneR,ctsJavaSmallQOneG,ctsJavaSmallQFiveR,ctsJavaSmallQFiveG"
7+
echo -n ",csnJavaId,csnJavaQOneR,csnJavaQOneG,csnJavaQFiveR,csnJavaQFiveG"
8+
echo -n ",csnPythonId,csnPythonQOneR,csnPythonQOneG,csnPythonQFiveR,csnPythonQFiveG"
9+
echo ",sriPyId,sriPyQOneR,sriPyQOneG,sriPyQFiveR,sriPyQFiveG"
10+
11+
for MODEL in normal-model augmented-model adv-random-model adv-gradient-model; do
12+
if [ "${MODEL}" = "normal-model" ]; then
13+
FULL_STR='code2seq,$\nrmTrain{}$,'
14+
elif [ "${MODEL}" = "augmented-model" ]; then
15+
FULL_STR=',$\augTrain{}$,'
16+
elif [ "${MODEL}" = "adv-random-model" ]; then
17+
FULL_STR=',$\advTrain{\testRandDepthOne{}}$,'
18+
else
19+
FULL_STR=',$\advTrain{\testGradDepthOne{}}$,'
20+
fi
21+
22+
for DATASET in c2s/java-small csn/java csn/python sri/py150; do
23+
for ATTACK in no-attack depth-1-random-attack depth-1-gradient-attack depth-5-random-attack depth-5-gradient-attack; do
24+
if [ "${ATTACK}" = "no-attack" ]; then
25+
THE_PATH="${DIR}/code2seq/${DATASET}/${MODEL}/no-attack/log-normal.txt"
26+
27+
F1_NORM=0.0
28+
if [ -f "${THE_PATH}" ]; then
29+
F1_NORM=$(grep -Po 'F1: \d+.\d+' "${THE_PATH}" | awk '{ print $2*100.0 }')
30+
else
31+
echo "MISSING: ${THE_PATH}"
32+
fi
33+
34+
FULL_STR+=$(printf %.1f, ${F1_NORM})
35+
else
36+
THE_PATH="${DIR}/code2seq/${DATASET}/${MODEL}/${ATTACK}/attack_metrics.txt"
37+
ALT_PATH="${DIR}/code2seq/${DATASET}/normal-model/${ATTACK}/attack_metrics.txt"
38+
39+
F1_ATTACK=0.0
40+
DELTA=0.0
41+
if [ -f "${THE_PATH}" ]; then
42+
F1_ATTACK=$(grep -Po 'f1"?: \d+.\d+' "${THE_PATH}" | awk '{ print $2 }')
43+
DELTA=$(grep -Po 'f1"?: \d+.\d+' "${ALT_PATH}" | awk "{ print $F1_ATTACK - \$2 }")
44+
else
45+
echo "MISSING: ${THE_PATH}"
46+
fi
47+
48+
if [ "${MODEL}" = "normal-model" ]; then
49+
FULL_STR+=$(printf "%.1f", ${F1_ATTACK})
50+
else
51+
FULL_STR+=$(printf "%.1f [+%.1f]", ${F1_ATTACK} ${DELTA})
52+
fi
53+
fi
54+
done
55+
done
56+
57+
TEMP=${FULL_STR::-1}
58+
echo ${TEMP//+-/-}
59+
done

‎final-results/make-csvs-seq2seq.sh

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#!/bin/bash
2+
3+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
4+
5+
echo -n "modelName,trainingType"
6+
echo -n ",ctsJavaSmallId,ctsJavaSmallQOneR,ctsJavaSmallQOneG,ctsJavaSmallQFiveR,ctsJavaSmallQFiveG"
7+
echo -n ",csnJavaId,csnJavaQOneR,csnJavaQOneG,csnJavaQFiveR,csnJavaQFiveG"
8+
echo -n ",csnPythonId,csnPythonQOneR,csnPythonQOneG,csnPythonQFiveR,csnPythonQFiveG"
9+
echo ",sriPyId,sriPyQOneR,sriPyQOneG,sriPyQFiveR,sriPyQFiveG"
10+
11+
for MODEL in normal-model augmented-model adv-random-model adv-gradient-model; do
12+
if [ "${MODEL}" = "normal-model" ]; then
13+
FULL_STR='seq2seq,$\nrmTrain{}$,'
14+
elif [ "${MODEL}" = "augmented-model" ]; then
15+
FULL_STR=',$\augTrain{}$,'
16+
elif [ "${MODEL}" = "adv-random-model" ]; then
17+
FULL_STR=',$\advTrain{\testRandDepthOne{}}$,'
18+
else
19+
FULL_STR=',$\advTrain{\testGradDepthOne{}}$,'
20+
fi
21+
22+
for DATASET in c2s/java-small csn/java csn/python sri/py150; do
23+
for ATTACK in no-attack depth-1-random-attack depth-1-gradient-attack depth-5-random-attack depth-5-gradient-attack; do
24+
if [ "${ATTACK}" = "no-attack" ]; then
25+
THE_PATH="${DIR}/seq2seq/${DATASET}/${MODEL}/depth-1-random-attack/log-normal.txt"
26+
27+
F1_NORM=0.0
28+
if [ -f "${THE_PATH}" ]; then
29+
F1_NORM=$(grep -Po 'f1: \d+.\d+' "${THE_PATH}" | awk '{ print $2 }')
30+
else
31+
echo "MISSING: ${THE_PATH}"
32+
fi
33+
34+
FULL_STR+=$(printf %.1f, ${F1_NORM})
35+
else
36+
THE_PATH="${DIR}/seq2seq/${DATASET}/${MODEL}/${ATTACK}/attacked_metrics.txt"
37+
ALT_PATH="${DIR}/seq2seq/${DATASET}/normal-model/${ATTACK}/attacked_metrics.txt"
38+
39+
F1_ATTACK=0.0
40+
DELTA=0.0
41+
if [ -f "${THE_PATH}" ]; then
42+
F1_ATTACK=$(grep -Po 'f1"?: \d+.\d+' "${THE_PATH}" | awk '{ print $2 }')
43+
DELTA=$(grep -Po 'f1"?: \d+.\d+' "${ALT_PATH}" | awk "{ print $F1_ATTACK - \$2 }")
44+
else
45+
echo "MISSING: ${THE_PATH}"
46+
fi
47+
48+
if [ "${MODEL}" = "normal-model" ]; then
49+
FULL_STR+=$(printf "%.1f", ${F1_ATTACK})
50+
else
51+
FULL_STR+=$(printf "%.1f [+%.1f]", ${F1_ATTACK} ${DELTA})
52+
fi
53+
fi
54+
done
55+
done
56+
57+
TEMP=${FULL_STR::-1}
58+
echo ${TEMP//+-/-}
59+
done

‎final-results/regen-all.sh

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash
2+
3+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
4+
5+
rm -f "${DIR}/finalized/*.csv"
6+
7+
"${DIR}/make-csvs-seq2seq.sh" | grep -v MISSING > "${DIR}/finalized/table-seq2seq.csv"
8+
"${DIR}/make-csvs-code2seq.sh" | grep -v MISSING > "${DIR}/finalized/table-code2seq.csv"
9+
10+
cat "${DIR}/finalized/table-seq2seq.csv" | grep -v MISSING | python3 "${DIR}/gen-graphs.py" > "${DIR}/finalized/graph-seq2seq.csv"
11+
cat "${DIR}/finalized/table-code2seq.csv" | grep -v MISSING | python3 "${DIR}/gen-graphs.py" > "${DIR}/finalized/graph-code2seq.csv"
12+
13+
"${DIR}/extract-single-attacks.sh" > "${DIR}/finalized/individual-attacks.csv"

0 commit comments

Comments
 (0)