-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathrun.sh
executable file
·138 lines (127 loc) · 5.14 KB
/
run.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
#!/bin/bash
if [ $# -ne 1 ]; then
echo -e "Usage: $0 <all | network | pytorch | mathlib | thread | gpu | seeds-cpu | seeds-gpu | seeds>"
echo -e "\tRun the experiment set given in the argument"
exit
fi
if [[ $1 = "all" || $1 = "network" ]]; then
for NETWORK_VERSION in $( docker images --filter=reference='snapbug/qqa:sha-*' --format "{{.Repository}}:{{.Tag}}" )
do
bn=$( basename ${NETWORK_VERSION} )
for dataset in "TrecQA" "WikiQA"
do
docker run -it --name qqa ${NETWORK_VERSION} sh -c "python main.py qqa.${dataset}.model.${bn} --paper-ext-feats --num_threads=1 --dataset_folder=../../data/${dataset}"
docker logs qqa > qqa.${dataset}.log.${bn}
docker cp qqa:/castorini/castor/sm_cnn/qqa.${dataset}.model.${bn}
docker rm qqa
done
done
fi
if [[ $1 = "all" || $1 = "pytorch" ]]; then
for PYTORCH_VERSION in $( docker images --filter=reference='snapbug/qqa:pytorch-*' --format "{{.Repository}}:{{.Tag}}" )
do
bn=$( basename ${PYTORCH_VERSION} )
for dataset in "TrecQA" "WikiQA"
do
docker run -it --name qqa ${PYTORCH_VERSION} sh -c "python main.py qqa.${dataset}.model.${bn} --paper-ext-feats --num_threads=1 --dataset_folder=../../data/${dataset}"
docker logs qqa > qqa.${dataset}.log.${bn}
docker cp qqa:/castorini/castor/sm_cnn/qqa.${dataset}.model.${bn}
docker rm qqa
done
done
fi
if [[ $1 = "all" || $1 = "mathlib" ]]; then
for MATH_LIB in $( docker images --filter=reference='snapbug/qqa:*mkl' --format "{{.Repository}}:{{.Tag}}" )
do
bn=$( basename ${MATH_LIB} )
for dataset in "TrecQA" "WikiQA"
do
docker run -it --name qqa ${MATH_LIB} sh -c "python main.py qqa.${dataset}.model.${bn} --paper-ext-feats --num_threads=1 --dataset_folder=../../data/${dataset}"
docker logs qqa > qqa.${dataset}.log.${bn}
docker cp qqa:/castorini/castor/sm_cnn/qqa.${dataset}.model.${bn}
docker rm qqa
done
done
fi
if [[ $1 = "all" || $1 = "thread" ]]; then
for threads in `seq 1 6`
do
for dataset in "TrecQA" "WikiQA"
do
docker run -it --name qqa snapbug/qqa:sha-cf0e269 sh -c "OMP_NUM_THREADS=${threads} MKL_NUM_THREADS=${threads} python main.py qqa.${dataset}.model.threads-${threads} --paper-ext-feats --num_threads=${threads} --dataset_folder=../../data/${dataset}"
docker logs qqa > qqa.${dataset}.log.threads-${threads}
docker cp qqa:/castorini/castor/sm_cnn/qqa.${dataset}.model.threads-${threads} .
docker rm qqa
done
done
fi
if [[ $1 = "all" || $1 = "seeds-cpu" || $1 = "seeds" ]]; then
RANDOM=1234
reps=0
# 200 reps, 2 datasets, 400 repetitions ...
# because we check whether a seed has been repeated by checking for a log file, which changes name based on the dataset
while [ ${reps} -lt 400 ]
do
seed=$(( RANDOM ))
for dataset in "TrecQA" "WikiQA"
do
if [ ! -f qqa.${dataset}.log.cpu.seed-${seed} ]
then
docker run -it --name qqa snapbug/qqa:sha-cf0e269 sh -c "python main.py qqa.${dataset}.model.cpu-seed-${seed} --paper-ext-feats --num_threads=1 --seed ${seed} --dataset_folder=../../data/${dataset}"
docker logs qqa > qqa.${dataset}.log.cpu-seed-${seed}
docker cp qqa:castorini/castor/sm_cnn/qqa.${dataset}.model.cpu.seed-${seed} .
docker rm qqa
reps=$[$reps + 1]
fi
done
done
fi
GPU_DOCKER=
GPU_RUNTIME=
if [[ $1 = "all" || $1 = "gpu" || $1 = "seeds-gpu" || $1 = "seeds" ]]; then
if [ $( command -v nvidia-docker ) ]; then
echo Found \`nvidia-docker\` for GPU experiments.
GPU_DOCKER=nvidia-docker
elif [ $( docker run --runtime=nvida hello-world 1>&2 2>/dev/null ) ]; then
echo Found \`--runtime=nvidia\` option for docker, using this for GPU experiments, but it may not match!
GPU_DOCKER=docker
GPU_RUNTIME=--runtime=nvidia
fi
if [[ -z "$GPU_DOCKER" && -z "$GPU_RUNTIME" ]]; then
echo No suitable GPU runtime found, skipping GPU experiments
exit
fi
fi
if [[ $1 = "all" || $1 = "gpu" ]]; then
for cudnn in "" "--nocudnn"
do
for dataset in "TrecQA" "WikiQA"
do
${GPU_DOCKER} run ${GPU_RUNTIME} -it --name qqa snapbug/qqa:sha-cf0e269 sh -c "python main.py qqa.${dataset}.model.gpu${cudnn} --paper-ext-feats --cuda ${cudnn} --dataset_folder=../../data/${dataset}"
${GPU_DOCKER} logs ${GPU_RUNTIME} qqa > qqa.${dataset}.log.gpu${cudnn}
${GPU_DOCKER} cp ${GPU_RUNTIME} qqa:/castorini/castor/sm_cnn/qqa.${dataset}.model.gpu${cudnn} .
${GPU_DOCKER} rm ${GPU_RUNTIME} qqa
done
done
fi
if [[ $1 = "all" || $1 = "seeds-gpu" || $1 = "seeds" ]]; then
RANDOM=1234
reps=0
# 200 reps, 2 datasets, 400 repetitions ...
# because we check whether a seed has been repeated by checking for a log file, which changes name based on the dataset
while [ ${reps} -lt 400 ]
do
seed=$(( RANDOM ))
for dataset in "TrecQA" "WikiQA"
do
if [ ! -f qqa.${dataset}.log.gpu.seed-${seed} ]
then
${GPU_DOCKER} run ${GPU_RUNTIME} -it --name qqa snapbug/qqa:sha-cf0e269 sh -c "python main.py qqa.${dataset}.model.gpu-seed-${seed} --paper-ext-feats --cuda --nocudnn --seed ${seed} --dataset_folder=../../data/${dataset}"
${GPU_DOCKER} logs ${GPU_RUNTIME} qqa > qqa.${dataset}.log.gpu-seed-${seed}
${GPU_DOCKER} cp ${GPU_RUNTIME} qqa:castorini/castor/sm_cnn/qqa.${dataset}.model.gpu-seed-${seed} .
${GPU_DOCKER} rm ${GPU_RUNTIME} qqa
reps=$[$reps + 1]
fi
done
done
fi