src.bart_base/

Author: XinnuoXu

src.bart_base/scripts/preprocess_data.sh

@@ -3,6 +3,8 @@
 JSON_PATH=/scratch/xxu/multi-multi/json/
 DATA_PATH=/scratch/xxu/multi-multi/data/
 
+rm -rf $DATA_PATH/*
+
 python preprocess.py \
 	-mode pretrain_to_data \
 	-raw_path ${JSON_PATH} \

src.bart_base/scripts/preprocess_json.sh

@@ -4,6 +4,8 @@ RAW_PATH=/scratch/xxu/multi-multi/data_single_trunk_1000/multi_
 #RAW_PATH=/home/xx6/Factroid_Summarization/XSum/bbc-summary-no_structure/xsum_
 JSON_PATH=/scratch/xxu/multi-multi/json/multi
 
+rm ${JSON_PATH}*
+
 python preprocess.py \
 	-mode pretrain_to_json \
 	-raw_path ${RAW_PATH} \