update paracrawl download scripts

Author: jbrry

configs/opusfilter/paracrawl_ga-en.yaml

@@ -1,5 +1,5 @@
 common:
-  output_directory: data/ga/paracrawl   
+  output_directory: data/ga/paracrawl/raw   
 
 steps:
   - type: opus_read

scripts/download_handler.py

@@ -23,6 +23,7 @@ def argparser():
             'gdrive',
             'NCI',
             'oscar',
+            'paracrawl',
             'sampleNCI',
         },
         nargs='+',
@@ -35,7 +36,7 @@ def main(argv):
 
     for dataset in args.datasets:
 
-        if dataset in ("conll17", "NCI", "oscar", "sampleNCI"):
+        if dataset in ("conll17", "NCI", "oscar", "paracrawl", "sampleNCI"):
             print(f"Downloading {dataset} data.")
             cmd = script_dir + f"download_{dataset}_data.sh"
             rcmd = subprocess.call(cmd)
@@ -44,7 +45,7 @@ def main(argv):
             print(f"Downloading {dataset} data.")
             # download all files from gdrive
             cmd = script_dir + f"download_{dataset}_data.sh"
-            rcmd = subprocess.call(cmd)
+            rcmd = subprocess.call(cmd, shell=True)
 
             # gather the various files into a common directory
             fcmd = "find data/ga/gdrive/ -maxdepth 3 -type f | python3 scripts/gather_gdrive_data_by_filelist.py"

scripts/download_scripts/download_NCI_data.sh

@@ -8,6 +8,8 @@
 echo "Downloading data from Google Drive ..."
 
 OUTDIR=data/ga/NCI/raw
+OUTFILE=${OUTDIR}/NCI_v2.txt
+
 mkdir -p $OUTDIR
 
 if [[ -n $(rclone lsf "gdrive:Theme A DCU" 2> /dev/null) ]]; then
@@ -21,6 +23,11 @@ else
     fi
 fi
 
-rclone copy "${THEME_A_DCU}/Irish_Data/ForasNaGaeilge/new-extraction/NCI_extracted_v2.txt" $OUTDIR --bwlimit 1000M --transfers 1
+rclone cat \
+    "${THEME_A_DCU}/Irish_Data/ForasNaGaeilge/9MqDsdf834ms2NfS8L2joi7u_NCIv2.vert" \
+    --bwlimit 1000M --transfers 1 | \
+    scripts/extract_text_from_nci_vert.py --document-newline > ${OUTFILE}
+
+bzip2 ${OUTFILE} 
 
 echo "Done"

scripts/download_scripts/download_paracrawl_data.sh

@@ -1,4 +1,4 @@
-#/bin/bash
+#!/bin/bash
 
 echo "Running OpusFilter to collect Opus data ..."
 
@@ -7,6 +7,7 @@ TRG=en
 
 # OPUS corpus name, e.g. 'paracrawl'
 CORPUS=paracrawl
+
 echo "Using corpus $CORPUS"
 mkdir -p data/ga/$CORPUS