train-unity-pytorch-gpu4.pbs

#PBS -lwalltime=12:00:00
#PBS -lselect=1:ncpus=16:mem=96gb:ngpus=4:gpu_type=RTX6000
#PBS -N unity-52-rcs-gpu4

export PT_EXPERIMENT=$PBS_JOBNAME

export PT_DATA_DIR=$TMPDIR/scantensus-data/
export PT_OUTPUT_DIR=$RDS_PROJECT/scantensus/live/matt-output/

module load anaconda3/personal

rsync -rthP $RDS_PROJECT/scantensus/live/scantensus-data/ $PT_DATA_DIR

cd $TMPDIR || exit
rm -rf scantensus-echo
git clone git@github.com:mshunshin/scantensus-echo.git
cd scantensus-echo || exit
git pull
git checkout $PT_EXPERIMENT


(python train-unity-pytorch-ddp.py 0 0 4) &
(python train-unity-pytorch-ddp.py 1 1 4) &
(python train-unity-pytorch-ddp.py 2 2 4) &
(python train-unity-pytorch-ddp.py 3 3 4) &
wait