diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..94578513 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,196 @@ +version: 2 +jobs: + build: + machine: + image: circleci/classic:201808-01 + working_directory: /tmp/src/nitransforms + environment: + TZ: "/usr/share/zoneinfo/America/Los_Angeles" + SCRATCH: "/scratch" + + steps: + - restore_cache: + keys: + - build-v1-{{ .Branch }}-{{ epoch }} + - build-v1-{{ .Branch }}- + - build-v1-master- + - build-v1- + paths: + - /tmp/docker + - run: + name: Set-up a Docker registry + command: | + docker run -d -p 5000:5000 --restart=always --name=registry \ + -v /tmp/docker:/var/lib/registry registry:2 + - run: + name: Pull existing images + command: | + set +e + docker pull localhost:5000/ubuntu + success=$? + set -e + if [[ "$success" = "0" ]]; then + echo "Pulling from local registry" + docker tag localhost:5000/ubuntu ubuntu:xenial-20200114 + docker pull localhost:5000/nitransforms + docker tag localhost:5000/nitransforms nitransforms:latest + else + echo "Pulling from Docker Hub" + docker pull ubuntu:xenial-20200114 + docker tag ubuntu:xenial-20200114 localhost:5000/ubuntu + docker push localhost:5000/ubuntu + fi + - checkout + - run: + name: Build Docker image & push to registry + no_output_timeout: 60m + command: | + e=1 && for i in {1..5}; do + docker build --rm --cache-from=nitransforms:latest \ + -t nitransforms:latest \ + --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \ + --build-arg VCS_REF=`git rev-parse --short HEAD` . \ + && e=0 && break || sleep 15 + done && [ "$e" -eq "0" ] + docker tag nitransforms:latest localhost:5000/nitransforms + docker push localhost:5000/nitransforms + - run: + name: Docker registry garbage collection + command: | + docker exec -it registry /bin/registry garbage-collect --delete-untagged \ + /etc/docker/registry/config.yml + - save_cache: + key: build-v1-{{ .Branch }}-{{ epoch }} + paths: + - /tmp/docker + - run: + name: Store FreeSurfer license file + command: | + mkdir -p /tmp/fslicense + cd /tmp/fslicense + echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IGxpY2Vuc2UudHh0Cg==" | base64 -d | sh + - persist_to_workspace: + root: /tmp + paths: + - fslicense + + test_pytest: + machine: + image: circleci/classic:201808-01 + environment: + FS_LICENSE: "/tmp/fslicense/license.txt" + working_directory: /tmp/tests + steps: + - attach_workspace: + at: /tmp + - checkout: + path: /tmp/src/nitransforms + - run: + name: Get codecov + command: python -m pip install codecov + - restore_cache: + keys: + - build-v1-{{ .Branch }}-{{ epoch }} + - build-v1-{{ .Branch }}- + - build-v1-master- + - build-v1- + - run: + name: Set-up a Docker registry & pull + command: | + docker run -d -p 5000:5000 --restart=always --name=registry \ + -v /tmp/docker:/var/lib/registry registry:2 + docker pull localhost:5000/nitransforms + docker tag localhost:5000/nitransforms nitransforms:latest + - run: + name: Run unit tests + no_output_timeout: 2h + command: | + mkdir -p $PWD/artifacts $PWD/summaries + sudo setfacl -d -m group:ubuntu:rwx $PWD + sudo setfacl -m group:ubuntu:rwx $PWD + docker run -u $( id -u ) -it --rm=false -w /src/nitransforms \ + -e COVERAGE_FILE=/tmp/summaries/.pytest.coverage \ + -v /tmp/fslicense/license.txt:/opt/freesurfer/license.txt:ro \ + -v ${PWD}:/tmp nitransforms:latest \ + pytest --junit-xml=/tmp/summaries/pytest.xml \ + --cov nitransforms --cov-report xml:/tmp/summaries/unittests.xml \ + nitransforms/ + - run: + name: Submit unit test coverage + command: | + cd /tmp/src/nitransforms + python -m codecov --file /tmp/tests/summaries/unittests.xml \ + --flags unittests -e CIRCLE_JOB + - run: + name: Clean up tests directory + when: on_success + command: | + rm -rf /tmp/tests/pytest-of-root + - store_artifacts: + path: /tmp/tests/artifacts + + - store_test_results: + path: /tmp/tests/summaries/ + + test_packaging_and_deploy: + machine: + image: circleci/classic:201808-01 + working_directory: /tmp/src/nitransforms + steps: + - checkout + - run: pyenv local 3.7.0 + - run: + name: Install build depends + command: python3 -m pip install "setuptools>=30.4.0" "pip>=10.0.1" "twine<2.0" docutils + - run: + name: Build and check + command: | + python3 setup.py check -r -s + python3 setup.py sdist + python3 -m twine check dist/* + - run: + name: Validate version + command: | + THISVERSION=$( python3 get_version.py ) + python3 -m pip install dist/*.tar.gz + mkdir empty + cd empty + INSTALLED=$( python3 -c 'import nitransforms; print(nitransforms.__version__)' ) + test "${CIRCLE_TAG:-$THISVERSION}" == "$INSTALLED" + - run: + name: Upload to PyPi + command: | + python3 -m twine upload dist/* + + +workflows: + version: 2 + build_test_deploy: + jobs: + - build: + filters: + branches: + ignore: + - /docs?\/.*/ + tags: + only: /.*/ + + - test_pytest: + requires: + - build + filters: + branches: + ignore: + - /docs?\/.*/ + - /docker\/.*/ + tags: + only: /.*/ + + - test_packaging_and_deploy: + requires: + - test_pytest + filters: + branches: + ignore: /.*/ + tags: + only: /.*/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..84407998 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,155 @@ +FROM ubuntu:xenial-20200114 + +# Pre-cache neurodebian key +COPY docker/files/neurodebian.gpg /usr/local/etc/neurodebian.gpg + +# Prepare environment +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + curl \ + bzip2 \ + ca-certificates \ + xvfb \ + build-essential \ + autoconf \ + libtool \ + pkg-config \ + git && \ + curl -sL https://deb.nodesource.com/setup_10.x | bash - && \ + apt-get install -y --no-install-recommends \ + nodejs && \ + apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Installing freesurfer +RUN curl -sSL https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.1/freesurfer-Linux-centos6_x86_64-stable-pub-v6.0.1.tar.gz | tar zxv --no-same-owner -C /opt \ + --exclude='freesurfer/diffusion' \ + --exclude='freesurfer/docs' \ + --exclude='freesurfer/fsfast' \ + --exclude='freesurfer/lib/cuda' \ + --exclude='freesurfer/lib/qt' \ + --exclude='freesurfer/matlab' \ + --exclude='freesurfer/mni/share/man' \ + --exclude='freesurfer/subjects/fsaverage_sym' \ + --exclude='freesurfer/subjects/fsaverage3' \ + --exclude='freesurfer/subjects/fsaverage4' \ + --exclude='freesurfer/subjects/cvs_avg35' \ + --exclude='freesurfer/subjects/cvs_avg35_inMNI152' \ + --exclude='freesurfer/subjects/bert' \ + --exclude='freesurfer/subjects/lh.EC_average' \ + --exclude='freesurfer/subjects/rh.EC_average' \ + --exclude='freesurfer/subjects/sample-*.mgz' \ + --exclude='freesurfer/subjects/V1_average' \ + --exclude='freesurfer/trctrain' + +ENV FSL_DIR="/usr/share/fsl/5.0" \ + OS="Linux" \ + FS_OVERRIDE=0 \ + FIX_VERTEX_AREA="" \ + FSF_OUTPUT_FORMAT="nii.gz" \ + FREESURFER_HOME="/opt/freesurfer" +ENV SUBJECTS_DIR="$FREESURFER_HOME/subjects" \ + FUNCTIONALS_DIR="$FREESURFER_HOME/sessions" \ + MNI_DIR="$FREESURFER_HOME/mni" \ + LOCAL_DIR="$FREESURFER_HOME/local" \ + MINC_BIN_DIR="$FREESURFER_HOME/mni/bin" \ + MINC_LIB_DIR="$FREESURFER_HOME/mni/lib" \ + MNI_DATAPATH="$FREESURFER_HOME/mni/data" +ENV PERL5LIB="$MINC_LIB_DIR/perl5/5.8.5" \ + MNI_PERL5LIB="$MINC_LIB_DIR/perl5/5.8.5" \ + PATH="$FREESURFER_HOME/bin:$FSFAST_HOME/bin:$FREESURFER_HOME/tktools:$MINC_BIN_DIR:$PATH" + +# Installing Neurodebian packages (FSL, AFNI, git) +RUN curl -sSL "http://neuro.debian.net/lists/$( lsb_release -c | cut -f2 ).us-ca.full" >> /etc/apt/sources.list.d/neurodebian.sources.list && \ + apt-key add /usr/local/etc/neurodebian.gpg && \ + (apt-key adv --refresh-keys --keyserver hkp://ha.pool.sks-keyservers.net 0xA5D32F012649A5A9 || true) + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + fsl-core=5.0.9-5~nd16.04+1 \ + fsl-mni152-templates=5.0.7-2 \ + afni=16.2.07~dfsg.1-5~nd16.04+1 \ + convert3d \ + connectome-workbench=1.3.2-2~nd16.04+1 \ + git-annex-standalone && \ + apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +ENV FSLDIR="/usr/share/fsl/5.0" \ + FSLOUTPUTTYPE="NIFTI_GZ" \ + FSLMULTIFILEQUIT="TRUE" \ + POSSUMDIR="/usr/share/fsl/5.0" \ + LD_LIBRARY_PATH="/usr/lib/fsl/5.0:$LD_LIBRARY_PATH" \ + FSLTCLSH="/usr/bin/tclsh" \ + FSLWISH="/usr/bin/wish" \ + AFNI_MODELPATH="/usr/lib/afni/models" \ + AFNI_IMSAVE_WARNINGS="NO" \ + AFNI_TTATLAS_DATASET="/usr/share/afni/atlases" \ + AFNI_PLUGINPATH="/usr/lib/afni/plugins" +ENV PATH="/usr/lib/fsl/5.0:/usr/lib/afni/bin:$PATH" + +# Installing ANTs 2.2.0 (NeuroDocker build) +ENV ANTSPATH=/usr/lib/ants +RUN mkdir -p $ANTSPATH && \ + curl -sSL "https://dl.dropbox.com/s/2f4sui1z6lcgyek/ANTs-Linux-centos5_x86_64-v2.2.0-0740f91.tar.gz" \ + | tar -xzC $ANTSPATH --strip-components 1 +ENV PATH=$ANTSPATH:$PATH + +# Installing and setting up miniconda +RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh && \ + bash Miniconda3-4.5.11-Linux-x86_64.sh -b -p /usr/local/miniconda && \ + rm Miniconda3-4.5.11-Linux-x86_64.sh + +# Set CPATH for packages relying on compiled libs (e.g. indexed_gzip) +ENV PATH="/usr/local/miniconda/bin:$PATH" \ + CPATH="/usr/local/miniconda/include/:$CPATH" \ + LANG="C.UTF-8" \ + LC_ALL="C.UTF-8" \ + PYTHONNOUSERSITE=1 + +# Installing precomputed python packages +RUN conda install -y python=3.7.1 \ + pip=19.1 \ + mkl=2018.0.3 \ + mkl-service \ + numpy=1.15.4 \ + scipy=1.1.0 \ + libxml2=2.9.8 \ + libxslt=1.1.32 \ + zlib; sync && \ + chmod -R a+rX /usr/local/miniconda; sync && \ + chmod +x /usr/local/miniconda/bin/*; sync && \ + conda build purge-all; sync && \ + conda clean -tipsy && sync + +# Unless otherwise specified each process should only use one thread - nipype +# will handle parallelization +ENV MKL_NUM_THREADS=1 \ + OMP_NUM_THREADS=1 + +# Create a shared $HOME directory +RUN useradd -m -s /bin/bash -G users neuro +WORKDIR /home/neuro +ENV HOME="/home/neuro" + +# Install package +COPY . /src/nitransforms +ARG VERSION +# Force static versioning within container +RUN echo "${VERSION}" > /src/nitransforms/nitransforms/VERSION && \ + echo "include nitransforms/VERSION" >> /src/nitransforms/MANIFEST.in && \ + pip install --no-cache-dir "/src/nitransforms[all]" + +RUN find $HOME -type d -exec chmod go=u {} + && \ + find $HOME -type f -exec chmod go=u {} + + + +RUN ldconfig +WORKDIR /tmp/ + +ARG BUILD_DATE +ARG VCS_REF +ARG VERSION +LABEL org.label-schema.build-date=$BUILD_DATE \ + org.label-schema.name="nitransforms" \ + org.label-schema.vcs-url="https://github.com/poldracklab/nitransforms" \ + org.label-schema.version=$VERSION \ + org.label-schema.schema-version="1.0" diff --git a/docker/files/neurodebian.gpg b/docker/files/neurodebian.gpg new file mode 100644 index 00000000..c546d45d --- /dev/null +++ b/docker/files/neurodebian.gpg @@ -0,0 +1,71 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 + +mQGiBEQ7TOgRBADvaRsIZ3VZ6Qy7PlDpdMm97m0OfvouOj/HhjOM4M3ECbGn4cYh +vN1gK586s3sUsUcNQ8LuWvNsYhxYsVTZymCReJMEDxod0U6/z/oIbpWv5svF3kpl +ogA66Ju/6cZx62RiCSOkskI6A3Waj6xHyEo8AGOPfzbMoOOQ1TS1u9s2FwCgxziL +wADvKYlDZnWM03QtqIJVD8UEAOks9Q2OqFoqKarj6xTRdOYIBVEp2jhozZUZmLmz +pKL9E4NKGfixqxdVimFcRUGM5h7R2w7ORqXjCzpiPmgdv3jJLWDnmHLmMYRYQc8p +5nqo8mxuO3zJugxBemWoacBDd1MJaH7nK20Hsk9L/jvU/qLxPJotMStTnwO+EpsK +HlihA/9ZpvzR1QWNUd9nSuNR3byJhaXvxqQltsM7tLqAT4qAOJIcMjxr+qESdEbx +NHM5M1Y21ZynrsQw+Fb1WHXNbP79vzOxHoZR0+OXe8uUpkri2d9iOocre3NUdpOO +JHtl6cGGTFILt8tSuOVxMT/+nlo038JQB2jARe4B85O0tkPIPbQybmV1cm8uZGVi +aWFuLm5ldCBhcmNoaXZlIDxtaWNoYWVsLmhhbmtlQGdtYWlsLmNvbT6IRgQQEQgA +BgUCTVHJKwAKCRCNEUVjdcAkyOvzAJ0abJz+f2a6VZG1c9T8NHMTYh1atwCgt0EE +3ZZd/2in64jSzu0miqhXbOKISgQQEQIACgUCSotRlwMFAXgACgkQ93+NsjFEvg8n +JgCfWcdJbILBtpLZCocvOzlLPqJ0Fn0AoI4EpJRxoUnrtzBGUC1MqecU7WsDiGAE +ExECACAFAkqLUWcCGwMGCwkIBwMCBBUCCAMEFgIDAQIeAQIXgAAKCRCl0y8BJkml +qVklAJ4h2V6MdQkSAThF5c2Gkq6eSoIQYQCeM0DWyB9Bl+tTPSTYXwwZi2uoif20 +QmFwc3kuZ3NlLnVuaS1tYWdkZWJ1cmcuZGUgRGViaWFuIEFyY2hpdmUgPG1pY2hh +ZWwuaGFua2VAZ21haWwuY29tPohGBBARAgAGBQJEO03FAAoJEPd/jbIxRL4PU18A +n3tn7i4qdlMi8kHbYWFoabsKc9beAJ9sl/leZNCYNMGhz+u6BQgyeLKw94heBBMR +AgAeBQJEO0zoAhsDBgsJCAcDAgMVAgMDFgIBAh4BAheAAAoJEKXTLwEmSaWpVdoA +n27DvtZizNEbhz3wRUPQMiQjtqdvAJ9rS9YdPe5h5o5gHx3mw3BSkOttdYheBBMR +AgAeBQJEO0zoAhsDBgsJCAcDAgMVAgMDFgIBAh4BAheAAAoJEKXTLwEmSaWpVdoA +oLhwWL+E+2I9lrUf4Lf26quOK9vLAKC9ZpIF2tUirFFkBWnQvu13/TA0SokCHAQQ +AQIABgUCTSNBgQAKCRDAc9Iof/uem4NpEACQ8jxmaCaS/qk/Y4GiwLA5bvKosG3B +iARZ2v5UWqCZQ1tS56yKse/lCIzXQqU9BnYW6wOI2rvFf9meLfd8h96peG6oKscs +fbclLDIf68bBvGBQaD0VYFi/Fk/rxmTQBOCQ3AJZs8O5rIM4gPGE0QGvSZ1h7VRw +3Uyeg4jKXLIeJn2xEmOJgt3auAR2FyKbzHaX9JCoByJZ/eU23akNl9hgt7ePlpXo +74KNYC58auuMUhCq3BQDB+II4ERYMcmFp1N5ZG05Cl6jcaRRHDXz+Ax6DWprRI1+ +RH/Yyae6LmKpeJNwd+vM14aawnNO9h8IAQ+aJ3oYZdRhGyybbin3giJ10hmWveg/ +Pey91Nh9vBCHdDkdPU0s9zE7z/PHT0c5ccZRukxfZfkrlWQ5iqu3V064ku5f4PBy +8UPSkETcjYgDnrdnwqIAO+oVg/SFlfsOzftnwUrvwIcZlXAgtP6MEEAs/38e/JIN +g4VrpdAy7HMGEUsh6Ah6lvGQr+zBnG44XwKfl7e0uCYkrAzUJRGM5vx9iXvFMcMu +jv9EBNNBOU8/Y6MBDzGZhgaoeI27nrUvaveJXjAiDKAQWBLjtQjINZ8I9uaSGOul +8kpbFavE4eS3+KhISrSHe4DuAa3dk9zI+FiPvXY1ZyfQBtNpR+gYFY6VxMbHhY1U +lSLHO2eUIQLdYbRITmV1cm9EZWJpYW4gQXJjaGl2ZSBLZXkgPHBrZy1leHBwc3kt +bWFpbnRhaW5lcnNAbGlzdHMuYWxpb3RoLmRlYmlhbi5vcmc+iEYEEBEIAAYFAk1R +yQYACgkQjRFFY3XAJMgEWwCggx4Gqlcrt76TSMlbU94cESo55AEAoJ3asQEMpe8t +QUX+5aikw3z1AUoCiEoEEBECAAoFAkqf/3cDBQF4AAoJEPd/jbIxRL4PxyMAoKUI +RPWlHCj/+HSFfwhos68wcSwmAKChuC00qutDro+AOo+uuq6YoHXj+ohgBBMRAgAg +BQJKn/8bAhsDBgsJCAcDAgQVAggDBBYCAwECHgECF4AACgkQpdMvASZJpalDggCe +KF9KOgOPdQbFnKXl8KtHory4EEwAnA7jxgorE6kk2QHEXFSF8LzOOH4GiGMEExEC +ACMCGwMGCwkIBwMCBBUCCAMEFgIDAQIeAQIXgAUCSp//RgIZAQAKCRCl0y8BJkml +qekFAKCRyt4+FoCzmBbRUUP3Cr8PzH++IgCgkno4vdjsWdyAey8e0KpITTXMFrmJ +AhwEEAECAAYFAk0jQYEACgkQwHPSKH/7npsFfw/+P8B8hpM3+T1fgboBa4R32deu +n8m6b8vZMXwuo/awQtMpzjem8JGXSUQm8iiX4hDtjq6ZoPrlN8T4jNmviBt/F5jI +Jji/PYmhq+Zn9s++mfx+aF4IJrcHJWFkg/6kJzn4oSdl/YlvKf4VRCcQNtj4xV87 +GsdamnzU17XapLVMbSaVKh+6Af7ZLDerEH+iAq733HsYaTK+1xKmN7EFVXgS7bZ1 +9C4LTzc97bVHSywpT9yIrg9QQs/1kshfVIHDKyhjF6IwzSVbeGAIL3Oqo5zOMkWv +7JlEIkkhTyl+FETxNMTMYjAk+Uei3kRodneq3YBF2uFYSEzrXQgHAyn37geiaMYj +h8wu6a85nG1NS0SdxiZDIePmbvD9vWxFZUWYJ/h9ifsLivWcVXlvHoQ0emd+n2ai +FhAck2xsuyHgnGIZMHww5IkQdu/TMqvbcR6d8Xulh+C4Tq7ppy+oTLADSBKII++p +JQioYydRD529EUJgVlhyH27X6YAk3FuRD3zYZRYS2QECiKXvS665o3JRJ0ZSqNgv +YOom8M0zz6bI9grnUoivMI4o7ISpE4ZwffEd37HVzmraaUHDXRhkulFSf1ImtXoj +V9nNSM5p/+9eP7OioTZhSote6Vj6Ja1SZeRkXZK7BwqPbdO0VsYOb7G//ZiOlqs+ +paRr92G/pwBfj5Dq8EK5Ag0ERDtM9RAIAN0EJqBPvLN0tEin/y4Fe0R4n+E+zNXg +bBsq4WidwyUFy3h/6u86FYvegXwUqVS2OsEs5MwPcCVJOfaEthF7I89QJnP9Nfx7 +V5I9yFB53o9ii38BN7X+9gSjpfwXOvf/wIDfggxX8/wRFel37GRB7TiiABRArBez +s5x+zTXvT++WPhElySj0uY8bjVR6tso+d65K0UesvAa7PPWeRS+3nhqABSFLuTTT +MMbnVXCGesBrYHlFVXClAYrSIOX8Ub/UnuEYs9+hIV7U4jKzRF9WJhIC1cXHPmOh +vleAf/I9h/0KahD7HLYud40pNBo5tW8jSfp2/Q8TIE0xxshd51/xy4MAAwUH+wWn +zsYVk981OKUEXul8JPyPxbw05fOd6gF4MJ3YodO+6dfoyIl3bewk+11KXZQALKaO +1xmkAEO1RqizPeetoadBVkQBp5xPudsVElUTOX0pTYhkUd3iBilsCYKK1/KQ9KzD +I+O/lRsm6L9lc6rV0IgPU00P4BAwR+x8Rw7TJFbuS0miR3lP1NSguz+/kpjxzmGP +LyHJ+LVDYFkk6t0jPXhqFdUY6McUTBDEvavTGlVO062l9APTmmSMVFDsPN/rBes2 +rYhuuT+lDp+gcaS1UoaYCIm9kKOteQBnowX9V74Z+HKEYLtwILaSnNe6/fNSTvyj +g0z+R+sPCY4nHewbVC+ISQQYEQIACQUCRDtM9QIbDAAKCRCl0y8BJkmlqbecAJ9B +UdSKVg9H+fQNyP5sbOjj4RDtdACfXHrRHa2+XjJP0dhpvJ8IfvYnQsU= +=fAJZ +-----END PGP PUBLIC KEY BLOCK-----