chore: split binary test-data out from gh repo

Author: oesteban

.circleci/config.yml

@@ -1,114 +1,107 @@
 version: 2
 jobs:
-  build:
+  build_pytest:
     machine:
       image: circleci/classic:201808-01
     working_directory: /tmp/src/nitransforms
     environment:
       TZ: "/usr/share/zoneinfo/America/Los_Angeles"
-      SCRATCH: "/scratch"
-
-    steps:
-    - restore_cache:
-        keys:
-          - build-v1-{{ .Branch }}-{{ epoch }}
-          - build-v1-{{ .Branch }}-
-          - build-v1-master-
-          - build-v1-
-        paths:
-          - /tmp/docker
-    - run:
-        name: Set-up a Docker registry
-        command: |
-            docker run -d -p 5000:5000 --restart=always --name=registry \
-                -v /tmp/docker:/var/lib/registry registry:2
-    - run:
-        name: Pull existing images
-        command: |
-            set +e
-            docker pull localhost:5000/ubuntu
-            success=$?
-            set -e
-            if [[ "$success" = "0" ]]; then
-                echo "Pulling from local registry"
-                docker tag localhost:5000/ubuntu ubuntu:xenial-20200114
-                docker pull localhost:5000/nitransforms
-                docker tag localhost:5000/nitransforms nitransforms:latest
-            else
-                echo "Pulling from Docker Hub"
-                docker pull ubuntu:xenial-20200114
-                docker tag ubuntu:xenial-20200114 localhost:5000/ubuntu
-                docker push localhost:5000/ubuntu
-            fi
-    - checkout
-    - run:
-        name: Build Docker image & push to registry
-        no_output_timeout: 60m
-        command: |
-            e=1 && for i in {1..5}; do
-                docker build --rm --cache-from=nitransforms:latest \
-                -t nitransforms:latest \
-                --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \
-                --build-arg VCS_REF=`git rev-parse --short HEAD` . \
-                && e=0 && break || sleep 15
-            done && [ "$e" -eq "0" ]
-            docker tag nitransforms:latest localhost:5000/nitransforms
-            docker push localhost:5000/nitransforms
-    - run:
-        name: Docker registry garbage collection
-        command: |
-            docker exec -it registry /bin/registry garbage-collect --delete-untagged \
-                /etc/docker/registry/config.yml
-    - save_cache:
-        key: build-v1-{{ .Branch }}-{{ epoch }}
-        paths:
-          - /tmp/docker
-    - run:
-        name: Store FreeSurfer license file
-        command: |
-            mkdir -p /tmp/fslicense
-            cd /tmp/fslicense
-            echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IGxpY2Vuc2UudHh0Cg==" | base64 -d | sh
-    - persist_to_workspace:
-        root: /tmp
-        paths:
-          - fslicense
-
-  test_pytest:
-    machine:
-      image: circleci/classic:201808-01
-    environment:
       FS_LICENSE: "/tmp/fslicense/license.txt"
-    working_directory: /tmp/tests
+
     steps:
-      - attach_workspace:
-          at: /tmp
-      - checkout:
-          path: /tmp/src/nitransforms
-      - run:
-          name: Get codecov
-          command: python -m pip install codecov
       - restore_cache:
           keys:
             - build-v1-{{ .Branch }}-{{ epoch }}
             - build-v1-{{ .Branch }}-
             - build-v1-master-
             - build-v1-
+          paths:
+            - /tmp/docker
+      - run:
+          name: Set-up a Docker registry
+          command: |
+              docker run -d -p 5000:5000 --restart=always --name=registry \
+                  -v /tmp/docker:/var/lib/registry registry:2
+      - run:
+          name: Pull existing images
+          command: |
+              set +e
+              docker pull localhost:5000/ubuntu
+              success=$?
+              set -e
+              if [[ "$success" = "0" ]]; then
+                  echo "Pulling from local registry"
+                  docker tag localhost:5000/ubuntu ubuntu:xenial-20200114
+                  docker pull localhost:5000/nitransforms
+                  docker tag localhost:5000/nitransforms nitransforms:latest
+              else
+                  echo "Pulling from Docker Hub"
+                  docker pull ubuntu:xenial-20200114
+                  docker tag ubuntu:xenial-20200114 localhost:5000/ubuntu
+                  docker push localhost:5000/ubuntu
+              fi
+      - checkout
+      - run:
+          name: Build Docker image & push to registry
+          no_output_timeout: 60m
+          command: |
+              e=1 && for i in {1..5}; do
+                  docker build --rm --cache-from=nitransforms:latest \
+                  -t nitransforms:latest \
+                  --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \
+                  --build-arg VCS_REF=`git rev-parse --short HEAD` . \
+                  && e=0 && break || sleep 15
+              done && [ "$e" -eq "0" ]
+              docker tag nitransforms:latest localhost:5000/nitransforms
+              docker push localhost:5000/nitransforms
+      - run:
+          name: Docker registry garbage collection
+          command: |
+              docker exec -it registry /bin/registry garbage-collect --delete-untagged \
+                  /etc/docker/registry/config.yml
+      - save_cache:
+          key: build-v1-{{ .Branch }}-{{ epoch }}
+          paths:
+            - /tmp/docker
       - run:
-          name: Set-up a Docker registry & pull
+          name: Store FreeSurfer license file
           command: |
-            docker run -d -p 5000:5000 --restart=always --name=registry \
-                -v /tmp/docker:/var/lib/registry registry:2
-            docker pull localhost:5000/nitransforms
-            docker tag localhost:5000/nitransforms nitransforms:latest
+              mkdir -p /tmp/fslicense
+              cd /tmp/fslicense
+              echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IGxpY2Vuc2UudHh0Cg==" | base64 -d | sh
+
+      - restore_cache:
+          keys:
+            - data-v1-{{ .Branch }}-{{ epoch }}
+            - data-v1-{{ .Branch }}-
+            - data-v1-master-
+            - data-v1-
+          paths:
+            - /tmp/data
+      - run:
+          name: Download data
+          command: |
+              if [[ ! -d /tmp/data ]]; then
+                wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q \
+                  -O data.zip "https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/5e7d5b65c3f8d300bafa05e0/?zip="
+                unzip -d /tmp/data data.zip
+              else
+                echo "Dataset was cached"
+              fi
+      - save_cache:
+          key: data-v1-{{ .Branch }}-{{ epoch }}
+          paths:
+            - /tmp/data
+      - run:
+          name: Get codecov
+          command: python -m pip install codecov
       - run:
           name: Run unit tests
           no_output_timeout: 2h
           command: |
-            mkdir -p $PWD/artifacts $PWD/summaries
-            sudo setfacl -d -m group:ubuntu:rwx $PWD
-            sudo setfacl -m group:ubuntu:rwx $PWD
-            docker run -u $( id -u ) -it --rm=false -w /src/nitransforms \
+            mkdir -p $PWD/{artifacts,summaries}
+            docker run -u $( id -u ) -it --rm -w /src/nitransforms \
+              -v /tmp/data:/data -e TEST_DATA_HOME=/data \
               -e COVERAGE_FILE=/tmp/summaries/.pytest.coverage \
               -v /tmp/fslicense/license.txt:/opt/freesurfer/license.txt:ro \
               -v ${PWD}:/tmp nitransforms:latest \
@@ -162,33 +155,21 @@ jobs:
           command: |
             python3 -m twine upload dist/*
 
-
 workflows:
   version: 2
   build_test_deploy:
     jobs:
-      - build:
-          filters:
-            branches:
-              ignore:
-                - /docs?\/.*/
-            tags:
-              only: /.*/
-
-      - test_pytest:
-          requires:
-            - build
+      - build_pytest:
           filters:
             branches:
               ignore:
                 - /docs?\/.*/
-                - /docker\/.*/
             tags:
               only: /.*/
 
       - test_packaging_and_deploy:
           requires:
-            - test_pytest
+            - build_pytest
           filters:
             branches:
               ignore: /.*/

.travis.yml

@@ -12,18 +12,31 @@ env:
   global:
     - BUILD_TYPE="src"
     - DOCKER_IMAGE="poldracklab/fmriprep:latest"
+    - TEST_DATA_HOME="$HOME/.nitransforms/testdata"
 
 matrix:
   include:
   - python: 3.7
     env: BUILD_TYPE="docker"
 
+cache:
+  directories:
+    - $HOME/.nitransforms/testdata
+
 before_install:
   - |
+    if [[ ! -d $TEST_DATA_HOME ]]; then
+      mkdir -p $HOME/.nitransforms
+      wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q \
+        -O data.zip "https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/5e7d5b65c3f8d300bafa05e0/?zip="
+      unzip -d $TEST_DATA_HOME data.zip
+    else
+      echo "Dataset was cached"
+    fi
     if [ ${BUILD_TYPE} == "docker" ]; then
       CI_ENV=`bash <(curl -s https://codecov.io/env)`
       docker pull ${DOCKER_IMAGE}  # pull latest release
-      docker run $CI_ENV -itd --name fmriprep --entrypoint=/bin/bash -v `pwd`:/opt/nt -w /opt/nt ${DOCKER_IMAGE}
+      docker run $CI_ENV -v $TEST_DATA_HOME:/data -e TEST_DATA_HOME=/data -itd --name fmriprep --entrypoint=/bin/bash -v `pwd`:/opt/nt -w /opt/nt ${DOCKER_IMAGE}
       docker exec fmriprep sh -c 'echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IC9vcHQvZnJlZXN1cmZlci9saWNlbnNlLnR4dAo=" | base64 -d | sh'
     fi
 

nitransforms/conftest.py

@@ -5,11 +5,9 @@
 import nibabel as nb
 import pytest
 import tempfile
-import pkg_resources
 
-SOMEONES_ANATOMY = pkg_resources.resource_filename(
-    'nitransforms', 'tests/data/someones_anatomy.nii.gz')
 _data = None
+_testdir = Path(os.getenv('TEST_DATA_HOME', '~/.nitransforms/testdata')).expanduser()
 
 
 @pytest.fixture(autouse=True)
@@ -19,7 +17,8 @@ def doctest_autoimport(doctest_namespace):
     doctest_namespace['nb'] = nb
     doctest_namespace['os'] = os
     doctest_namespace['Path'] = Path
-    doctest_namespace['datadir'] = Path(__file__).parent / 'tests' / 'data'
+    doctest_namespace['regress_dir'] = Path(__file__).parent / 'tests' / 'data'
+    doctest_namespace['test_dir'] = _testdir
 
     tmpdir = tempfile.TemporaryDirectory()
     doctest_namespace['tmpdir'] = tmpdir.name
@@ -38,6 +37,12 @@ def data_path():
     return Path(__file__).parent / 'tests' / 'data'
 
 
+@pytest.fixture
+def testdata_path():
+    """Return the heavy test-data folder."""
+    return _testdir
+
+
 @pytest.fixture
 def get_testdata():
     """Generate data in the requested orientation."""
@@ -46,7 +51,7 @@ def get_testdata():
     if _data is not None:
         return _data
 
-    img = nb.load(SOMEONES_ANATOMY)
+    img = nb.load(_testdir / 'someones_anatomy.nii.gz')
     imgaff = img.affine
 
     _data = {'RAS': img}

nitransforms/linear.py

@@ -40,7 +40,7 @@ def __init__(self, matrix=None, reference=None):
 
         Examples
         --------
-        >>> xfm = Affine(reference=datadir / "someones_anatomy.nii.gz")
+        >>> xfm = Affine(reference=test_dir / "someones_anatomy.nii.gz")
         >>> xfm.matrix  # doctest: +NORMALIZE_WHITESPACE
         array([[1., 0., 0., 0.],
                [0., 1., 0., 0.],
@@ -452,11 +452,11 @@ def load(filename, fmt="X5", reference=None, moving=None):
 
     Examples
     --------
-    >>> xfm = load(datadir / "affine-LAS.itk.tfm", fmt="itk")
+    >>> xfm = load(regress_dir / "affine-LAS.itk.tfm", fmt="itk")
     >>> isinstance(xfm, Affine)
     True
 
-    >>> xfm = load(datadir / "itktflist.tfm", fmt="itk")
+    >>> xfm = load(regress_dir / "itktflist.tfm", fmt="itk")
     >>> isinstance(xfm, LinearTransformsMapping)
     True
 

nitransforms/tests/data/ds-005_sub-01_T1w.nii.gz

@@ -7,11 +7,11 @@
 from ..base import SpatialReference, SampledSpatialData, ImageGrid, TransformBase
 
 
-def test_SpatialReference(data_path):
+def test_SpatialReference(testdata_path):
     """Ensure the reference factory is working properly."""
-    obj1 = data_path / 'someones_anatomy.nii.gz'
-    obj2 = data_path / 'sub-200148_hemi-R_pial.surf.gii'
-    obj3 = data_path / 'func.nii.gz'
+    obj1 = testdata_path / 'someones_anatomy.nii.gz'
+    obj2 = testdata_path / 'sub-200148_hemi-R_pial.surf.gii'
+    obj3 = testdata_path / 'func.nii.gz'
 
     assert isinstance(SpatialReference.factory(obj1), ImageGrid)
     assert isinstance(SpatialReference.factory(str(obj1)), ImageGrid)
@@ -58,12 +58,12 @@ def test_ImageGrid(get_testdata, image_orientation):
     assert (img2 != img) is False
 
 
-def test_ImageGrid_utils(tmpdir, data_path, get_testdata):
+def test_ImageGrid_utils(tmpdir, testdata_path, get_testdata):
     """Check that images can be objects or paths and equality."""
     tmpdir.chdir()
 
     im1 = get_testdata['RAS']
-    im2 = data_path / 'someones_anatomy.nii.gz'
+    im2 = testdata_path / 'someones_anatomy.nii.gz'
 
     ig = ImageGrid(im1)
     assert ig == ImageGrid(im2)
@@ -73,7 +73,7 @@ def test_ImageGrid_utils(tmpdir, data_path, get_testdata):
         ImageGrid(im1)._to_hdf5(f.create_group('Reference'))
 
 
-def test_TransformBase(monkeypatch, data_path, tmpdir):
+def test_TransformBase(monkeypatch, testdata_path, tmpdir):
     """Check the correctness of TransformBase components."""
     tmpdir.chdir()
 
@@ -85,7 +85,7 @@ def _to_hdf5(klass, x5_root):
 
     monkeypatch.setattr(TransformBase, 'map', _fakemap)
     monkeypatch.setattr(TransformBase, '_to_hdf5', _to_hdf5)
-    fname = data_path / 'someones_anatomy.nii.gz'
+    fname = testdata_path / 'someones_anatomy.nii.gz'
 
     # Test identity transform
     xfm = TransformBase()
@@ -114,9 +114,9 @@ def _to_hdf5(klass, x5_root):
     xfm.to_filename('data.x5')
 
 
-def test_SampledSpatialData(data_path):
+def test_SampledSpatialData(testdata_path):
     """Check the reference generated by cifti files."""
-    gii = data_path / 'sub-200148_hemi-R_pial.surf.gii'
+    gii = testdata_path / 'sub-200148_hemi-R_pial.surf.gii'
 
     ssd = SampledSpatialData(gii)
     assert ssd.npoints == 249277