chore: split binary test-data out from gh repo

Author: oesteban

.circleci/config.yml

@@ -1,12 +1,12 @@
 version: 2
 jobs:
-  build:
+  build_pytest:
     machine:
       image: circleci/classic:201808-01
     working_directory: /tmp/src/nitransforms
     environment:
       TZ: "/usr/share/zoneinfo/America/Los_Angeles"
-      SCRATCH: "/scratch"
+      FS_LICENSE: "/tmp/fslicense/license.txt"
 
     steps:
     - restore_cache:
@@ -69,46 +69,39 @@ jobs:
             mkdir -p /tmp/fslicense
             cd /tmp/fslicense
             echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IGxpY2Vuc2UudHh0Cg==" | base64 -d | sh
-    - persist_to_workspace:
-        root: /tmp
-        paths:
-          - fslicense
 
-  test_pytest:
-    machine:
-      image: circleci/classic:201808-01
-    environment:
-      FS_LICENSE: "/tmp/fslicense/license.txt"
-    working_directory: /tmp/tests
-    steps:
-      - attach_workspace:
-          at: /tmp
-      - checkout:
-          path: /tmp/src/nitransforms
+    - restore_cache:
+        keys:
+          - data-v1-{{ .Branch }}-{{ epoch }}
+          - data-v1-{{ .Branch }}-
+          - data-v1-master-
+          - data-v1-
+        paths:
+          - /tmp/data
+    - run:
+        name: Download data
+        command: |
+            if [[ ! -d /tmp/data ]]; then
+              wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q \
+                -O data.zip "https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/5e7d5b65c3f8d300bafa05e0/?zip="
+              unzip -d /tmp/data data.zip
+            else
+              echo "Dataset was cached"
+            fi
+    - save_cache:
+        key: data-v1-{{ .Branch }}-{{ epoch }}
+        paths:
+          - /tmp/data
       - run:
           name: Get codecov
           command: python -m pip install codecov
-      - restore_cache:
-          keys:
-            - build-v1-{{ .Branch }}-{{ epoch }}
-            - build-v1-{{ .Branch }}-
-            - build-v1-master-
-            - build-v1-
-      - run:
-          name: Set-up a Docker registry & pull
-          command: |
-            docker run -d -p 5000:5000 --restart=always --name=registry \
-                -v /tmp/docker:/var/lib/registry registry:2
-            docker pull localhost:5000/nitransforms
-            docker tag localhost:5000/nitransforms nitransforms:latest
       - run:
           name: Run unit tests
           no_output_timeout: 2h
           command: |
-            mkdir -p $PWD/artifacts $PWD/summaries
-            sudo setfacl -d -m group:ubuntu:rwx $PWD
-            sudo setfacl -m group:ubuntu:rwx $PWD
-            docker run -u $( id -u ) -it --rm=false -w /src/nitransforms \
+            mkdir -p $PWD/{artifacts,summaries}
+            docker run -u $( id -u ) -it --rm -w /src/nitransforms \
+              -v /tmp/data:/data -e TEST_DATA_HOME=/data \
               -e COVERAGE_FILE=/tmp/summaries/.pytest.coverage \
               -v /tmp/fslicense/license.txt:/opt/freesurfer/license.txt:ro \
               -v ${PWD}:/tmp nitransforms:latest \
@@ -167,28 +160,17 @@ workflows:
   version: 2
   build_test_deploy:
     jobs:
-      - build:
-          filters:
-            branches:
-              ignore:
-                - /docs?\/.*/
-            tags:
-              only: /.*/
-
-      - test_pytest:
-          requires:
-            - build
+      - build_pytest:
           filters:
             branches:
               ignore:
                 - /docs?\/.*/
-                - /docker\/.*/
             tags:
               only: /.*/
 
       - test_packaging_and_deploy:
           requires:
-            - test_pytest
+            - build_pytest
           filters:
             branches:
               ignore: /.*/

.travis.yml

@@ -12,18 +12,31 @@ env:
   global:
     - BUILD_TYPE="src"
     - DOCKER_IMAGE="poldracklab/fmriprep:latest"
+    - TEST_DATA_HOME="$HOME/.nitransforms/testdata"
 
 matrix:
   include:
   - python: 3.7
     env: BUILD_TYPE="docker"
 
+cache:
+  directories:
+    - $HOME/.nitransforms/testdata
+
 before_install:
   - |
+    if [[ ! -d $TEST_DATA_HOME ]]; then
+      mkdir -p $HOME/.nitransforms
+      wget --retry-connrefused --waitretry=5 --read-timeout=20 --timeout=15 -t 0 -q \
+        -O data.zip "https://files.osf.io/v1/resources/fvuh8/providers/osfstorage/5e7d5b65c3f8d300bafa05e0/?zip="
+      unzip -d $TEST_DATA_HOME data.zip
+    else
+      echo "Dataset was cached"
+    fi
     if [ ${BUILD_TYPE} == "docker" ]; then
       CI_ENV=`bash <(curl -s https://codecov.io/env)`
       docker pull ${DOCKER_IMAGE}  # pull latest release
-      docker run $CI_ENV -itd --name fmriprep --entrypoint=/bin/bash -v `pwd`:/opt/nt -w /opt/nt ${DOCKER_IMAGE}
+      docker run $CI_ENV -v $TEST_DATA_HOME:/data -e TEST_DATA_HOME=/data -itd --name fmriprep --entrypoint=/bin/bash -v `pwd`:/opt/nt -w /opt/nt ${DOCKER_IMAGE}
       docker exec fmriprep sh -c 'echo "cHJpbnRmICJrcnp5c3p0b2YuZ29yZ29sZXdza2lAZ21haWwuY29tXG41MTcyXG4gKkN2dW12RVYzelRmZ1xuRlM1Si8yYzFhZ2c0RVxuIiA+IC9vcHQvZnJlZXN1cmZlci9saWNlbnNlLnR4dAo=" | base64 -d | sh'
     fi
 

nitransforms/conftest.py

@@ -5,11 +5,9 @@
 import nibabel as nb
 import pytest
 import tempfile
-import pkg_resources
 
-SOMEONES_ANATOMY = pkg_resources.resource_filename(
-    'nitransforms', 'tests/data/someones_anatomy.nii.gz')
 _data = None
+_testdir = Path(os.getenv('TEST_DATA_HOME', '~/.nitransforms/testdata')).expanduser()
 
 
 @pytest.fixture(autouse=True)
@@ -19,7 +17,8 @@ def doctest_autoimport(doctest_namespace):
     doctest_namespace['nb'] = nb
     doctest_namespace['os'] = os
     doctest_namespace['Path'] = Path
-    doctest_namespace['datadir'] = Path(__file__).parent / 'tests' / 'data'
+    doctest_namespace['regress_dir'] = Path(__file__).parent / 'tests' / 'data'
+    doctest_namespace['test_dir'] = _testdir
 
     tmpdir = tempfile.TemporaryDirectory()
     doctest_namespace['tmpdir'] = tmpdir.name
@@ -38,6 +37,12 @@ def data_path():
     return Path(__file__).parent / 'tests' / 'data'
 
 
+@pytest.fixture
+def testdata_path():
+    """Return the heavy test-data folder."""
+    return _testdir
+
+
 @pytest.fixture
 def get_testdata():
     """Generate data in the requested orientation."""
@@ -46,7 +51,7 @@ def get_testdata():
     if _data is not None:
         return _data
 
-    img = nb.load(SOMEONES_ANATOMY)
+    img = nb.load(_testdir / 'someones_anatomy.nii.gz')
     imgaff = img.affine
 
     _data = {'RAS': img}

nitransforms/linear.py

@@ -40,7 +40,7 @@ def __init__(self, matrix=None, reference=None):
 
         Examples
         --------
-        >>> xfm = Affine(reference=datadir / "someones_anatomy.nii.gz")
+        >>> xfm = Affine(reference=test_dir / "someones_anatomy.nii.gz")
         >>> xfm.matrix  # doctest: +NORMALIZE_WHITESPACE
         array([[1., 0., 0., 0.],
                [0., 1., 0., 0.],
@@ -452,11 +452,11 @@ def load(filename, fmt="X5", reference=None, moving=None):
 
     Examples
     --------
-    >>> xfm = load(datadir / "affine-LAS.itk.tfm", fmt="itk")
+    >>> xfm = load(regress_dir / "affine-LAS.itk.tfm", fmt="itk")
     >>> isinstance(xfm, Affine)
     True
 
-    >>> xfm = load(datadir / "itktflist.tfm", fmt="itk")
+    >>> xfm = load(regress_dir / "itktflist.tfm", fmt="itk")
     >>> isinstance(xfm, LinearTransformsMapping)
     True
 

nitransforms/tests/data/ds-005_sub-01_T1w.nii.gz

@@ -7,11 +7,11 @@
 from ..base import SpatialReference, SampledSpatialData, ImageGrid, TransformBase
 
 
-def test_SpatialReference(data_path):
+def test_SpatialReference(testdata_path):
     """Ensure the reference factory is working properly."""
-    obj1 = data_path / 'someones_anatomy.nii.gz'
-    obj2 = data_path / 'sub-200148_hemi-R_pial.surf.gii'
-    obj3 = data_path / 'func.nii.gz'
+    obj1 = testdata_path / 'someones_anatomy.nii.gz'
+    obj2 = testdata_path / 'sub-200148_hemi-R_pial.surf.gii'
+    obj3 = testdata_path / 'func.nii.gz'
 
     assert isinstance(SpatialReference.factory(obj1), ImageGrid)
     assert isinstance(SpatialReference.factory(str(obj1)), ImageGrid)
@@ -58,12 +58,12 @@ def test_ImageGrid(get_testdata, image_orientation):
     assert (img2 != img) is False
 
 
-def test_ImageGrid_utils(tmpdir, data_path, get_testdata):
+def test_ImageGrid_utils(tmpdir, testdata_path, get_testdata):
     """Check that images can be objects or paths and equality."""
     tmpdir.chdir()
 
     im1 = get_testdata['RAS']
-    im2 = data_path / 'someones_anatomy.nii.gz'
+    im2 = testdata_path / 'someones_anatomy.nii.gz'
 
     ig = ImageGrid(im1)
     assert ig == ImageGrid(im2)
@@ -73,7 +73,7 @@ def test_ImageGrid_utils(tmpdir, data_path, get_testdata):
         ImageGrid(im1)._to_hdf5(f.create_group('Reference'))
 
 
-def test_TransformBase(monkeypatch, data_path, tmpdir):
+def test_TransformBase(monkeypatch, testdata_path, tmpdir):
     """Check the correctness of TransformBase components."""
     tmpdir.chdir()
 
@@ -85,7 +85,7 @@ def _to_hdf5(klass, x5_root):
 
     monkeypatch.setattr(TransformBase, 'map', _fakemap)
     monkeypatch.setattr(TransformBase, '_to_hdf5', _to_hdf5)
-    fname = data_path / 'someones_anatomy.nii.gz'
+    fname = testdata_path / 'someones_anatomy.nii.gz'
 
     # Test identity transform
     xfm = TransformBase()
@@ -114,9 +114,9 @@ def _to_hdf5(klass, x5_root):
     xfm.to_filename('data.x5')
 
 
-def test_SampledSpatialData(data_path):
+def test_SampledSpatialData(testdata_path):
     """Check the reference generated by cifti files."""
-    gii = data_path / 'sub-200148_hemi-R_pial.surf.gii'
+    gii = testdata_path / 'sub-200148_hemi-R_pial.surf.gii'
 
     ssd = SampledSpatialData(gii)
     assert ssd.npoints == 249277

nitransforms/tests/data/ds-005_sub-01_from-OASIS_to-T1_warp_afni.nii.gz

@@ -47,10 +47,10 @@ def test_apply_linear(tmpdir, data_path, get_testdata):
     assert output.check()
 
 
-def test_apply_nl(tmpdir, data_path):
+def test_apply_nl(tmpdir, testdata_path):
     tmpdir.chdir()
-    img = str(data_path / 'tpl-OASIS30ANTs_T1w.nii.gz')
-    nl_xform = str(data_path / 'ds-005_sub-01_from-OASIS_to-T1_warp_afni.nii.gz')
+    img = str(testdata_path / 'tpl-OASIS30ANTs_T1w.nii.gz')
+    nl_xform = str(testdata_path / 'ds-005_sub-01_from-OASIS_to-T1_warp_afni.nii.gz')
 
     nlargs = ['apply', nl_xform, img]
     # format not specified

nitransforms/tests/data/ds-005_sub-01_from-OASIS_to-T1_warp_itk.nii.gz

@@ -28,13 +28,13 @@ def test_lta2itk_conversions(data_path, filename):
     ("from-scanner_to-bold_mode-image", "T1w_scanner.nii.gz", "bold.nii.gz"),
     ("from-scanner_to-fsnative_mode-image", "T1w_scanner.nii.gz", "T1w_fsnative.nii.gz"),
 ])
-def test_itk2lta_conversions(data_path, tmp_path, filename, moving, reference):
+def test_itk2lta_conversions(data_path, testdata_path, tmp_path, filename, moving, reference):
     """Check conversions between formats."""
     itk = _l.load(data_path / "regressions" / "".join((filename, ".tfm")),
                   fmt="itk")
-    itk.reference = data_path / "regressions" / reference
+    itk.reference = testdata_path / reference
     itk.to_filename(tmp_path / "test.lta", fmt="fs",
-                    moving=data_path / "regressions" / moving)
+                    moving=testdata_path / moving)
 
     converted_lta = LTA.from_filename(tmp_path / "test.lta")
     expected_fname = data_path / "regressions" / "".join((filename, "_type-ras2ras.lta"))

nitransforms/tests/data/ds-005_sub-01_from-T1_to-OASIS_affine.mat

@@ -187,10 +187,10 @@ def test_LinearList_common(tmpdir, data_path, sw, image_orientation,
                 for x1, x2 in zip(tflist1.xforms, tflist2.xforms)])
 
 
-def test_ITKLinearTransform(tmpdir, data_path):
+def test_ITKLinearTransform(tmpdir, testdata_path):
     tmpdir.chdir()
 
-    matlabfile = data_path / 'ds-005_sub-01_from-T1_to-OASIS_affine.mat'
+    matlabfile = testdata_path / 'ds-005_sub-01_from-T1_to-OASIS_affine.mat'
     mat = loadmat(str(matlabfile))
     with open(str(matlabfile), 'rb') as f:
         itkxfm = itk.ITKLinearTransform.from_fileobj(f)
@@ -339,15 +339,15 @@ def test_afni_Displacements():
         afni.AFNIDisplacementsField.from_image(field)
 
 
-def test_itk_h5(data_path):
+def test_itk_h5(testdata_path):
     """Test displacements fields."""
     assert len(list(itk.ITKCompositeH5.from_filename(
-        data_path / 'ds-005_sub-01_from-T1w_to-MNI152NLin2009cAsym_mode-image_xfm.h5'
+        testdata_path / 'ds-005_sub-01_from-T1w_to-MNI152NLin2009cAsym_mode-image_xfm.h5'
     ))) == 2
 
     with pytest.raises(RuntimeError):
         list(itk.ITKCompositeH5.from_filename(
-            data_path / 'ds-005_sub-01_from-T1w_to-MNI152NLin2009cAsym_mode-image_xfm.x5'
+            testdata_path / 'ds-005_sub-01_from-T1w_to-MNI152NLin2009cAsym_mode-image_xfm.x5'
         ))