Lint the whole codebase and add coverage test.

.ci_helpers/docker/docker-compose.yaml

@@ -20,4 +20,3 @@ services:
       - echopypenetwork
 networks:
   echopypenetwork:
-

.ci_helpers/docker/http.dockerfile

@@ -1,3 +1,3 @@
 FROM httpd:2.4
 
-COPY echopype/test_data /usr/local/apache2/htdocs/data
+COPY echopype/test_data /usr/local/apache2/htdocs/data

.ci_helpers/docker/minioci.dockerfile

@@ -3,4 +3,4 @@ FROM minio/minio
 # Install git
 RUN microdnf install git
 
-CMD ["minio", "server", "/data"]
+CMD ["minio", "server", "/data"]

.ci_helpers/docker/setup-services.py

@@ -3,26 +3,26 @@
 Script to help bring up docker services for testing.
 
 """
+import argparse
 import os
 import shutil
 import sys
 from pathlib import Path
-import argparse
 
-HERE = Path('.').absolute()
+HERE = Path(".").absolute()
 BASE = Path(__file__).parent.absolute()
 COMPOSE_FILE = BASE / "docker-compose.yaml"
 TEST_DATA_PATH = HERE / "echopype" / "test_data"
 
 
 def parse_args():
-    parser = argparse.ArgumentParser(description='Setup services for testing')
+    parser = argparse.ArgumentParser(description="Setup services for testing")
     parser.add_argument(
-        '--deploy', action='store_true', help="Flag to setup docker services"
+        "--deploy", action="store_true", help="Flag to setup docker services"
     )
     parser.add_argument(
-        '--tear-down',
-        action='store_true',
+        "--tear-down",
+        action="store_true",
         help="Flag to tear down docker services",
     )
 
@@ -48,7 +48,9 @@ def parse_args():
         os.system(f"docker-compose -f {COMPOSE_FILE} pull")
 
         print("3) Bringing up services.")
-        os.system(f"docker-compose -f {COMPOSE_FILE} up -d --remove-orphans --force-recreate")
+        os.system(
+            f"docker-compose -f {COMPOSE_FILE} up -d --remove-orphans --force-recreate"
+        )
 
         print(f"4) Deleting old test folder at {TEST_DATA_PATH}")
         if TEST_DATA_PATH.exists():

.ci_helpers/py3.7.yaml

@@ -16,4 +16,4 @@ dependencies:
   - requests
   - aiohttp
   - s3fs==0.5.2
-  - mamba
+  - mamba

.ci_helpers/py3.8.yaml

@@ -16,4 +16,4 @@ dependencies:
   - requests
   - aiohttp
   - s3fs==0.5.2
-  - mamba
+  - mamba

.ci_helpers/py3.9.yaml

@@ -16,4 +16,4 @@ dependencies:
   - requests
   - aiohttp
   - s3fs==0.5.2
-  - mamba
+  - mamba

.ci_helpers/run-test.py

@@ -30,7 +30,7 @@
         type=str,
         nargs="?",
         default="",
-        help="Comma separated list of changed files."
+        help="Comma separated list of changed files.",
     )
     parser.add_argument(
         "--pytest-args", type=str, help="Optional pytest args", default=""

.ci_helpers/user_environment.yml

@@ -7,4 +7,3 @@ dependencies:
   - echopype
   - dask
   - pandas
-

.coveragerc

@@ -0,0 +1,8 @@
+[run]
+omit =
+    */echopype/tests/*
+    */echopype/test_data/*
+    */echopype/process/*
+    */echopype/convert/add_ancillary.py
+    */echopype/convert/convert_combine.py
+    */echopype/visualize/*

.github/actions/gdrive-rclone/action.yaml

@@ -2,4 +2,4 @@ name: 'GDrive Rclone'
 description: 'Performs RClone from a google drive folder'
 runs:
   using: 'docker'
-  image: 'Dockerfile'
+  image: 'Dockerfile'

.github/actions/gdrive-rclone/entrypoint.sh

@@ -27,10 +27,9 @@ then
     echo "Copying new test data from google drive"
     rclone copy gdrive: $TEST_DATA_FOLDER
     echo "Done"
-    
+
     chmod -R ugoa+w $TEST_DATA_FOLDER
     ls -lah $TEST_DATA_FOLDER
 else
     echo "${TEST_DATA_FOLDER} not found"
 fi
-

.github/workflows/ci.yaml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - dev
     paths-ignore: [".ci_helpers/docker/**", "**/docker.yaml"]
   workflow_dispatch:
 
@@ -49,6 +50,9 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           architecture: x64
+      - name: Set environment variables
+        run: |
+          echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV
       - name: Cache conda
         uses: actions/cache@v2
         env:
@@ -83,10 +87,18 @@ jobs:
       - name: Running All Tests
         shell: bash -l {0}
         run: |
-          python .ci_helpers/run-test.py --local --pytest-args="--log-cli-level=WARNING,-vv,--disable-warnings" |& tee ci_test_log.log
+          python .ci_helpers/run-test.py --local --pytest-args="--cov=echopype,--cov-report=xml,--log-cli-level=WARNING,-vv,--disable-warnings" |& tee ci_test_log.log
       - name: Upload ci test log
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
           name: ci_test_log
           path: ci_test_log.log
+      - name: Upload code coverage to Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          file: ./coverage.xml
+          flags: unittests
+          env_vars: RUNNER_OS,PYTHON_VERSION
+          name: codecov-umbrella
+          fail_ci_if_error: false

.github/workflows/docker.yaml

@@ -36,7 +36,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
       - name: Login to DockerHub
-        uses: docker/login-action@v1 
+        uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}

.github/workflows/lint.yaml

@@ -15,5 +15,3 @@ jobs:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2
     - uses: pre-commit/[email protected]
-      # allow to error for now
-      continue-on-error: true

.github/workflows/pr.yaml

@@ -46,6 +46,9 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           architecture: x64
+      - name: Set environment variables
+        run: |
+          echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV
       - name: Cache conda
         uses: actions/cache@v2
         env:
@@ -85,13 +88,27 @@ jobs:
       - name: Print Changed files
         shell: bash -l {0}
         run: echo "${{ steps.files.outputs.added_modified }}"
+      - name: Running all Tests
+        if: contains(github.event.pull_request.labels.*.name, 'Needs Complete Testing')
+        shell: bash -l {0}
+        run: |
+          python .ci_helpers/run-test.py --local --pytest-args="--cov=echopype,--cov-report=xml,--log-cli-level=WARNING,-vv,--disable-warnings" |& tee ci_test_log.log
       - name: Running Tests
+        if: "!contains(github.event.pull_request.labels.*.name, 'Needs Complete Testing')"
         shell: bash -l {0}
         run: |
-          python .ci_helpers/run-test.py --pytest-args="--log-cli-level=WARNING,-vv,--disable-warnings" ${{ steps.files.outputs.added_modified }} |& tee ci_test_log.log
+          python .ci_helpers/run-test.py --pytest-args="--cov=echopype,--cov-report=xml,--log-cli-level=WARNING,-vv,--disable-warnings" ${{ steps.files.outputs.added_modified }} |& tee ci_test_log.log
       - name: Upload ci test log
         if: ${{ success() || failure() }}
         uses: actions/upload-artifact@v2
         with:
           name: ci_test_log
           path: ci_test_log.log
+      - name: Upload code coverage to Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          file: ./coverage.xml
+          flags: unittests
+          env_vars: RUNNER_OS,PYTHON_VERSION
+          name: codecov-umbrella
+          fail_ci_if_error: false

.github/workflows/pypi.yaml

@@ -39,7 +39,7 @@ jobs:
         # local part of the version isn't included, making the version string
         # compatible with PyPI.
         sed --in-place "s/node-and-date/no-local-version/g" setup.py
-      
+
     - name: Build source and wheel distributions
       run: |
         python setup.py sdist bdist_wheel
@@ -84,14 +84,14 @@ jobs:
         python -m pip install --extra-index-url https://test.pypi.org/simple --upgrade --pre echopype
         python -c "import echopype; print(echopype.__version__)"
         echo "=== Done testing wheel file ==="
-        
+
         echo "=== Testing source tar file ==="
         # Install tar gz and check import
         python -m pip uninstall --yes echopype
         python -m pip install --extra-index-url https://test.pypi.org/simple --upgrade --pre --no-binary=:all: echopype
         python -c "import echopype; print(echopype.__version__)"
         echo "=== Done testing source tar file ==="
-  
+
   publish-pypi:
     name: Push echopype to production pypi
     needs: test-built-dist

.pre-commit-config.yaml

@@ -1,37 +1,36 @@
+exclude: |
+  (?x)^(
+    docs/|
+    echopype/tests/|
+    echopype/test_data/|
+    echopype/process/|
+    echopype/visualize/|
+    echopype/convert/add_ancillary.py|
+    echopype/convert/convert_combine.py
+  )
 repos:
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.1.0
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.4.0
     hooks:
-    - id: trailing-whitespace
-    - id: end-of-file-fixer
-    - id: check-docstring-first
-    - id: check-json
-    - id: check-yaml
-    - id: pretty-format-json
-      args: ["--autofix", "--indent=2", "--no-sort-keys"]
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-docstring-first
+      - id: check-json
+      - id: check-yaml
+      - id: pretty-format-json
+        args: ["--autofix", "--indent=2", "--no-sort-keys"]
 
--   repo: https://github.com/ambv/black
-    rev: 19.10b0
+  - repo: https://github.com/psf/black
+    rev: 20.8b1
     hooks:
-    - id: black
-      args: ["--line-length", "100"]
+      - id: black
 
--   repo: https://gitlab.com/pycqa/flake8
+  - repo: https://gitlab.com/pycqa/flake8
     rev: 3.8.3
     hooks:
-    - id: flake8
+      - id: flake8
 
--   repo: https://github.com/asottile/seed-isort-config
-    rev: v2.2.0
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.8.0
     hooks:
-    - id: seed-isort-config
-
--   repo: https://github.com/pre-commit/mirrors-isort
-    rev: v5.2.0
-    hooks:
-    - id: isort
-
--   repo: https://github.com/myint/rstcheck
-    rev: master
-    hooks:
-    -   id: rstcheck
+      - id: isort

README.md

@@ -30,7 +30,7 @@
 
 Echopype is a package built to enable interoperability and scalability in ocean sonar data processing. These data are widely used for obtaining information about the distribution and abundance of marine animals, such as fish and krill. Our ability to collect large volumes of sonar data from a variety of ocean platforms has grown significantly in the last decade. However, most of the new data remain under-utilized. echopype aims to address the root cause of this problem - the lack of interoperable data format and scalable analysis workflows that adapt well with increasing data volume - by providing open-source tools as entry points for scientists to make discovery using these new data.
 
-Watch the [echopype talk](https://www.youtube.com/watch?v=qboH7MyHrpU) 
+Watch the [echopype talk](https://www.youtube.com/watch?v=qboH7MyHrpU)
 at SciPy 2019 for background, discussions and a quick demo!
 
 ## Documentation
@@ -53,7 +53,7 @@ Please report any bugs by [creating issues on GitHub](https://medium.com/nyc-pla
 Contributors
 ------------
 
-[Wu-Jung Lee](http://leewujung.github.io) (@leewujung) leads this project and together with 
+[Wu-Jung Lee](http://leewujung.github.io) (@leewujung) leads this project and together with
 [Kavin Nguyen](https://github.com/ngkavin) (@ngkavin), [Landung "Don" Setiawan](https://github.com/lsetiawan) (@lsetiawan), and [Imran Majeed](https://github.com/imranmaj) (@imranmaj) are primary developers of this package.
 [Emilio Mayorga](https://www.apl.washington.edu/people/profile.php?last_name=Mayorga&first_name=Emilio) (@emiliom)
 and [Valentina Staneva](https://escience.washington.edu/people/valentina-staneva/) (@valentina-s)

echopype/__init__.py

@@ -1,9 +1,10 @@
 from __future__ import absolute_import, division, print_function
-from .convert.api import open_raw
-from .echodata.api import open_converted
-from .process import Process
-from . import calibrate
 
 from _echopype_version import version as __version__  # noqa
 
+from . import calibrate
+from .convert.api import open_raw
+from .echodata.api import open_converted
+from .process import Process  # noqa
+
 __all__ = [open_raw, open_converted, calibrate]

echopype/calibrate/__init__.py

@@ -1,3 +1,3 @@
-from .api import compute_Sv, compute_Sp
+from .api import compute_Sp, compute_Sv
 
-# __all__ = [compute_Sv, compute_Sp]
+__all__ = [compute_Sv, compute_Sp]