From e1373ef407148f50966755db9ffaaf12052fb410 Mon Sep 17 00:00:00 2001 From: lsetiawan Date: Mon, 28 Mar 2022 12:02:01 -0700 Subject: [PATCH 1/8] Upgrade python minimum to 3.8 --- .ci_helpers/py3.10.yaml | 6 ++-- .ci_helpers/py3.7.yaml | 19 ------------ .ci_helpers/py3.8.yaml | 6 ++-- .ci_helpers/py3.9.yaml | 6 ++-- .github/workflows/ci.yaml | 12 ++------ pyproject.toml | 15 ++++++++++ requirements.txt | 6 +--- setup.cfg | 61 +++++++++++++++++++++++++++++++++++++++ setup.py | 40 ++----------------------- 9 files changed, 88 insertions(+), 83 deletions(-) delete mode 100644 .ci_helpers/py3.7.yaml create mode 100644 pyproject.toml diff --git a/.ci_helpers/py3.10.yaml b/.ci_helpers/py3.10.yaml index 9a70cbdc4..fdc303267 100644 --- a/.ci_helpers/py3.10.yaml +++ b/.ci_helpers/py3.10.yaml @@ -4,16 +4,14 @@ channels: dependencies: - python=3.10 - dask - - matplotlib - netCDF4 - - numpy - pynmea2 - pytz - scipy - xarray - zarr - fsspec - - requests - - aiohttp - s3fs + - matplotlib-base + - cmocean - mamba=0.20.0 diff --git a/.ci_helpers/py3.7.yaml b/.ci_helpers/py3.7.yaml deleted file mode 100644 index dfbd82c20..000000000 --- a/.ci_helpers/py3.7.yaml +++ /dev/null @@ -1,19 +0,0 @@ -name: echopype -channels: - - conda-forge -dependencies: - - python=3.7 - - dask - - matplotlib - - netCDF4 - - numpy - - pynmea2 - - pytz - - scipy - - xarray - - zarr - - fsspec - - requests - - aiohttp - - s3fs - - mamba=0.20.0 diff --git a/.ci_helpers/py3.8.yaml b/.ci_helpers/py3.8.yaml index 5657c3ff7..d7169d9aa 100644 --- a/.ci_helpers/py3.8.yaml +++ b/.ci_helpers/py3.8.yaml @@ -4,16 +4,14 @@ channels: dependencies: - python=3.8 - dask - - matplotlib - netCDF4 - - numpy - pynmea2 - pytz - scipy - xarray - zarr - fsspec - - requests - - aiohttp - s3fs + - matplotlib-base + - cmocean - mamba=0.20.0 diff --git a/.ci_helpers/py3.9.yaml b/.ci_helpers/py3.9.yaml index 07af97b1e..9cbca8fc3 100644 --- a/.ci_helpers/py3.9.yaml +++ b/.ci_helpers/py3.9.yaml @@ -4,16 +4,14 @@ channels: dependencies: - python=3.9 - dask - - matplotlib - netCDF4 - - numpy - pynmea2 - pytz - scipy - xarray - zarr - fsspec - - requests - - aiohttp - s3fs + - matplotlib-base + - cmocean - mamba=0.20.0 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 70c80c4f4..581f0e131 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,12 +21,10 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.7 - experimental: false - python-version: 3.8 experimental: false - python-version: 3.9 - experimental: true + experimental: false - python-version: "3.10" experimental: true services: @@ -83,15 +81,11 @@ jobs: - name: Remove docker-compose python if: ${{ matrix.python-version == '3.10' }} shell: bash -l {0} - run: sed -i "/docker-compose/d" requirements-dev.txt - - name: Install dev tools - shell: bash -l {0} - run: | - mamba install -c conda-forge --yes --file requirements-dev.txt + run: sed -i "/docker-compose/d" setup.cfg - name: Install echopype shell: bash -l {0} run: | - python -m pip install -e . + python -m pip install -e .[dev] - name: Running All Tests shell: bash -l {0} run: | diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..57e952e16 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ + +[build-system] +requires = [ + "setuptools >= 48", + "setuptools_scm[toml] >= 4, <6", + "setuptools_scm_git_archive", + "wheel >= 0.29.0", +] +build-backend = 'setuptools.build_meta' + +[tool.setuptools_scm] +fallback_version = "unknown" +local_scheme = "node-and-date" +write_to = "_echopype_version.py" +write_to_template = 'version = "{version}" # noqa' diff --git a/requirements.txt b/requirements.txt index ab0c21b1d..b79f2c9fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,8 +7,4 @@ scipy xarray zarr fsspec -# https://github.com/fsspec/s3fs/pull/554 -s3fs==2021.11.0 -requests -aiohttp -typing-extensions +s3fs diff --git a/setup.cfg b/setup.cfg index 8ba672d65..d20c4f339 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,6 +3,7 @@ name = echopype maintainer = Wu-Jung Lee maintainer_email = leewujung@gmail.com description = Enhancing the interoperability and scalability in analyzing ocean sonar data +long_description = file: README.md long_description_content_type = text/markdown url = https://github.com/OSOceanAcoustics/echopype license = Apache License, Version 2.0 @@ -18,6 +19,66 @@ author = Wu-Jung Lee author_email = leewujung@gmail.com platforms = OS Independent +[options] +packages = find: +platforms = any +py_modules = + _echopype_version +include_package_data = True +install_requires = + dask[array] + netCDF4 + pynmea2 + pytz + scipy + xarray + zarr + fsspec + s3fs +python_requires = >=3.8 + +[options.extras_require] +dev = + black + check-manifest + codespell + docker-compose + flake8 + flake8-builtins + flake8-comprehensions + flake8-mutable + flake8-print + isort + mypy + numpydoc + pre-commit + pylint + pytest + pytest-cov + pytest-flake8 + pytest-xdist + setuptools_scm + sphinx + sphinx-automodapi + sphinx_rtd_theme + sphinxcontrib-mermaid + twine + wheel +plot = + matplotlib + cmocean +docs = + sphinx_rtd_theme + sphinx-automodapi + sphinxcontrib-mermaid + jupyter-book + numpydoc + docutils<0.18 +all = + %(dev)s + %(plot)s + %(docs)s + [black] line-length = 100 diff --git a/setup.py b/setup.py index f7843bb5f..ff1b148d6 100644 --- a/setup.py +++ b/setup.py @@ -1,42 +1,6 @@ from __future__ import absolute_import, division, print_function -from setuptools import find_packages, setup - -# Long description will go up on the pypi page -with open("README.md") as file: - LONG_DESCRIPTION = file.read() - -# Dependencies. -with open("requirements.txt") as f: - requirements = f.readlines() - -with open("requirements-dev.txt") as f: - dev_reqs = f.readlines() - -plot_reqs = ["cmocean", "matplotlib"] -all_reqs = requirements + dev_reqs + plot_reqs - -EXTRA_REQUIRES = {"dev": dev_reqs, "plot": plot_reqs, "all": all_reqs} - -INSTALL_REQUIRES = [t.strip() for t in requirements] - -opts = dict( - long_description=LONG_DESCRIPTION, - packages=find_packages(), - include_package_data=True, - install_requires=INSTALL_REQUIRES, - extras_require=EXTRA_REQUIRES, - python_requires=">=3.6", - py_modules=["_echopype_version"], - use_scm_version={ - "fallback_version": "unknown", - "local_scheme": "node-and-date", - "write_to": "_echopype_version.py", - "write_to_template": 'version = "{version}"\n', - }, - setup_requires=["setuptools>=45", "wheel", "setuptools_scm"], -) - +from setuptools import setup if __name__ == "__main__": - setup(**opts) + setup() From 2b55a0e5d34fc21eefe569c65847f613c9adb43f Mon Sep 17 00:00:00 2001 From: lsetiawan Date: Mon, 28 Mar 2022 12:10:52 -0700 Subject: [PATCH 2/8] Update RTD to install from setup.cfg --- .readthedocs.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 076c6b974..f760b1a9b 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -18,13 +18,12 @@ formats: [] # Optionally set the version of Python and requirements required to build your docs python: - version: 3.7 + version: "3.8" install: - - requirements: requirements.txt - - requirements: docs/requirements.txt - method: pip path: . extra_requirements: + - docs - plot - method: setuptools path: . From 48fcda3c1fdfcfe9d53cde19962f81666a1aff89 Mon Sep 17 00:00:00 2001 From: lsetiawan Date: Mon, 28 Mar 2022 12:14:16 -0700 Subject: [PATCH 3/8] Fix PR action to not use 3.7 --- .github/workflows/pr.yaml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 77480c469..9c794f8da 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -17,12 +17,10 @@ jobs: fail-fast: false matrix: include: - - python-version: 3.7 - experimental: false - python-version: 3.8 experimental: false - python-version: 3.9 - experimental: true + experimental: false - python-version: "3.10" experimental: true services: @@ -79,15 +77,11 @@ jobs: - name: Remove docker-compose python if: ${{ matrix.python-version == '3.10' }} shell: bash -l {0} - run: sed -i "/docker-compose/d" requirements-dev.txt - - name: Install dev tools - shell: bash -l {0} - run: | - mamba install -c conda-forge --yes --file requirements-dev.txt + run: sed -i "/docker-compose/d" setup.cfg - name: Install echopype shell: bash -l {0} run: | - python -m pip install -e . + python -m pip install -e .[dev] - name: Finding changed files id: files uses: lsetiawan/get-changed-files@pr_target From d1d67f138462588a2ee7c5d83d890868b10235e9 Mon Sep 17 00:00:00 2001 From: lsetiawan Date: Mon, 28 Mar 2022 13:32:32 -0700 Subject: [PATCH 4/8] Moving out 'build' deps out of setup.cfg --- .github/workflows/ci.yaml | 8 +++-- .github/workflows/pr.yaml | 8 +++-- .readthedocs.yml | 2 +- docs/source/contributing.rst | 4 +-- pyproject.toml | 18 ++++++++++++ requirements-dev.txt | 2 +- requirements.txt | 4 +++ setup.cfg | 57 +++--------------------------------- 8 files changed, 42 insertions(+), 61 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 581f0e131..a82be491d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -81,11 +81,15 @@ jobs: - name: Remove docker-compose python if: ${{ matrix.python-version == '3.10' }} shell: bash -l {0} - run: sed -i "/docker-compose/d" setup.cfg + run: sed -i "/docker-compose/d" requirements-dev.txt + - name: Install dev tools + shell: bash -l {0} + run: | + mamba install -c conda-forge --yes --file requirements-dev.txt - name: Install echopype shell: bash -l {0} run: | - python -m pip install -e .[dev] + python -m pip install -e .[plot] - name: Running All Tests shell: bash -l {0} run: | diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 9c794f8da..284a78c09 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -77,11 +77,15 @@ jobs: - name: Remove docker-compose python if: ${{ matrix.python-version == '3.10' }} shell: bash -l {0} - run: sed -i "/docker-compose/d" setup.cfg + run: sed -i "/docker-compose/d" requirements-dev.txt + - name: Install dev tools + shell: bash -l {0} + run: | + mamba install -c conda-forge --yes --file requirements-dev.txt - name: Install echopype shell: bash -l {0} run: | - python -m pip install -e .[dev] + python -m pip install -e .[plot] - name: Finding changed files id: files uses: lsetiawan/get-changed-files@pr_target diff --git a/.readthedocs.yml b/.readthedocs.yml index f760b1a9b..7b276ef73 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -20,10 +20,10 @@ formats: [] python: version: "3.8" install: + - requirements: docs/requirements.txt - method: pip path: . extra_requirements: - - docs - plot - method: setuptools path: . diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index 9b2c59e01..83c2d70fc 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -76,12 +76,12 @@ Create a `conda `_ environment for echopype development .. code-block:: bash - conda create -c conda-forge -n echopype --yes python=3.9 --file requirements.txt --file requirements-dev.txt + conda create -c conda-forge -n echopype --yes python=3.9 --file requirements-dev.txt conda activate echopype # ipykernel is recommended, in order to use with JupyterLab and IPython # to aid with development. We recommend you install JupyterLab separately conda install -c conda-forge ipykernel - pip install -e . + pip install -e .[all] See the :doc:`installation` page to simply install the latest echopype release from conda or PyPI. diff --git a/pyproject.toml b/pyproject.toml index 57e952e16..6c2859584 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,3 +13,21 @@ fallback_version = "unknown" local_scheme = "node-and-date" write_to = "_echopype_version.py" write_to_template = 'version = "{version}" # noqa' + +[tool.black] +line-length = 100 + +[tool.flake8] +max-line-length = 100 +output-file = flake8_log.txt +tee = True +ignore = E722,E203,W503,T001 + +[tool.isort] +known_first_party = echopype +known_third_party = _echopype_version,setuptools +multi_line_output = 3 +include_trailing_comma = True +force_grid_wrap = 0 +combine_as_imports = True +line_length = 100 diff --git a/requirements-dev.txt b/requirements-dev.txt index 0a9a1dacc..fe3bd8841 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ black check-manifest codespell -docker-compose +docker-compose # Not available in py3.10 flake8 flake8-builtins flake8-comprehensions diff --git a/requirements.txt b/requirements.txt index b79f2c9fe..0bba1ef8a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,7 @@ +# This file is redundant with setup.cfg; +# it exists to let GitHub build the repository dependency graph +# https://help.github.com/en/github/visualizing-repository-data-with-graphs/listing-the-packages-that-a-repository-depends-on + dask[array] netCDF4 numpy diff --git a/setup.cfg b/setup.cfg index d20c4f339..5185bf6c0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,6 +14,10 @@ classifiers = License :: OSI Approved :: Apache Software License Operating System :: OS Independent Programming Language :: Python + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 Topic :: Scientific/Engineering author = Wu-Jung Lee author_email = leewujung@gmail.com @@ -38,61 +42,8 @@ install_requires = python_requires = >=3.8 [options.extras_require] -dev = - black - check-manifest - codespell - docker-compose - flake8 - flake8-builtins - flake8-comprehensions - flake8-mutable - flake8-print - isort - mypy - numpydoc - pre-commit - pylint - pytest - pytest-cov - pytest-flake8 - pytest-xdist - setuptools_scm - sphinx - sphinx-automodapi - sphinx_rtd_theme - sphinxcontrib-mermaid - twine - wheel plot = matplotlib cmocean -docs = - sphinx_rtd_theme - sphinx-automodapi - sphinxcontrib-mermaid - jupyter-book - numpydoc - docutils<0.18 all = - %(dev)s %(plot)s - %(docs)s - -[black] -line-length = 100 - -[flake8] -max-line-length = 100 -output-file = flake8_log.txt -tee = True -ignore = E722,E203,W503,T001 - -[isort] -known_first_party = echopype -known_third_party = _echopype_version,setuptools -multi_line_output = 3 -include_trailing_comma = True -force_grid_wrap = 0 -combine_as_imports = True -line_length = 100 From 8a2a03949f194d1dcc8e4eecbe40ff7dd5d1340d Mon Sep 17 00:00:00 2001 From: lsetiawan Date: Mon, 28 Mar 2022 13:42:18 -0700 Subject: [PATCH 5/8] Fix isort and flak8 configs in toml --- .flake8 | 4 ++++ pyproject.toml | 14 ++++---------- 2 files changed, 8 insertions(+), 10 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..bd60313a8 --- /dev/null +++ b/.flake8 @@ -0,0 +1,4 @@ +max-line-length = 100 +output-file = flake8_log.txt +tee = True +ignore = E722,E203,W503,T001 diff --git a/pyproject.toml b/pyproject.toml index 6c2859584..64ea29a37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,17 +17,11 @@ write_to_template = 'version = "{version}" # noqa' [tool.black] line-length = 100 -[tool.flake8] -max-line-length = 100 -output-file = flake8_log.txt -tee = True -ignore = E722,E203,W503,T001 - [tool.isort] -known_first_party = echopype -known_third_party = _echopype_version,setuptools +known_first_party = "echopype" +known_third_party = ["_echopype_version", "setuptools"] multi_line_output = 3 -include_trailing_comma = True +include_trailing_comma = true force_grid_wrap = 0 -combine_as_imports = True +combine_as_imports = true line_length = 100 From 78b42898a5464e2585b76ca80ac6afb3d3277580 Mon Sep 17 00:00:00 2001 From: lsetiawan Date: Mon, 28 Mar 2022 13:47:35 -0700 Subject: [PATCH 6/8] Add flake8 section header --- .flake8 | 1 + 1 file changed, 1 insertion(+) diff --git a/.flake8 b/.flake8 index bd60313a8..1254d6a7c 100644 --- a/.flake8 +++ b/.flake8 @@ -1,3 +1,4 @@ +[flake8] max-line-length = 100 output-file = flake8_log.txt tee = True From 4dad804cb4481b523ec1d6ac0a48e5bdfa1b752e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 28 Mar 2022 20:48:26 +0000 Subject: [PATCH 7/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .ci_helpers/docker/setup-services.py | 12 +- .ci_helpers/run-test.py | 4 +- echopype/calibrate/api.py | 8 +- echopype/calibrate/calibrate_azfp.py | 16 +- echopype/calibrate/calibrate_base.py | 37 +- echopype/calibrate/calibrate_ek.py | 104 +-- echopype/calibrate/ecs_parser.py | 16 +- echopype/convert/api.py | 4 +- echopype/convert/parse_ad2cp.py | 747 ++++++++----------- echopype/convert/parse_azfp.py | 41 +- echopype/convert/parse_base.py | 67 +- echopype/convert/set_groups_ad2cp.py | 32 +- echopype/convert/set_groups_azfp.py | 13 +- echopype/convert/set_groups_base.py | 22 +- echopype/convert/set_groups_ek60.py | 47 +- echopype/convert/set_groups_ek80.py | 21 +- echopype/convert/utils/ek_date_conversion.py | 4 +- echopype/convert/utils/ek_raw_io.py | 8 +- echopype/convert/utils/ek_raw_parsers.py | 91 +-- echopype/echodata/combine.py | 28 +- echopype/echodata/convention/conv.py | 4 +- echopype/echodata/echodata.py | 64 +- echopype/metrics/summary_statistics.py | 14 +- echopype/preprocess/api.py | 16 +- echopype/qc/api.py | 4 +- echopype/utils/io.py | 20 +- echopype/utils/repr.py | 3 +- echopype/utils/uwa.py | 28 +- 28 files changed, 514 insertions(+), 961 deletions(-) diff --git a/.ci_helpers/docker/setup-services.py b/.ci_helpers/docker/setup-services.py index 4172383f9..66e61e209 100755 --- a/.ci_helpers/docker/setup-services.py +++ b/.ci_helpers/docker/setup-services.py @@ -17,9 +17,7 @@ def parse_args(): parser = argparse.ArgumentParser(description="Setup services for testing") - parser.add_argument( - "--deploy", action="store_true", help="Flag to setup docker services" - ) + parser.add_argument("--deploy", action="store_true", help="Flag to setup docker services") parser.add_argument( "--tear-down", action="store_true", @@ -36,9 +34,7 @@ def parse_args(): sys.exit(1) if not any([args.deploy, args.tear_down]): - print( - "Please provide either --deploy or --tear-down flags. For more help use --help flag." - ) + print("Please provide either --deploy or --tear-down flags. For more help use --help flag.") sys.exit(0) if args.deploy: @@ -48,9 +44,7 @@ def parse_args(): os.system(f"docker-compose -f {COMPOSE_FILE} pull") print("3) Bringing up services.") - os.system( - f"docker-compose -f {COMPOSE_FILE} up -d --remove-orphans --force-recreate" - ) + os.system(f"docker-compose -f {COMPOSE_FILE} up -d --remove-orphans --force-recreate") print(f"4) Deleting old test folder at {TEST_DATA_PATH}") if TEST_DATA_PATH.exists(): diff --git a/.ci_helpers/run-test.py b/.ci_helpers/run-test.py index 8bd031e96..a73f164a3 100644 --- a/.ci_helpers/run-test.py +++ b/.ci_helpers/run-test.py @@ -45,9 +45,7 @@ default="", help="Comma separated list of changed files.", ) - parser.add_argument( - "--pytest-args", type=str, help="Optional pytest args", default="" - ) + parser.add_argument("--pytest-args", type=str, help="Optional pytest args", default="") parser.add_argument( "--local", action="store_true", diff --git a/echopype/calibrate/api.py b/echopype/calibrate/api.py index 92c87c27a..1c85b5a35 100644 --- a/echopype/calibrate/api.py +++ b/echopype/calibrate/api.py @@ -20,9 +20,7 @@ def _compute_cal( # Check on waveform_mode and encode_mode inputs if echodata.sonar_model == "EK80": if waveform_mode is None or encode_mode is None: - raise ValueError( - "waveform_mode and encode_mode must be specified for EK80 calibration" - ) + raise ValueError("waveform_mode and encode_mode must be specified for EK80 calibration") elif waveform_mode not in ("BB", "CW"): raise ValueError("Input waveform_mode not recognized!") elif encode_mode not in ("complex", "power"): @@ -54,9 +52,7 @@ def _compute_cal( # Perform calibration if cal_type == "Sv": - sv_dataset = cal_obj.compute_Sv( - waveform_mode=waveform_mode, encode_mode=encode_mode - ) + sv_dataset = cal_obj.compute_Sv(waveform_mode=waveform_mode, encode_mode=encode_mode) if "water_level" in echodata.platform.data_vars.keys(): # add water_level to the created xr.Dataset diff --git a/echopype/calibrate/calibrate_azfp.py b/echopype/calibrate/calibrate_azfp.py index 3d2642769..e2f08e4c4 100644 --- a/echopype/calibrate/calibrate_azfp.py +++ b/echopype/calibrate/calibrate_azfp.py @@ -31,9 +31,7 @@ def get_cal_params(self, cal_params): # Params from the Beam group for p in ["EL", "DS", "TVR", "VTX", "Sv_offset", "equivalent_beam_angle"]: # substitute if None in user input - self.cal_params[p] = ( - cal_params[p] if p in cal_params else self.echodata.beam[p] - ) + self.cal_params[p] = cal_params[p] if p in cal_params else self.echodata.beam[p] def get_env_params(self): """Get env params using user inputs or values from data file. @@ -51,9 +49,7 @@ def get_env_params(self): # Salinity and pressure always come from user input if ("salinity" not in self.env_params) or ("pressure" not in self.env_params): - raise ReferenceError( - "Please supply both salinity and pressure in env_params." - ) + raise ReferenceError("Please supply both salinity and pressure in env_params.") else: self.env_params["salinity"] = self.env_params["salinity"] self.env_params["pressure"] = self.env_params["pressure"] @@ -84,9 +80,7 @@ def compute_range_meter(self, cal_type): 'Sv' for calculating volume backscattering strength, or 'Sp' for calculating point backscattering strength """ - self.range_meter = self.echodata.compute_range( - self.env_params, azfp_cal_type=cal_type - ) + self.range_meter = self.echodata.compute_range(self.env_params, azfp_cal_type=cal_type) def _cal_power(self, cal_type, **kwargs): """Calibrate to get volume backscattering strength (Sv) from AZFP power data. @@ -112,9 +106,7 @@ def _cal_power(self, cal_type, **kwargs): # scaling factor (slope) in Fig.G-1, units Volts/dB], see p.84 a = self.cal_params["DS"] EL = ( - self.cal_params["EL"] - - 2.5 / a - + self.echodata.beam.backscatter_r / (26214 * a) + self.cal_params["EL"] - 2.5 / a + self.echodata.beam.backscatter_r / (26214 * a) ) # eq.(5) if cal_type == "Sv": diff --git a/echopype/calibrate/calibrate_base.py b/echopype/calibrate/calibrate_base.py index bca3aa3f5..7b76007e6 100644 --- a/echopype/calibrate/calibrate_base.py +++ b/echopype/calibrate/calibrate_base.py @@ -83,9 +83,7 @@ def __init__( >>> echopype.calibrate.compute_Sv(echodata, env_params=env_params) """ # noqa if interp_method not in VALID_INTERP_METHODS[data_kind]: - raise ValueError( - f"invalid interp_method {interp_method} for data_kind {data_kind}" - ) + raise ValueError(f"invalid interp_method {interp_method} for data_kind {data_kind}") self.env_params = env_params self.data_kind = data_kind @@ -112,13 +110,9 @@ def _apply(self, echodata) -> Dict[str, xr.DataArray]: if self.data_kind == "mobile": if np.isnan(echodata.platform["location_time"]).all(): - raise ValueError( - "cannot perform mobile interpolation without location_time" - ) + raise ValueError("cannot perform mobile interpolation without location_time") # compute_range needs indexing by ping_time - interp_plat = echodata.platform.interp( - {"location_time": echodata.beam["ping_time"]} - ) + interp_plat = echodata.platform.interp({"location_time": echodata.beam["ping_time"]}) result = {} for var, values in env_params.data_vars.items(): @@ -132,9 +126,7 @@ def _apply(self, echodata) -> Dict[str, xr.DataArray]: interp_plat["longitude"].data, ) ) - interp = scipy.interpolate.griddata( - points, values, xi, method=self.interp_method - ) + interp = scipy.interpolate.griddata(points, values, xi, method=self.interp_method) result[var] = ("ping_time", interp) env_params = xr.Dataset( data_vars=result, coords={"ping_time": interp_plat["ping_time"]} @@ -142,8 +134,7 @@ def _apply(self, echodata) -> Dict[str, xr.DataArray]: else: # TODO: organized case min_max = { - dim: {"min": env_params[dim].min(), "max": env_params[dim].max()} - for dim in dims + dim: {"min": env_params[dim].min(), "max": env_params[dim].max()} for dim in dims } extrap = env_params.interp( @@ -153,25 +144,18 @@ def _apply(self, echodata) -> Dict[str, xr.DataArray]: kwargs={"fill_value": "extrapolate" if len(dims) == 1 else None}, ) # only keep unique indexes; xarray requires that indexes be unique - extrap_unique_idx = { - dim: np.unique(extrap[dim], return_index=True)[1] for dim in dims - } + extrap_unique_idx = {dim: np.unique(extrap[dim], return_index=True)[1] for dim in dims} extrap = extrap.isel(**extrap_unique_idx) interp = env_params.interp( {dim: echodata.platform[dim].data for dim in dims}, method=self.interp_method, ) - interp_unique_idx = { - dim: np.unique(interp[dim], return_index=True)[1] for dim in dims - } + interp_unique_idx = {dim: np.unique(interp[dim], return_index=True)[1] for dim in dims} interp = interp.isel(**interp_unique_idx) if self.extrap_method is not None: less = extrap.sel( - { - dim: extrap[dim][extrap[dim] < min_max[dim]["min"]] - for dim in dims - } + {dim: extrap[dim][extrap[dim] < min_max[dim]["min"]] for dim in dims} ) middle = interp.sel( { @@ -185,10 +169,7 @@ def _apply(self, echodata) -> Dict[str, xr.DataArray]: } ) greater = extrap.sel( - { - dim: extrap[dim][extrap[dim] > min_max[dim]["max"]] - for dim in dims - } + {dim: extrap[dim][extrap[dim] > min_max[dim]["max"]] for dim in dims} ) # remove empty datasets (xarray does not allow any dims from any datasets diff --git a/echopype/calibrate/calibrate_ek.py b/echopype/calibrate/calibrate_ek.py index d433269a4..4faa65ddc 100644 --- a/echopype/calibrate/calibrate_ek.py +++ b/echopype/calibrate/calibrate_ek.py @@ -93,9 +93,7 @@ def _get_vend_cal_params_power(self, param, waveform_mode): pulse_length = ds_vend["pulse_length"][relevant_indexes] # Find index with correct pulse length - idx_wanted = np.abs(pulse_length - unique_pulse_length).argmin( - dim="pulse_length_bin" - ) + idx_wanted = np.abs(pulse_length - unique_pulse_length).argmin(dim="pulse_length_bin") # Checks for dask array and compute first if isinstance(idx_wanted.data, Array): @@ -303,9 +301,7 @@ def __init__(self, echodata, env_params, cal_params, waveform_mode, encode_mode) self.get_env_params(waveform_mode=waveform_mode, encode_mode=encode_mode) if cal_params is None: cal_params = {} - self.get_cal_params( - cal_params, waveform_mode=waveform_mode, encode_mode=encode_mode - ) + self.get_cal_params(cal_params, waveform_mode=waveform_mode, encode_mode=encode_mode) # self.range_meter computed under self._compute_cal() # because the implementation is different depending on waveform_mode and encode_mode @@ -381,9 +377,7 @@ def get_env_params(self, waveform_mode=None, encode_mode="complex"): ["temperature", "salinity", "depth"], ): self.env_params[p1] = ( - self.env_params[p1] - if p1 in self.env_params - else self.echodata.environment[p2] + self.env_params[p1] if p1 in self.env_params else self.echodata.environment[p2] ) self.env_params["sound_speed"] = ( self.env_params["sound_speed"] @@ -423,9 +417,7 @@ def _get_vend_cal_params_complex(self, channel_id, filter_name, param_type): v = np.expand_dims(v, axis=0) # expand dims for convolution return v else: - return self.echodata.vendor.attrs[ - "%s %s decimation" % (channel_id, filter_name) - ] + return self.echodata.vendor.attrs["%s %s decimation" % (channel_id, filter_name)] def _tapered_chirp( self, @@ -480,9 +472,7 @@ def _filter_decimate_chirp(self, y, ch_id): pc_fil = [pc_fil.squeeze()] ytx_pc = signal.convolve(ytx_wbt_deci, pc_fil) ytx_pc_deci = ytx_pc[0::pc_decifac] - ytx_pc_deci_time = ( - np.arange(ytx_pc_deci.size) * 1 / self.fs * wbt_decifac * pc_decifac - ) + ytx_pc_deci_time = np.arange(ytx_pc_deci.size) * 1 / self.fs * wbt_decifac * pc_decifac return ytx_pc_deci, ytx_pc_deci_time @@ -501,9 +491,7 @@ def _get_tau_effective(ytx, fs_deci, waveform_mode): ``BB`` for BB-mode samples, recorded as complex samples """ if waveform_mode == "BB": - ytxa = ( - signal.convolve(ytx, np.flip(np.conj(ytx))) / np.linalg.norm(ytx) ** 2 - ) + ytxa = signal.convolve(ytx, np.flip(np.conj(ytx))) / np.linalg.norm(ytx) ** 2 ptxa = abs(ytxa) ** 2 elif waveform_mode == "CW": ptxa = np.abs(ytx) ** 2 # energy of transmit signal @@ -556,18 +544,12 @@ def get_transmit_chirp(self, waveform_mode): y_tmp, _ = self._tapered_chirp(**tx_params) # Filter and decimate chirp template - channel_id = str( - self.echodata.beam.sel(frequency=freq)["channel_id"].values - ) - fs_deci = ( - 1 / self.echodata.beam.sel(frequency=freq)["sample_interval"].values - ) + channel_id = str(self.echodata.beam.sel(frequency=freq)["channel_id"].values) + fs_deci = 1 / self.echodata.beam.sel(frequency=freq)["sample_interval"].values y_tmp, y_tmp_time = self._filter_decimate_chirp(y_tmp, channel_id) # Compute effective pulse length - tau_effective_tmp = self._get_tau_effective( - y_tmp, fs_deci, waveform_mode=waveform_mode - ) + tau_effective_tmp = self._get_tau_effective(y_tmp, fs_deci, waveform_mode=waveform_mode) y_all[channel_id] = y_tmp y_time_all[channel_id] = y_tmp_time @@ -598,15 +580,11 @@ def compress_pulse(self, chirp, freq_BB=None): .dropna(dim="quadrant", how="all") .dropna(dim="ping_time") ) - channel_id = str( - self.echodata.beam.sel(frequency=freq)["channel_id"].values - ) + channel_id = str(self.echodata.beam.sel(frequency=freq)["channel_id"].values) replica = xr.DataArray(np.conj(chirp[channel_id]), dims="window") # Pulse compression via rolling pc = ( - backscatter_freq.rolling(range_sample=replica.size) - .construct("window") - .dot(replica) + backscatter_freq.rolling(range_sample=replica.size).construct("window").dot(replica) / np.linalg.norm(chirp[channel_id]) ** 2 ) # Expand dimension and add name to allow merge @@ -653,9 +631,9 @@ def _get_gain_for_complex(self, waveform_mode, freq_center) -> xr.DataArray: if ch_id in self.echodata.vendor.cal_channel_id: gain_vec = self.echodata.vendor.gain.sel(cal_channel_id=ch_id) gain_temp = ( - gain_vec.interp( - cal_frequency=freq_center.sel(frequency=fn) - ).drop(["cal_channel_id", "cal_frequency"]) + gain_vec.interp(cal_frequency=freq_center.sel(frequency=fn)).drop( + ["cal_channel_id", "cal_frequency"] + ) ).expand_dims("frequency") # if no freq-dependent gain use CW gain else: @@ -663,9 +641,7 @@ def _get_gain_for_complex(self, waveform_mode, freq_center) -> xr.DataArray: gain_single.sel(frequency=fn) .assign_coords(ping_time=np.datetime64(0, "ns")) .expand_dims("ping_time") - .reindex_like( - self.echodata.beam.backscatter_r, method="nearest" - ) + .reindex_like(self.echodata.beam.backscatter_r, method="nearest") .expand_dims("frequency") ) gain_temp.name = "gain" @@ -706,13 +682,9 @@ def _cal_complex(self, cal_type, waveform_mode) -> xr.Dataset: # use center frequency for each ping to select BB or CW channels # when all samples are encoded as complex samples - if ( - "frequency_start" in self.echodata.beam - and "frequency_end" in self.echodata.beam - ): + if "frequency_start" in self.echodata.beam and "frequency_end" in self.echodata.beam: freq_center = ( - self.echodata.beam["frequency_start"] - + self.echodata.beam["frequency_end"] + self.echodata.beam["frequency_start"] + self.echodata.beam["frequency_end"] ) / 2 else: freq_center = None @@ -747,8 +719,7 @@ def _cal_complex(self, cal_type, waveform_mode) -> xr.Dataset: # backscatter data backscatter_cw = ( - self.echodata.beam["backscatter_r"] - + 1j * self.echodata.beam["backscatter_i"] + self.echodata.beam["backscatter_r"] + 1j * self.echodata.beam["backscatter_i"] ) prx = ( self.echodata.beam.quadrant.size @@ -762,29 +733,21 @@ def _cal_complex(self, cal_type, waveform_mode) -> xr.Dataset: # derived params sound_speed = self.env_params["sound_speed"].squeeze() - absorption = ( - self.env_params["sound_absorption"] - .sel(frequency=freq_sel.frequency) - .squeeze() - ) + absorption = self.env_params["sound_absorption"].sel(frequency=freq_sel.frequency).squeeze() range_meter = self.range_meter.sel(frequency=freq_sel.frequency).squeeze() if waveform_mode == "BB": # use true center frequency for BB pulse wavelength = sound_speed / freq_sel # use true center frequency to interpolate for gain factor - gain = self._get_gain_for_complex( - waveform_mode=waveform_mode, freq_center=freq_sel - ) + gain = self._get_gain_for_complex(waveform_mode=waveform_mode, freq_center=freq_sel) else: # use nominal channel frequency for CW pulse wavelength = sound_speed / freq_sel # use nominal channel frequency to select gain factor - gain = self._get_gain_for_complex( - waveform_mode=waveform_mode, freq_center=freq_sel - ) + gain = self._get_gain_for_complex(waveform_mode=waveform_mode, freq_center=freq_sel) # Transmission loss spreading_loss = 20 * np.log10(range_meter.where(range_meter >= 1, other=1)) @@ -801,9 +764,7 @@ def _cal_complex(self, cal_type, waveform_mode) -> xr.Dataset: ).sel(frequency=freq_sel.frequency) # other params - transmit_power = self.echodata.beam["transmit_power"].sel( - frequency=freq_sel.frequency - ) + transmit_power = self.echodata.beam["transmit_power"].sel(frequency=freq_sel.frequency) if waveform_mode == "BB": psifc = self.echodata.beam["equivalent_beam_angle"].sel( frequency=freq_sel.frequency @@ -817,10 +778,7 @@ def _cal_complex(self, cal_type, waveform_mode) -> xr.Dataset: 10 * np.log10(prx) + spreading_loss + absorption_loss - - 10 - * np.log10( - wavelength ** 2 * transmit_power * sound_speed / (32 * np.pi ** 2) - ) + - 10 * np.log10(wavelength ** 2 * transmit_power * sound_speed / (32 * np.pi ** 2)) - 2 * gain - 10 * np.log10(tau_effective) - psifc @@ -828,9 +786,7 @@ def _cal_complex(self, cal_type, waveform_mode) -> xr.Dataset: out = out.rename_vars({list(out.data_vars.keys())[0]: "Sv"}) elif cal_type == "Sp": - transmit_power = self.echodata.beam["transmit_power"].sel( - frequency=freq_sel.frequency - ) + transmit_power = self.echodata.beam["transmit_power"].sel(frequency=freq_sel.frequency) out = ( 10 * np.log10(prx) @@ -897,9 +853,7 @@ def _compute_cal(self, cal_type, waveform_mode, encode_mode) -> xr.Dataset: # BB: complex only, CW: complex or power if waveform_mode == "BB": if encode_mode == "power": # BB waveform forces to collect complex samples - raise ValueError( - "encode_mode='power' not allowed when waveform_mode='BB'!" - ) + raise ValueError("encode_mode='power' not allowed when waveform_mode='BB'!") flag_complex = True else: # waveform_mode="CW" if encode_mode == "complex": @@ -941,9 +895,7 @@ def _compute_cal(self, cal_type, waveform_mode, encode_mode) -> xr.Dataset: "Only complex samples are calibrated, but power samples also exist in the raw data file!" # noqa ) else: # only power OR complex samples exist - if ( - "quadrant" in self.echodata.beam.dims - ): # data contain only complex samples + if "quadrant" in self.echodata.beam.dims: # data contain only complex samples if encode_mode == "power": raise TypeError( "File does not contain power samples! Use encode_mode='complex'" @@ -957,9 +909,7 @@ def _compute_cal(self, cal_type, waveform_mode, encode_mode) -> xr.Dataset: # Compute Sv if flag_complex: # Complex samples can be BB or CW - self.compute_range_meter( - waveform_mode=waveform_mode, encode_mode=encode_mode - ) + self.compute_range_meter(waveform_mode=waveform_mode, encode_mode=encode_mode) ds_cal = self._cal_complex(cal_type=cal_type, waveform_mode=waveform_mode) else: # Power samples only make sense for CW mode data diff --git a/echopype/calibrate/ecs_parser.py b/echopype/calibrate/ecs_parser.py index 827c27716..11366af55 100644 --- a/echopype/calibrate/ecs_parser.py +++ b/echopype/calibrate/ecs_parser.py @@ -17,9 +17,7 @@ PARAM_MATCHER = re.compile( r"\s*(?P#?)\s*(?P\w+)\s*=\s*(?P((-?\d+(?:\.\d+))|\w+)?)?\s*#?(.*)\n" # noqa ) -CAL = re.compile( - r"(SourceCal|LocalCal) (?P\w+)\s*\n", re.I # ignore case # noqa -) +CAL = re.compile(r"(SourceCal|LocalCal) (?P\w+)\s*\n", re.I) # ignore case # noqa class ECSParser: @@ -112,9 +110,7 @@ def convert_type(input_dict): for k, v in input_dict.items(): if k == "TvgRangeCorrection": if v not in self.TvgRangeCorrection_allowed_str: - raise ValueError( - "TvgRangeCorrection contains unexpected setting!" - ) + raise ValueError("TvgRangeCorrection contains unexpected setting!") else: input_dict[k] = np.float(v) @@ -144,9 +140,7 @@ def parse(self): status_str = STATUS_CRUDE.match(line)["status"].lower() if "ecs" in status_str: status = "ecs" - self.data_type = ECS_HEADER.match(line)[ - "data_type" - ] # get data type + self.data_type = ECS_HEADER.match(line)["data_type"] # get data type self._parse_header(fid) elif ( "fileset" in status_str @@ -156,9 +150,7 @@ def parse(self): status = STATUS_FINE.match(line)["status"].lower() parsed_params[status] = self._parse_block(fid, status) else: - raise ValueError( - "Expecting a new block but got something else!" - ) + raise ValueError("Expecting a new block but got something else!") line = fid.readline() # read next line # Make FileSet settings dict less awkward diff --git a/echopype/convert/api.py b/echopype/convert/api.py index f64842c7b..82bc4cce0 100644 --- a/echopype/convert/api.py +++ b/echopype/convert/api.py @@ -251,9 +251,7 @@ def _set_convert_params(param_dict: Dict[str, str]) -> Dict[str, str]: out_params["platform_code_ICES"] = param_dict.get("platform_code_ICES", "") out_params["platform_type"] = param_dict.get("platform_type", "") out_params["water_level"] = param_dict.get("water_level", None) - out_params["nmea_gps_sentence"] = param_dict.get( - "nmea_gps_sentence", NMEA_SENTENCE_DEFAULT - ) + out_params["nmea_gps_sentence"] = param_dict.get("nmea_gps_sentence", NMEA_SENTENCE_DEFAULT) # Parameters for the Top-level group out_params["survey_name"] = param_dict.get("survey_name", "") diff --git a/echopype/convert/parse_ad2cp.py b/echopype/convert/parse_ad2cp.py index 0a739bc47..8c4cd3c7c 100644 --- a/echopype/convert/parse_ad2cp.py +++ b/echopype/convert/parse_ad2cp.py @@ -139,9 +139,9 @@ def __init__( # field_entry_data_type: Union[DataType, Callable[["Ad2cpDataPacket"], DataType]], *, field_shape: Union[List[int], Callable[["Ad2cpDataPacket"], List[int]]] = [], - field_dimensions: Union[ - List[Dimension], Callable[[DataRecordType], List[Dimension]] - ] = [Dimension.TIME], + field_dimensions: Union[List[Dimension], Callable[[DataRecordType], List[Dimension]]] = [ + Dimension.TIME + ], field_units: Optional[str] = None, field_unit_conversion: Callable[ ["Ad2cpDataPacket", Union[int, float]], float @@ -240,9 +240,7 @@ def parse_raw(self): with open(self.source_file, "rb") as f: while True: try: - packet = Ad2cpDataPacket( - f, self, self.burst_average_data_record_version - ) + packet = Ad2cpDataPacket(f, self, self.burst_average_data_record_version) self.previous_packet_data_record_type = packet.data_record_type if packet.is_burst(): self.burst_packets.append(packet) @@ -260,9 +258,7 @@ def parse_raw(self): break else: if self.config is None and len(self.string_packets) > 0: - self.config = self.parse_config( - self.string_packets[0].data["string_data"] - ) + self.config = self.parse_config(self.string_packets[0].data["string_data"]) if self.config is not None and "GETCLOCKSTR" in self.config: self.ping_time.append(np.datetime64(self.config["GETCLOCKSTR"]["TIME"])) @@ -413,28 +409,16 @@ def _read_data_record(self, f: BinaryIO): """ if self.data_exclude["id"] in (0x15, 0x18): # burst - if ( - self.burst_average_data_record_version - == BurstAverageDataRecordVersion.VERSION2 - ): + if self.burst_average_data_record_version == BurstAverageDataRecordVersion.VERSION2: self.data_record_type = DataRecordType.BURST_VERSION2 - elif ( - self.burst_average_data_record_version - == BurstAverageDataRecordVersion.VERSION3 - ): + elif self.burst_average_data_record_version == BurstAverageDataRecordVersion.VERSION3: self.data_record_type = DataRecordType.BURST_VERSION3 else: raise ValueError("invalid burst/average data record version") elif self.data_exclude["id"] == 0x16: # average - if ( - self.burst_average_data_record_version - == BurstAverageDataRecordVersion.VERSION2 - ): + if self.burst_average_data_record_version == BurstAverageDataRecordVersion.VERSION2: self.data_record_type = DataRecordType.AVERAGE_VERSION2 - elif ( - self.burst_average_data_record_version - == BurstAverageDataRecordVersion.VERSION3 - ): + elif self.burst_average_data_record_version == BurstAverageDataRecordVersion.VERSION3: self.data_record_type = DataRecordType.AVERAGE_VERSION3 else: raise ValueError("invalid burst/average data record version") @@ -498,9 +482,7 @@ def _read_data(self, f: BinaryIO, data_format: "HeaderOrDataRecordFormat") -> by if callable(field_shape): field_shape = field_shape(self) - raw_field = self._read_exact( - f, field_entry_size_bytes * int(np.prod(field_shape)) - ) + raw_field = self._read_exact(f, field_entry_size_bytes * int(np.prod(field_shape))) raw_bytes += raw_field if len(field_shape) == 0: parsed_field = field_unit_conversion( @@ -509,16 +491,12 @@ def _read_data(self, f: BinaryIO, data_format: "HeaderOrDataRecordFormat") -> by else: # split the field into entries of size field_entry_size_bytes raw_field_entries = [ - raw_field[ - i * field_entry_size_bytes : (i + 1) * field_entry_size_bytes - ] + raw_field[i * field_entry_size_bytes : (i + 1) * field_entry_size_bytes] for i in range(np.prod(field_shape)) ] # parse each entry individually parsed_field_entries = [ - field_unit_conversion( - self, self._parse(raw_field_entry, field_entry_data_type) - ) + field_unit_conversion(self, self._parse(raw_field_entry, field_entry_data_type)) for raw_field_entry in raw_field_entries ] # reshape the list of entries into the correct shape @@ -557,10 +535,7 @@ def _parse(value: bytes, data_type: DataType) -> Any: # Although the specification states that the data is represented in a # signed-magnitude format, an email exchange with Nortek revealed that it is # actually in 2's complement form. - return ( - np.float64(int.from_bytes(value, byteorder="little", signed=True)) - / 2147483648.0 - ) + return np.float64(int.from_bytes(value, byteorder="little", signed=True)) / 2147483648.0 else: raise RuntimeError("unrecognized data type") @@ -657,8 +632,7 @@ def _postprocess(self, field_name): (self.data["dataset_description"] & 0b0000_0000_0011_1000) >> 3, (self.data["dataset_description"] & 0b0000_0001_1100_0000) >> 6, (self.data["dataset_description"] & 0b0000_1110_0000_0000) >> 9, - (self.data["dataset_description"] & 0b0111_0000_0000_0000) - >> 12, + (self.data["dataset_description"] & 0b0111_0000_0000_0000) >> 12, ] if beam > 0 ] @@ -666,9 +640,7 @@ def _postprocess(self, field_name): self.parser.echosounder_raw_packets[-1].data[ "echosounder_raw_beam" ] = self.data_exclude["beams"][0] - elif ( - self.parser.previous_packet_data_record_type.is_echosounder_raw_transmit() - ): + elif self.parser.previous_packet_data_record_type.is_echosounder_raw_transmit(): self.parser.echosounder_raw_transmit_packets[-1].data[ "echosounder_raw_beam" ] = self.data_exclude["beams"][0] @@ -757,9 +729,9 @@ def _postprocess(self, field_name): # The unit conversion for ambiguity velocity is done here because it # requires the velocity_scaling, which is not known # when ambiguity velocity field is parsed - self.data["ambiguity_velocity"] = self.data[ - "ambiguity_velocity" - ] * (10.0 ** self.data["velocity_scaling"]) + self.data["ambiguity_velocity"] = self.data["ambiguity_velocity"] * ( + 10.0 ** self.data["velocity_scaling"] + ) elif field_name == "dataset_description": self.data_exclude["beams"] = [ beam @@ -767,8 +739,7 @@ def _postprocess(self, field_name): self.data["dataset_description"] & 0b0000_0000_0000_1111, (self.data["dataset_description"] & 0b0000_0000_1111_0000) >> 4, (self.data["dataset_description"] & 0b0000_1111_0000_0000) >> 8, - (self.data["dataset_description"] & 0b1111_0000_0000_0000) - >> 12, + (self.data["dataset_description"] & 0b1111_0000_0000_0000) >> 12, ] if beam > 0 ] @@ -776,27 +747,20 @@ def _postprocess(self, field_name): self.parser.echosounder_raw_packets[-1].data[ "echosounder_raw_beam" ] = self.data_exclude["beams"][0] - elif ( - self.parser.previous_packet_data_record_type.is_echosounder_raw_transmit() - ): + elif self.parser.previous_packet_data_record_type.is_echosounder_raw_transmit(): self.parser.echosounder_raw_transmit_packets[-1].data[ "echosounder_raw_beam" ] = self.data_exclude["beams"][0] elif field_name == "status": if self.parser.previous_packet_data_record_type.is_echosounder_raw(): - self.parser.echosounder_raw_packets[-1].data[ - "echosounder_raw_echogram" - ] = ((self.data["status"] >> 12) & 0b1111) + 1 - elif ( - self.parser.previous_packet_data_record_type.is_echosounder_raw_transmit() - ): + self.parser.echosounder_raw_packets[-1].data["echosounder_raw_echogram"] = ( + (self.data["status"] >> 12) & 0b1111 + ) + 1 + elif self.parser.previous_packet_data_record_type.is_echosounder_raw_transmit(): self.parser.echosounder_raw_transmit_packets[-1].data[ "echosounder_raw_echogram" ] = ((self.data["status"] >> 12) & 0b1111) + 1 - elif ( - self.data_record_format - == HeaderOrDataRecordFormats.BOTTOM_TRACK_DATA_RECORD_FORMAT - ): + elif self.data_record_format == HeaderOrDataRecordFormats.BOTTOM_TRACK_DATA_RECORD_FORMAT: if field_name == "configuration": self.data["pressure_sensor_valid"] = ( self.data["data"]["configuration"] & 0b0000_0000_0000_0001 @@ -833,15 +797,9 @@ def _postprocess(self, field_name): & 0b1111_0000_0000_0000 ) >> 12 elif field_name == "dataset_description": - self.data["beam0"] = ( - self.data["dataset_description"] & 0b0000_0000_0000_1111 - ) - self.data["beam1"] = ( - self.data["dataset_description"] & 0b0000_0000_1111_0000 - ) >> 4 - self.data["beam2"] = ( - self.data["dataset_description"] & 0b0000_1111_0000_0000 - ) >> 8 + self.data["beam0"] = self.data["dataset_description"] & 0b0000_0000_0000_1111 + self.data["beam1"] = (self.data["dataset_description"] & 0b0000_0000_1111_0000) >> 4 + self.data["beam2"] = (self.data["dataset_description"] & 0b0000_1111_0000_0000) >> 8 self.data["beam3"] = ( self.data["dataset_description"] & 0b1111_0000_0000_0000 ) >> 12 @@ -854,16 +812,11 @@ def _postprocess(self, field_name): 10.0 ** self.data["velocity_scaling"] ) elif ( - self.data_record_format - == HeaderOrDataRecordFormats.ECHOSOUNDER_RAW_DATA_RECORD_FORMAT + self.data_record_format == HeaderOrDataRecordFormats.ECHOSOUNDER_RAW_DATA_RECORD_FORMAT ): if field_name == "echosounder_raw_samples": - self.data["echosounder_raw_samples_i"] = self.data[ - "echosounder_raw_samples" - ][:, 0] - self.data["echosounder_raw_samples_q"] = self.data[ - "echosounder_raw_samples" - ][:, 1] + self.data["echosounder_raw_samples_i"] = self.data["echosounder_raw_samples"][:, 0] + self.data["echosounder_raw_samples_q"] = self.data["echosounder_raw_samples"][:, 1] del self.data["echosounder_raw_samples"] elif field_name == "echosounder_raw_transmit_samples": self.data["echosounder_raw_transmit_samples_i"] = self.data[ @@ -936,9 +889,7 @@ def fields_iter(self) -> Iterable[Field]: class HeaderOrDataRecordFormats: @classmethod - def data_record_format( - cls, data_record_type: DataRecordType - ) -> HeaderOrDataRecordFormat: + def data_record_format(cls, data_record_type: DataRecordType) -> HeaderOrDataRecordFormat: """ Returns data record format that should be used to parse the given data record type """ @@ -973,290 +924,288 @@ def data_record_format( ), ] ) - BURST_AVERAGE_VERSION2_DATA_RECORD_FORMAT: HeaderOrDataRecordFormat = ( - HeaderOrDataRecordFormat( - [ - F("version", 1, UNSIGNED_INTEGER), - F("offset_of_data", 1, UNSIGNED_INTEGER, field_units="# of bytes"), - F("serial_number", 4, UNSIGNED_INTEGER), - F("configuration", 2, UNSIGNED_INTEGER), - F("year", 1, UNSIGNED_INTEGER), - F("month", 1, UNSIGNED_INTEGER), - F("day", 1, UNSIGNED_INTEGER), - F("hour", 1, UNSIGNED_INTEGER), - F("minute", 1, UNSIGNED_INTEGER), - F("seconds", 1, UNSIGNED_INTEGER), - F("microsec100", 2, UNSIGNED_INTEGER), - F( - "speed_of_sound", - 2, - UNSIGNED_INTEGER, - field_units="m/s", - field_unit_conversion=lambda packet, x: x / 10, - ), - F( - "temperature", - 2, - SIGNED_INTEGER, - field_units="degrees Celsius", - field_unit_conversion=lambda packet, x: x / 100, - ), - F( - "pressure", - 4, - UNSIGNED_INTEGER, - field_units="dBar", - field_unit_conversion=lambda packet, x: x / 1000, - ), - F( - "heading", - 2, - UNSIGNED_INTEGER, - field_units="degrees", - field_unit_conversion=lambda packet, x: x / 100, - ), - F( - "pitch", - 2, - SIGNED_INTEGER, - field_units="degrees", - field_unit_conversion=lambda packet, x: x / 100, - ), - F( - "roll", - 2, - SIGNED_INTEGER, - field_units="degrees", - field_unit_conversion=lambda packet, x: x / 100, - ), - F("error", 2, UNSIGNED_INTEGER), - F("status", 2, UNSIGNED_INTEGER), - F("num_beams_and_coordinate_system_and_num_cells", 2, UNSIGNED_INTEGER), - F( - "cell_size", - 2, - UNSIGNED_INTEGER, - field_units="m", - field_unit_conversion=lambda packet, x: x / 1000, - ), - F( - "blanking", - 2, - UNSIGNED_INTEGER, - field_units="m", - field_unit_conversion=lambda packet, x: x / 1000, - ), - F( - "velocity_range", - 2, - UNSIGNED_INTEGER, - field_units="m/s", - field_unit_conversion=lambda packet, x: x / 1000, - ), - F( - "battery_voltage", - 2, - UNSIGNED_INTEGER, - field_units="V", - field_unit_conversion=lambda packet, x: x / 10, - ), - F("magnetometer_raw_x", 2, SIGNED_INTEGER), - F("magnetometer_raw_y", 2, SIGNED_INTEGER), - F("magnetometer_raw_z", 2, SIGNED_INTEGER), - F( - "accelerometer_raw_x_axis", - 2, - SIGNED_INTEGER, - field_unit_conversion=lambda packet, x: x / 16384 * 9.819, - ), - F( - "accelerometer_raw_y_axis", - 2, - SIGNED_INTEGER, - field_unit_conversion=lambda packet, x: x / 16384 * 9.819, - ), - F( - "accelerometer_raw_z_axis", - 2, - SIGNED_INTEGER, - field_unit_conversion=lambda packet, x: x / 16384 * 9.819, - ), - F( - "ambiguity_velocity", - 2, - UNSIGNED_INTEGER, - field_units="m/s", - field_unit_conversion=lambda packet, x: x / 10000, - ), - F("dataset_description", 2, UNSIGNED_INTEGER), - F("transmit_energy", 2, UNSIGNED_INTEGER), - F("velocity_scaling", 1, SIGNED_INTEGER), - F("power_level", 1, SIGNED_INTEGER, field_units="dB"), - F(None, 4, UNSIGNED_INTEGER), - F( # used when burst - "velocity_data_burst", - 2, - SIGNED_INTEGER, - field_shape=lambda packet: [ - packet.data.get("num_beams", 0), - packet.data.get("num_cells", 0), - ], - field_dimensions=lambda data_record_type: [ - Dimension.TIME_BURST, - Dimension.BEAM, - range_sample(data_record_type), - ], - field_units="m/s", - field_unit_conversion=lambda packet, x: x - * (10.0 ** packet.data["velocity_scaling"]), - field_exists_predicate=lambda packet: packet.is_burst() - and packet.data["velocity_data_included"], - ), - F( # used when average - "velocity_data_average", - 2, - SIGNED_INTEGER, - field_shape=lambda packet: [ - packet.data.get("num_beams", 0), - packet.data.get("num_cells", 0), - ], - field_dimensions=lambda data_record_type: [ - Dimension.TIME_AVERAGE, - Dimension.BEAM, - range_sample(data_record_type), - ], - field_units="m/s", - field_unit_conversion=lambda packet, x: x - * (10.0 ** packet.data["velocity_scaling"]), - field_exists_predicate=lambda packet: packet.is_average() - and packet.data["velocity_data_included"], - ), - F( # used when echosounder - "velocity_data_echosounder", - 2, - SIGNED_INTEGER, - field_shape=lambda packet: [ - packet.data.get("num_beams", 0), - packet.data.get("num_cells", 0), - ], - field_dimensions=lambda data_record_type: [ - Dimension.TIME_ECHOSOUNDER, - Dimension.BEAM, - range_sample(data_record_type), - ], - field_units="m/s", - field_unit_conversion=lambda packet, x: x - * (10.0 ** packet.data["velocity_scaling"]), - field_exists_predicate=lambda packet: packet.is_echosounder() - and packet.data["velocity_data_included"], - ), - F( - "amplitude_data_burst", - 1, - UNSIGNED_INTEGER, - field_shape=lambda packet: [ - packet.data.get("num_beams", 0), - packet.data.get("num_cells", 0), - ], - field_dimensions=lambda data_record_type: [ - Dimension.TIME_BURST, - Dimension.BEAM, - range_sample(data_record_type), - ], - field_units="dB/count", - field_unit_conversion=lambda packet, x: x / 2, - field_exists_predicate=lambda packet: packet.is_burst() - and packet.data["amplitude_data_included"], - ), - F( - "amplitude_data_average", - 1, - UNSIGNED_INTEGER, - field_shape=lambda packet: [ - packet.data.get("num_beams", 0), - packet.data.get("num_cells", 0), - ], - field_dimensions=lambda data_record_type: [ - Dimension.TIME_AVERAGE, - Dimension.BEAM, - range_sample(data_record_type), - ], - field_units="dB/count", - field_unit_conversion=lambda packet, x: x / 2, - field_exists_predicate=lambda packet: packet.is_average() - and packet.data["amplitude_data_included"], - ), - F( - "amplitude_data_echosounder", - 1, - UNSIGNED_INTEGER, - field_shape=lambda packet: [ - packet.data.get("num_beams", 0), - packet.data.get("num_cells", 0), - ], - field_dimensions=lambda data_record_type: [ - Dimension.TIME_ECHOSOUNDER, - Dimension.BEAM, - range_sample(data_record_type), - ], - field_units="dB/count", - field_unit_conversion=lambda packet, x: x / 2, - field_exists_predicate=lambda packet: packet.is_echosounder() - and packet.data["amplitude_data_included"], - ), - F( - "correlation_data_burst", - 1, - UNSIGNED_INTEGER, - field_shape=lambda packet: [ - packet.data.get("num_beams", 0), - packet.data.get("num_cells", 0), - ], - field_dimensions=lambda data_record_type: [ - Dimension.TIME_BURST, - Dimension.BEAM, - range_sample(data_record_type), - ], - field_units="0-100", - field_exists_predicate=lambda packet: packet.is_burst() - and packet.data["correlation_data_included"], - ), - F( - "correlation_data_average", - 1, - UNSIGNED_INTEGER, - field_shape=lambda packet: [ - packet.data.get("num_beams", 0), - packet.data.get("num_cells", 0), - ], - field_dimensions=lambda data_record_type: [ - Dimension.TIME_AVERAGE, - Dimension.BEAM, - range_sample(data_record_type), - ], - field_units="0-100", - field_exists_predicate=lambda packet: packet.is_average() - and packet.data["correlation_data_included"], - ), - F( - "correlation_data_echosounder", - 1, - UNSIGNED_INTEGER, - field_shape=lambda packet: [ - packet.data.get("num_beams", 0), - packet.data.get("num_cells", 0), - ], - field_dimensions=lambda data_record_type: [ - Dimension.TIME_ECHOSOUNDER, - Dimension.BEAM, - range_sample(data_record_type), - ], - field_units="0-100", - field_exists_predicate=lambda packet: packet.is_echosounder() - and packet.data["correlation_data_included"], - ), - ] - ) + BURST_AVERAGE_VERSION2_DATA_RECORD_FORMAT: HeaderOrDataRecordFormat = HeaderOrDataRecordFormat( + [ + F("version", 1, UNSIGNED_INTEGER), + F("offset_of_data", 1, UNSIGNED_INTEGER, field_units="# of bytes"), + F("serial_number", 4, UNSIGNED_INTEGER), + F("configuration", 2, UNSIGNED_INTEGER), + F("year", 1, UNSIGNED_INTEGER), + F("month", 1, UNSIGNED_INTEGER), + F("day", 1, UNSIGNED_INTEGER), + F("hour", 1, UNSIGNED_INTEGER), + F("minute", 1, UNSIGNED_INTEGER), + F("seconds", 1, UNSIGNED_INTEGER), + F("microsec100", 2, UNSIGNED_INTEGER), + F( + "speed_of_sound", + 2, + UNSIGNED_INTEGER, + field_units="m/s", + field_unit_conversion=lambda packet, x: x / 10, + ), + F( + "temperature", + 2, + SIGNED_INTEGER, + field_units="degrees Celsius", + field_unit_conversion=lambda packet, x: x / 100, + ), + F( + "pressure", + 4, + UNSIGNED_INTEGER, + field_units="dBar", + field_unit_conversion=lambda packet, x: x / 1000, + ), + F( + "heading", + 2, + UNSIGNED_INTEGER, + field_units="degrees", + field_unit_conversion=lambda packet, x: x / 100, + ), + F( + "pitch", + 2, + SIGNED_INTEGER, + field_units="degrees", + field_unit_conversion=lambda packet, x: x / 100, + ), + F( + "roll", + 2, + SIGNED_INTEGER, + field_units="degrees", + field_unit_conversion=lambda packet, x: x / 100, + ), + F("error", 2, UNSIGNED_INTEGER), + F("status", 2, UNSIGNED_INTEGER), + F("num_beams_and_coordinate_system_and_num_cells", 2, UNSIGNED_INTEGER), + F( + "cell_size", + 2, + UNSIGNED_INTEGER, + field_units="m", + field_unit_conversion=lambda packet, x: x / 1000, + ), + F( + "blanking", + 2, + UNSIGNED_INTEGER, + field_units="m", + field_unit_conversion=lambda packet, x: x / 1000, + ), + F( + "velocity_range", + 2, + UNSIGNED_INTEGER, + field_units="m/s", + field_unit_conversion=lambda packet, x: x / 1000, + ), + F( + "battery_voltage", + 2, + UNSIGNED_INTEGER, + field_units="V", + field_unit_conversion=lambda packet, x: x / 10, + ), + F("magnetometer_raw_x", 2, SIGNED_INTEGER), + F("magnetometer_raw_y", 2, SIGNED_INTEGER), + F("magnetometer_raw_z", 2, SIGNED_INTEGER), + F( + "accelerometer_raw_x_axis", + 2, + SIGNED_INTEGER, + field_unit_conversion=lambda packet, x: x / 16384 * 9.819, + ), + F( + "accelerometer_raw_y_axis", + 2, + SIGNED_INTEGER, + field_unit_conversion=lambda packet, x: x / 16384 * 9.819, + ), + F( + "accelerometer_raw_z_axis", + 2, + SIGNED_INTEGER, + field_unit_conversion=lambda packet, x: x / 16384 * 9.819, + ), + F( + "ambiguity_velocity", + 2, + UNSIGNED_INTEGER, + field_units="m/s", + field_unit_conversion=lambda packet, x: x / 10000, + ), + F("dataset_description", 2, UNSIGNED_INTEGER), + F("transmit_energy", 2, UNSIGNED_INTEGER), + F("velocity_scaling", 1, SIGNED_INTEGER), + F("power_level", 1, SIGNED_INTEGER, field_units="dB"), + F(None, 4, UNSIGNED_INTEGER), + F( # used when burst + "velocity_data_burst", + 2, + SIGNED_INTEGER, + field_shape=lambda packet: [ + packet.data.get("num_beams", 0), + packet.data.get("num_cells", 0), + ], + field_dimensions=lambda data_record_type: [ + Dimension.TIME_BURST, + Dimension.BEAM, + range_sample(data_record_type), + ], + field_units="m/s", + field_unit_conversion=lambda packet, x: x + * (10.0 ** packet.data["velocity_scaling"]), + field_exists_predicate=lambda packet: packet.is_burst() + and packet.data["velocity_data_included"], + ), + F( # used when average + "velocity_data_average", + 2, + SIGNED_INTEGER, + field_shape=lambda packet: [ + packet.data.get("num_beams", 0), + packet.data.get("num_cells", 0), + ], + field_dimensions=lambda data_record_type: [ + Dimension.TIME_AVERAGE, + Dimension.BEAM, + range_sample(data_record_type), + ], + field_units="m/s", + field_unit_conversion=lambda packet, x: x + * (10.0 ** packet.data["velocity_scaling"]), + field_exists_predicate=lambda packet: packet.is_average() + and packet.data["velocity_data_included"], + ), + F( # used when echosounder + "velocity_data_echosounder", + 2, + SIGNED_INTEGER, + field_shape=lambda packet: [ + packet.data.get("num_beams", 0), + packet.data.get("num_cells", 0), + ], + field_dimensions=lambda data_record_type: [ + Dimension.TIME_ECHOSOUNDER, + Dimension.BEAM, + range_sample(data_record_type), + ], + field_units="m/s", + field_unit_conversion=lambda packet, x: x + * (10.0 ** packet.data["velocity_scaling"]), + field_exists_predicate=lambda packet: packet.is_echosounder() + and packet.data["velocity_data_included"], + ), + F( + "amplitude_data_burst", + 1, + UNSIGNED_INTEGER, + field_shape=lambda packet: [ + packet.data.get("num_beams", 0), + packet.data.get("num_cells", 0), + ], + field_dimensions=lambda data_record_type: [ + Dimension.TIME_BURST, + Dimension.BEAM, + range_sample(data_record_type), + ], + field_units="dB/count", + field_unit_conversion=lambda packet, x: x / 2, + field_exists_predicate=lambda packet: packet.is_burst() + and packet.data["amplitude_data_included"], + ), + F( + "amplitude_data_average", + 1, + UNSIGNED_INTEGER, + field_shape=lambda packet: [ + packet.data.get("num_beams", 0), + packet.data.get("num_cells", 0), + ], + field_dimensions=lambda data_record_type: [ + Dimension.TIME_AVERAGE, + Dimension.BEAM, + range_sample(data_record_type), + ], + field_units="dB/count", + field_unit_conversion=lambda packet, x: x / 2, + field_exists_predicate=lambda packet: packet.is_average() + and packet.data["amplitude_data_included"], + ), + F( + "amplitude_data_echosounder", + 1, + UNSIGNED_INTEGER, + field_shape=lambda packet: [ + packet.data.get("num_beams", 0), + packet.data.get("num_cells", 0), + ], + field_dimensions=lambda data_record_type: [ + Dimension.TIME_ECHOSOUNDER, + Dimension.BEAM, + range_sample(data_record_type), + ], + field_units="dB/count", + field_unit_conversion=lambda packet, x: x / 2, + field_exists_predicate=lambda packet: packet.is_echosounder() + and packet.data["amplitude_data_included"], + ), + F( + "correlation_data_burst", + 1, + UNSIGNED_INTEGER, + field_shape=lambda packet: [ + packet.data.get("num_beams", 0), + packet.data.get("num_cells", 0), + ], + field_dimensions=lambda data_record_type: [ + Dimension.TIME_BURST, + Dimension.BEAM, + range_sample(data_record_type), + ], + field_units="0-100", + field_exists_predicate=lambda packet: packet.is_burst() + and packet.data["correlation_data_included"], + ), + F( + "correlation_data_average", + 1, + UNSIGNED_INTEGER, + field_shape=lambda packet: [ + packet.data.get("num_beams", 0), + packet.data.get("num_cells", 0), + ], + field_dimensions=lambda data_record_type: [ + Dimension.TIME_AVERAGE, + Dimension.BEAM, + range_sample(data_record_type), + ], + field_units="0-100", + field_exists_predicate=lambda packet: packet.is_average() + and packet.data["correlation_data_included"], + ), + F( + "correlation_data_echosounder", + 1, + UNSIGNED_INTEGER, + field_shape=lambda packet: [ + packet.data.get("num_beams", 0), + packet.data.get("num_cells", 0), + ], + field_dimensions=lambda data_record_type: [ + Dimension.TIME_ECHOSOUNDER, + Dimension.BEAM, + range_sample(data_record_type), + ], + field_units="0-100", + field_exists_predicate=lambda packet: packet.is_echosounder() + and packet.data["correlation_data_included"], + ), + ] ) BURST_AVERAGE_VERSION3_DATA_RECORD_FORMAT: HeaderOrDataRecordFormat = HeaderOrDataRecordFormat( [ @@ -1553,17 +1502,13 @@ def data_record_format( 4, FLOAT, field_units="m", - field_exists_predicate=lambda packet: packet.data[ - "altimeter_data_included" - ], + field_exists_predicate=lambda packet: packet.data["altimeter_data_included"], ), F( "altimeter_quality", 2, UNSIGNED_INTEGER, - field_exists_predicate=lambda packet: packet.data[ - "altimeter_data_included" - ], + field_exists_predicate=lambda packet: packet.data["altimeter_data_included"], ), F( "ast_distance", @@ -1607,9 +1552,7 @@ def data_record_format( # sizes were likely incorrectly swapped. 2, UNSIGNED_INTEGER, - field_exists_predicate=lambda packet: packet.data[ - "altimeter_raw_data_included" - ], + field_exists_predicate=lambda packet: packet.data["altimeter_raw_data_included"], ), F( "altimeter_raw_data_sample_distance", @@ -1617,21 +1560,15 @@ def data_record_format( UNSIGNED_INTEGER, field_units="m", field_unit_conversion=lambda packet, x: x / 10000, - field_exists_predicate=lambda packet: packet.data[ - "altimeter_raw_data_included" - ], + field_exists_predicate=lambda packet: packet.data["altimeter_raw_data_included"], ), F( "altimeter_raw_data_samples", 2, SIGNED_FRACTION, - field_shape=lambda packet: [ - packet.data["altimeter_raw_data_num_samples"] - ], + field_shape=lambda packet: [packet.data["altimeter_raw_data_num_samples"]], field_dimensions=[Dimension.TIME, Dimension.NUM_ALTIMETER_SAMPLES], - field_exists_predicate=lambda packet: packet.data[ - "altimeter_raw_data_included" - ], + field_exists_predicate=lambda packet: packet.data["altimeter_raw_data_included"], ), F( "echosounder_data", @@ -1639,18 +1576,14 @@ def data_record_format( # Although the specification says that this should be an unsigned integer, # testing has shown that it should be a signed integer SIGNED_INTEGER, - field_shape=lambda packet: [ - packet.data.get("num_echosounder_cells", 0) - ], + field_shape=lambda packet: [packet.data.get("num_echosounder_cells", 0)], field_dimensions=[ Dimension.TIME_ECHOSOUNDER, Dimension.RANGE_SAMPLE_ECHOSOUNDER, ], field_units="dB/count", field_unit_conversion=lambda packet, x: x / 100, - field_exists_predicate=lambda packet: packet.data[ - "echosounder_data_included" - ], + field_exists_predicate=lambda packet: packet.data["echosounder_data_included"], ), F( "ahrs_rotation_matrix_m11", @@ -1762,9 +1695,7 @@ def data_record_format( range_sample(data_record_type), ], field_units="%", - field_exists_predicate=lambda packet: packet.data[ - "percentage_good_data_included" - ], + field_exists_predicate=lambda packet: packet.data["percentage_good_data_included"], ), # Only the pitch field is labeled as included when the "std dev data included" # bit is set, but this is likely a mistake @@ -1774,9 +1705,7 @@ def data_record_format( SIGNED_INTEGER, field_units="degrees", field_unit_conversion=lambda packet, x: x / 100, - field_exists_predicate=lambda packet: packet.data[ - "std_dev_data_included" - ], + field_exists_predicate=lambda packet: packet.data["std_dev_data_included"], ), F( "std_dev_roll", @@ -1784,9 +1713,7 @@ def data_record_format( SIGNED_INTEGER, field_units="degrees", field_unit_conversion=lambda packet, x: x / 100, - field_exists_predicate=lambda packet: packet.data[ - "std_dev_data_included" - ], + field_exists_predicate=lambda packet: packet.data["std_dev_data_included"], ), F( "std_dev_heading", @@ -1794,9 +1721,7 @@ def data_record_format( SIGNED_INTEGER, field_units="degrees", field_unit_conversion=lambda packet, x: x / 100, - field_exists_predicate=lambda packet: packet.data[ - "std_dev_data_included" - ], + field_exists_predicate=lambda packet: packet.data["std_dev_data_included"], ), F( "std_dev_pressure", @@ -1804,17 +1729,13 @@ def data_record_format( SIGNED_INTEGER, field_units="dBar", field_unit_conversion=lambda packet, x: x / 100, - field_exists_predicate=lambda packet: packet.data[ - "std_dev_data_included" - ], + field_exists_predicate=lambda packet: packet.data["std_dev_data_included"], ), F( None, 24, RAW_BYTES, - field_exists_predicate=lambda packet: packet.data[ - "std_dev_data_included" - ], + field_exists_predicate=lambda packet: packet.data["std_dev_data_included"], ), ] ) @@ -1946,9 +1867,7 @@ def data_record_format( field_units="m/s", field_unit_conversion=lambda packet, x: x * (10.0 ** packet.data["velocity_scaling"]), - field_exists_predicate=lambda packet: packet.data[ - "velocity_data_included" - ], + field_exists_predicate=lambda packet: packet.data["velocity_data_included"], ), F( "distance_data", @@ -1957,9 +1876,7 @@ def data_record_format( field_shape=lambda packet: [packet.data.get("num_beams", 0)], field_dimensions=[Dimension.TIME, Dimension.BEAM], field_unit_conversion=lambda packet, x: x / 1000, - field_exists_predicate=lambda packet: packet.data[ - "distance_data_included" - ], + field_exists_predicate=lambda packet: packet.data["distance_data_included"], ), F( "figure_of_merit_data", @@ -1967,9 +1884,7 @@ def data_record_format( UNSIGNED_INTEGER, field_shape=lambda packet: [packet.data.get("num_beams", 0)], field_dimensions=[Dimension.TIME, Dimension.BEAM], - field_exists_predicate=lambda packet: packet.data[ - "figure_of_merit_data_included" - ], + field_exists_predicate=lambda packet: packet.data["figure_of_merit_data_included"], ), ] ) diff --git a/echopype/convert/parse_azfp.py b/echopype/convert/parse_azfp.py index 369c4b7a2..77f91b9c9 100644 --- a/echopype/convert/parse_azfp.py +++ b/echopype/convert/parse_azfp.py @@ -130,9 +130,7 @@ def compute_battery(N): # Instrument specific constants HEADER_SIZE = 124 - HEADER_FORMAT = ( - ">HHHHIHHHHHHHHHHHHHHHHHHHHHHHHHHHHHBBBBHBBBBBBBBHHHHHHHHHHHHHHHHHHHH" - ) + HEADER_FORMAT = ">HHHHIHHHHHHHHHHHHHHHHHHHHHHHHHHHHHBBBBHBBBBBBBBHHHHHHHHHHHHHHHHHHHH" # Read xml file into dict self.load_AZFP_xml() @@ -192,9 +190,7 @@ def compute_battery(N): ) # Calculate voltage of main battery pack self.unpacked_data["battery_main"].append( - compute_battery( - self.unpacked_data["ancillary"][ping_num][2] - ) + compute_battery(self.unpacked_data["ancillary"][ping_num][2]) ) # If there is a Tx battery pack self.unpacked_data["battery_tx"].append( @@ -209,9 +205,7 @@ def compute_battery(N): self._check_uniqueness() self._get_ping_time() # Explicitly cast frequency to a float in accordance with the SONAR-netCDF4 convention - self.unpacked_data["frequency"] = self.unpacked_data["frequency"].astype( - np.float64 - ) + self.unpacked_data["frequency"] = self.unpacked_data["frequency"].astype(np.float64) @staticmethod def _get_fields(): @@ -275,10 +269,7 @@ def _print_status(self): self.unpacked_data["day"][0], self.unpacked_data["hour"][0], self.unpacked_data["minute"][0], - int( - self.unpacked_data["second"][0] - + self.unpacked_data["hundredths"][0] / 100 - ), + int(self.unpacked_data["second"][0] + self.unpacked_data["hundredths"][0] / 100), ) timestr = timestamp.strftime("%Y-%b-%d %H:%M:%S") pathstr, xml_name = os.path.split(self.xml_path) @@ -331,9 +322,7 @@ def _split_header(self, raw, header_unpacked): for field in fields: if field[0] in field_w_freq: # fields with num_freq data self.unpacked_data[field[0]].append( - header_unpacked[ - header_byte_cnt : header_byte_cnt + self.parameters["num_freq"] - ] + header_unpacked[header_byte_cnt : header_byte_cnt + self.parameters["num_freq"]] ) header_byte_cnt += firmware_freq_len elif len(field) == 3: # other longer fields ('ancillary' and 'ad') @@ -352,17 +341,13 @@ def _add_counts(self, raw, ping_num): for freq_ch in range(self.unpacked_data["num_chan"][ping_num]): counts_byte_size = self.unpacked_data["num_bins"][ping_num][freq_ch] if self.unpacked_data["data_type"][ping_num][freq_ch]: - if self.unpacked_data["avg_pings"][ - ping_num - ]: # if pings are averaged over time + if self.unpacked_data["avg_pings"][ping_num]: # if pings are averaged over time divisor = ( self.unpacked_data["ping_per_profile"][ping_num] * self.unpacked_data["range_samples_per_bin"][ping_num][freq_ch] ) else: - divisor = self.unpacked_data["range_samples_per_bin"][ping_num][ - freq_ch - ] + divisor = self.unpacked_data["range_samples_per_bin"][ping_num][freq_ch] ls = unpack( ">" + "I" * counts_byte_size, raw.read(counts_byte_size * 4) ) # Linear sum @@ -384,9 +369,7 @@ def _check_uniqueness(self): if not self.unpacked_data: self.parse_raw() - if ( - np.array(self.unpacked_data["profile_flag"]).size != 1 - ): # Only check uniqueness once. + if np.array(self.unpacked_data["profile_flag"]).size != 1: # Only check uniqueness once. # fields with num_freq data field_w_freq = ( "dig_rate", @@ -416,17 +399,13 @@ def _check_uniqueness(self): if uniq.shape[0] == 1: self.unpacked_data[field] = uniq.squeeze() else: - raise ValueError( - f"Header value {field} is not constant for each ping" - ) + raise ValueError(f"Header value {field} is not constant for each ping") for field in field_include: uniq = np.unique(self.unpacked_data[field]) if uniq.shape[0] == 1: self.unpacked_data[field] = uniq.squeeze() else: - raise ValueError( - f"Header value {field} is not constant for each ping" - ) + raise ValueError(f"Header value {field} is not constant for each ping") def _get_ping_time(self): """Assemble ping time from parsed values.""" diff --git a/echopype/convert/parse_base.py b/echopype/convert/parse_base.py index f81890f6a..620445741 100644 --- a/echopype/convert/parse_base.py +++ b/echopype/convert/parse_base.py @@ -16,9 +16,7 @@ class ParseBase: def __init__(self, file, storage_options): self.source_file = file - self.timestamp_pattern = ( - None # regex pattern used to grab datetime embedded in filename - ) + self.timestamp_pattern = None # regex pattern used to grab datetime embedded in filename self.ping_time = [] # list to store ping time self.storage_options = storage_options @@ -46,23 +44,15 @@ def __init__(self, file, params, storage_options): ) # Stores the channel ids for each data type (power, angle, complex) self.data_type = self._select_datagrams(params) - self.nmea = defaultdict( - list - ) # Dictionary to store NMEA data(timestamp and string) - self.mru = defaultdict( - list - ) # Dictionary to store MRU data (heading, pitch, roll, heave) - self.fil_coeffs = defaultdict( - dict - ) # Dictionary to store PC and WBT coefficients + self.nmea = defaultdict(list) # Dictionary to store NMEA data(timestamp and string) + self.mru = defaultdict(list) # Dictionary to store MRU data (heading, pitch, roll, heave) + self.fil_coeffs = defaultdict(dict) # Dictionary to store PC and WBT coefficients self.fil_df = defaultdict(dict) # Dictionary to store filter decimation factors self.CON1_datagram = None # Holds the ME70 CON1 datagram def _print_status(self): - time = ( - self.config_datagram["timestamp"].astype(dt).strftime("%Y-%b-%d %H:%M:%S") - ) + time = self.config_datagram["timestamp"].astype(dt).strftime("%Y-%b-%d %H:%M:%S") print( f"{dt.now().strftime('%H:%M:%S')} parsing file {os.path.basename(self.source_file)}, " f"time of first ping: {time}" @@ -70,9 +60,7 @@ def _print_status(self): def parse_raw(self): """Parse raw data file from Simrad EK60, EK80, and EA640 echosounders.""" - with RawSimradFile( - self.source_file, "r", storage_options=self.storage_options - ) as fid: + with RawSimradFile(self.source_file, "r", storage_options=self.storage_options) as fid: self.config_datagram = fid.read(1) self.config_datagram["timestamp"] = np.datetime64( self.config_datagram["timestamp"].replace(tzinfo=None), "[ms]" @@ -82,9 +70,7 @@ def parse_raw(self): if "pulse_duration" not in v and "pulse_length" in v: # it seems like sometimes this field can appear with the name "pulse_length" # and in the form of floats separated by semicolons - v["pulse_duration"] = [ - float(x) for x in v["pulse_length"].split(";") - ] + v["pulse_duration"] = [float(x) for x in v["pulse_length"].split(";")] # If exporting to XML file (EK80/EA640 only), print a message if "print_export_msg" in self.data_type: @@ -135,8 +121,7 @@ def parse_raw(self): self.ping_data_dict[data_type][k] = self.pad_shorter_ping(v) if data_type == "power": self.ping_data_dict[data_type][k] = ( - self.ping_data_dict[data_type][k].astype("float32") - * INDEX2POWER + self.ping_data_dict[data_type][k].astype("float32") * INDEX2POWER ) def _read_datagrams(self, fid): @@ -232,9 +217,7 @@ def _read_datagrams(self, fid): # Skip any datagram that the user does not want to save if ( - not any( - new_datagram["type"].startswith(dgram) for dgram in self.data_type - ) + not any(new_datagram["type"].startswith(dgram) for dgram in self.data_type) and "ALL" not in self.data_type ): continue @@ -249,17 +232,13 @@ def _read_datagrams(self, fid): # Don't parse anything else if only the environment xml is required. if "ENV" in self.data_type: break - elif new_datagram["subtype"] == "parameter" and ( - "ALL" in self.data_type - ): + elif new_datagram["subtype"] == "parameter" and ("ALL" in self.data_type): current_parameters = new_datagram["parameter"] # RAW0 datagrams store raw acoustic data for a channel for EK60 elif new_datagram["type"].startswith("RAW0"): # Save channel-specific ping time. The channels are stored as 1-based indices - self.ping_time[new_datagram["channel"]].append( - new_datagram["timestamp"] - ) + self.ping_time[new_datagram["channel"]].append(new_datagram["timestamp"]) # Append ping by ping data self._append_channel_ping_data(new_datagram) @@ -294,12 +273,12 @@ def _read_datagrams(self, fid): # FIL datagrams contain filters for processing bascatter data for EK80 elif new_datagram["type"].startswith("FIL"): - self.fil_coeffs[new_datagram["channel_id"]][ - new_datagram["stage"] - ] = new_datagram["coefficients"] - self.fil_df[new_datagram["channel_id"]][ - new_datagram["stage"] - ] = new_datagram["decimation_factor"] + self.fil_coeffs[new_datagram["channel_id"]][new_datagram["stage"]] = new_datagram[ + "coefficients" + ] + self.fil_df[new_datagram["channel_id"]][new_datagram["stage"]] = new_datagram[ + "decimation_factor" + ] # TAG datagrams contain time-stamped annotations inserted via the recording software elif new_datagram["type"].startswith("TAG"): @@ -321,9 +300,7 @@ def _append_channel_ping_data(self, datagram): # TODO: do a thorough check with the convention and processing # unsaved = ['channel', 'channel_id', 'low_date', 'high_date', # 'offset', 'frequency' , # 'transmit_mode', 'spare0', 'bytes_read', 'type'] #, 'n_complex'] - ch_id = ( - datagram["channel_id"] if "channel_id" in datagram else datagram["channel"] - ) + ch_id = datagram["channel_id"] if "channel_id" in datagram else datagram["channel"] for k, v in datagram.items(): # if k not in unsaved: self.ping_data_dict[k][ch_id].append(v) @@ -346,9 +323,7 @@ def pad_shorter_ping(data_list) -> np.ndarray: The array is NaN-padded if some pings are of different lengths. """ lens = np.array([len(item) for item in data_list]) - if ( - np.unique(lens).size != 1 - ): # if some pings have different lengths along range + if np.unique(lens).size != 1: # if some pings have different lengths along range if data_list[0].ndim == 2: # Angle data have an extra dimension for alongship and athwartship samples @@ -362,9 +337,7 @@ def pad_shorter_ping(data_list) -> np.ndarray: out_array = np.full(mask.shape, np.nan) # Fill in values - out_array[mask] = np.concatenate(data_list).reshape( - -1 - ) # reshape in case data > 1D + out_array[mask] = np.concatenate(data_list).reshape(-1) # reshape in case data > 1D else: out_array = np.array(data_list) return out_array diff --git a/echopype/convert/set_groups_ad2cp.py b/echopype/convert/set_groups_ad2cp.py index ed337959c..54cf0746f 100644 --- a/echopype/convert/set_groups_ad2cp.py +++ b/echopype/convert/set_groups_ad2cp.py @@ -37,9 +37,7 @@ def combine_packets(self): max_samples = 0 for packet in self.parser_obj.echosounder_raw_packets: # both _r and _i have same dimensions - max_samples = max( - max_samples, packet.data["echosounder_raw_samples_i"].shape[0] - ) + max_samples = max(max_samples, packet.data["echosounder_raw_samples_i"].shape[0]) for packet in self.parser_obj.echosounder_raw_packets: packet.data["echosounder_raw_samples_i"] = np.pad( packet.data["echosounder_raw_samples_i"], @@ -100,9 +98,7 @@ def make_dataset( else: return None - burst_ds = make_dataset( - self.parser_obj.burst_packets, ping_time_dim="ping_time_burst" - ) + burst_ds = make_dataset(self.parser_obj.burst_packets, ping_time_dim="ping_time_burst") average_ds = make_dataset( self.parser_obj.average_packets, ping_time_dim="ping_time_average" ) @@ -224,12 +220,8 @@ def set_beam(self) -> xr.Dataset: "ast_offset_100us": self.ds.get("ast_offset_100us"), "ast_pressure": self.ds.get("ast_pressure"), "altimeter_spare": self.ds.get("altimeter_spare"), - "altimeter_raw_data_num_samples": self.ds.get( - "altimeter_raw_data_num_samples" - ), - "altimeter_raw_data_sample_distance": self.ds.get( - "altimeter_raw_data_sample_distance" - ), + "altimeter_raw_data_num_samples": self.ds.get("altimeter_raw_data_num_samples"), + "altimeter_raw_data_sample_distance": self.ds.get("altimeter_raw_data_sample_distance"), "altimeter_raw_data_samples": self.ds.get("altimeter_raw_data_samples"), } @@ -278,19 +270,13 @@ def set_vendor(self) -> xr.Dataset: "status0": self.ds.get("status0"), "battery_voltage": self.ds.get("battery_voltage"), "power_level": self.ds.get("power_level"), - "temperature_of_pressure_sensor": self.ds.get( - "temperature_from_pressure_sensor" - ), + "temperature_of_pressure_sensor": self.ds.get("temperature_from_pressure_sensor"), "nominal_correlation": self.ds.get("nominal_correlation"), "magnetometer_temperature": self.ds.get("magnetometer_temperature"), - "real_ping_time_clock_temperature": self.ds.get( - "real_ping_time_clock_temperature" - ), + "real_ping_time_clock_temperature": self.ds.get("real_ping_time_clock_temperature"), "ensemble_counter": self.ds.get("ensemble_counter"), "ahrs_rotation_matrix_mij": ( - ("mij", "ping_time") - if "ahrs_rotation_matrix_m11" in self.ds - else "mij", + ("mij", "ping_time") if "ahrs_rotation_matrix_m11" in self.ds else "mij", [ self.ds.get("ahrs_rotation_matrix_m11"), self.ds.get("ahrs_rotation_matrix_m12"), @@ -304,9 +290,7 @@ def set_vendor(self) -> xr.Dataset: ], ), "ahrs_quaternions_wxyz": ( - ("wxyz", "ping_time") - if "ahrs_quaternions_w" in self.ds - else "wxyz", + ("wxyz", "ping_time") if "ahrs_quaternions_w" in self.ds else "wxyz", [ self.ds.get("ahrs_quaternions_w"), self.ds.get("ahrs_quaternions_x"), diff --git a/echopype/convert/set_groups_azfp.py b/echopype/convert/set_groups_azfp.py index c2e662ea4..367a4cd72 100644 --- a/echopype/convert/set_groups_azfp.py +++ b/echopype/convert/set_groups_azfp.py @@ -85,9 +85,7 @@ def set_beam(self) -> xr.Dataset: """Set the Beam group.""" unpacked_data = self.parser_obj.unpacked_data parameters = self.parser_obj.parameters - anc = np.array( - unpacked_data["ancillary"] - ) # convert to np array for easy slicing + anc = np.array(unpacked_data["ancillary"]) # convert to np array for easy slicing dig_rate = unpacked_data["dig_rate"] # dim: freq freq = np.array(unpacked_data["frequency"]) * 1000 # Frequency in Hz ping_time = self.parser_obj.ping_time @@ -102,10 +100,7 @@ def set_beam(self) -> xr.Dataset: ) N.append( np.array( - [ - unpacked_data["counts"][p][ich] - for p in range(len(unpacked_data["year"])) - ] + [unpacked_data["counts"][p][ich] for p in range(len(unpacked_data["year"]))] ) ) @@ -123,9 +118,7 @@ def set_beam(self) -> xr.Dataset: tdn = unpacked_data["pulse_length"] / 1e6 # Convert microseconds to seconds range_samples_xml = np.array(parameters["range_samples"]) # from xml file - range_samples_per_bin = unpacked_data[ - "range_samples_per_bin" - ] # from data header + range_samples_per_bin = unpacked_data["range_samples_per_bin"] # from data header # Calculate sample interval in seconds if len(dig_rate) == len(range_samples_per_bin): diff --git a/echopype/convert/set_groups_base.py b/echopype/convert/set_groups_base.py index b484b9a5b..147900961 100644 --- a/echopype/convert/set_groups_base.py +++ b/echopype/convert/set_groups_base.py @@ -77,8 +77,7 @@ def set_provenance(self) -> xr.Dataset: prov_dict = { "conversion_software_name": "echopype", "conversion_software_version": ECHOPYPE_VERSION, - "conversion_time": dt.utcnow().isoformat(timespec="seconds") - + "Z", # use UTC time + "conversion_time": dt.utcnow().isoformat(timespec="seconds") + "Z", # use UTC time "src_filenames": self.input_file, } # Save @@ -153,9 +152,7 @@ def set_vendor(self) -> xr.Dataset: def _parse_NMEA(self): """Get the lat and lon values from the raw nmea data""" messages = [string[3:6] for string in self.parser_obj.nmea["nmea_string"]] - idx_loc = np.argwhere( - np.isin(messages, self.ui_param["nmea_gps_sentence"]) - ).squeeze() + idx_loc = np.argwhere(np.isin(messages, self.ui_param["nmea_gps_sentence"])).squeeze() if idx_loc.size == 1: # in case of only 1 matching message idx_loc = np.expand_dims(idx_loc, axis=0) nmea_msg = [] @@ -170,26 +167,17 @@ def _parse_NMEA(self): ): nmea_msg.append(None) lat = ( - np.array( - [x.latitude if hasattr(x, "latitude") else np.nan for x in nmea_msg] - ) + np.array([x.latitude if hasattr(x, "latitude") else np.nan for x in nmea_msg]) if nmea_msg else [np.nan] ) lon = ( - np.array( - [x.longitude if hasattr(x, "longitude") else np.nan for x in nmea_msg] - ) + np.array([x.longitude if hasattr(x, "longitude") else np.nan for x in nmea_msg]) if nmea_msg else [np.nan] ) msg_type = ( - np.array( - [ - x.sentence_type if hasattr(x, "sentence_type") else np.nan - for x in nmea_msg - ] - ) + np.array([x.sentence_type if hasattr(x, "sentence_type") else np.nan for x in nmea_msg]) if nmea_msg else [np.nan] ) diff --git a/echopype/convert/set_groups_ek60.py b/echopype/convert/set_groups_ek60.py index 2b255d1d5..fb9814277 100644 --- a/echopype/convert/set_groups_ek60.py +++ b/echopype/convert/set_groups_ek60.py @@ -39,11 +39,8 @@ def __init__(self, *args, **kwargs): if duplicates.any(): if self.old_ping_time is None: if ( - len({arr.shape for arr in self.parser_obj.ping_time.values()}) - == 1 - and np.unique( - np.stack(self.parser_obj.ping_time.values()), axis=0 - ).shape[0] + len({arr.shape for arr in self.parser_obj.ping_time.values()}) == 1 + and np.unique(np.stack(self.parser_obj.ping_time.values()), axis=0).shape[0] == 1 ): self.old_ping_time = self.parser_obj.ping_time[ch] @@ -57,9 +54,7 @@ def __init__(self, *args, **kwargs): backscatter_r = self.parser_obj.ping_data_dict["power"][ch] # indexes of duplicates including the originals # (if there are 2 times that are the same, both will be included) - (all_duplicates_idx,) = np.where( - np.isin(ping_time, ping_time[duplicates][0]) - ) + (all_duplicates_idx,) = np.where(np.isin(ping_time, ping_time[duplicates][0])) if np.array_equal( backscatter_r[all_duplicates_idx[0]], backscatter_r[all_duplicates_idx[1]], @@ -74,9 +69,7 @@ def __init__(self, *args, **kwargs): v[ch] = v[ch][unique_idx] else: v[ch] = [v[ch][i] for i in unique_idx] - self.parser_obj.ping_time[ch] = self.parser_obj.ping_time[ch][ - unique_idx - ] + self.parser_obj.ping_time[ch] = self.parser_obj.ping_time[ch][unique_idx] else: warnings.warn( "duplicate ping times detected; the duplicate times will be incremented by 1 nanosecond and remain in the ping_time coordinate. The original ping times will be preserved in the Provenance group" # noqa @@ -92,8 +85,7 @@ def set_provenance(self) -> xr.Dataset: prov_dict = { "conversion_software_name": "echopype", "conversion_software_version": ECHOPYPE_VERSION, - "conversion_time": dt.utcnow().isoformat(timespec="seconds") - + "Z", # use UTC time + "conversion_time": dt.utcnow().isoformat(timespec="seconds") + "Z", # use UTC time "src_filenames": self.input_file, "duplicate_ping_times": 1 if self.old_ping_time is not None else 0, } @@ -147,11 +139,7 @@ def set_env(self) -> xr.Dataset: ) # Attach frequency dimension/coordinate ds_tmp = ds_tmp.expand_dims( - { - "frequency": [ - self.parser_obj.config_datagram["transceivers"][ch]["frequency"] - ] - } + {"frequency": [self.parser_obj.config_datagram["transceivers"][ch]["frequency"]]} ) ds_tmp["frequency"] = ds_tmp["frequency"].assign_attrs( units="Hz", @@ -276,9 +264,7 @@ def set_platform(self, NMEA_only=False) -> xr.Dataset: ds_tmp = ds_tmp.expand_dims( { "frequency": [ - self.parser_obj.config_datagram["transceivers"][ch][ - "frequency" - ] + self.parser_obj.config_datagram["transceivers"][ch]["frequency"] ] } ) @@ -307,10 +293,7 @@ def set_beam(self) -> xr.Dataset: # Get channel keys and frequency ch_ids = list(self.parser_obj.config_datagram["transceivers"].keys()) freq = np.array( - [ - v["frequency"] - for v in self.parser_obj.config_datagram["transceivers"].values() - ] + [v["frequency"] for v in self.parser_obj.config_datagram["transceivers"].values()] ) # Channel-specific variables @@ -336,9 +319,7 @@ def set_beam(self) -> xr.Dataset: beam_params = defaultdict() for param in params: beam_params[param] = [ - self.parser_obj.config_datagram["transceivers"][ch_seq].get( - param, np.nan - ) + self.parser_obj.config_datagram["transceivers"][ch_seq].get(param, np.nan) for ch_seq in ch_ids ] @@ -574,9 +555,7 @@ def set_beam(self) -> xr.Dataset: "transmit_mode": ( ["ping_time"], self.parser_obj.ping_data_dict["transmit_mode"][ch], - { - "long_name": "0 = Active, 1 = Passive, 2 = Test, -1 = Unknown" - }, + {"long_name": "0 = Active, 1 = Passive, 2 = Test, -1 = Unknown"}, ), }, coords={ @@ -616,11 +595,7 @@ def set_beam(self) -> xr.Dataset: # Attach frequency dimension/coordinate ds_tmp = ds_tmp.expand_dims( - { - "frequency": [ - self.parser_obj.config_datagram["transceivers"][ch]["frequency"] - ] - } + {"frequency": [self.parser_obj.config_datagram["transceivers"][ch]["frequency"]]} ) ds_tmp["frequency"] = ds_tmp["frequency"].assign_attrs( units="Hz", diff --git a/echopype/convert/set_groups_ek80.py b/echopype/convert/set_groups_ek80.py index f09fc214f..73cb7bb74 100644 --- a/echopype/convert/set_groups_ek80.py +++ b/echopype/convert/set_groups_ek80.py @@ -121,10 +121,7 @@ def set_platform(self) -> xr.Dataset: water_level = self.parser_obj.environment["water_level_draft"] else: water_level = np.nan - print( - "WARNING: The water_level_draft was not in the file. " - "Value set to NaN." - ) + print("WARNING: The water_level_draft was not in the file. " "Value set to NaN.") location_time, msg_type, lat, lon = self._parse_NMEA() mru_time = self.parser_obj.mru.get("timestamp", None) @@ -241,9 +238,7 @@ def _assemble_ds_ping_invariant(self, params, data_type): ch_ids = self.parser_obj.ch_ids[data_type] freq = np.array( [ - self.parser_obj.config_datagram["configuration"][ch][ - "transducer_frequency" - ] + self.parser_obj.config_datagram["configuration"][ch]["transducer_frequency"] for ch in ch_ids ] ) @@ -388,9 +383,7 @@ def _assemble_ds_complex(self, ch): np.array(self.parser_obj.ping_data_dict["n_complex"][ch]) ) if num_transducer_sectors.size > 1: # this is not supposed to happen - raise ValueError( - "Transducer sector number changes in the middle of the file!" - ) + raise ValueError("Transducer sector number changes in the middle of the file!") else: num_transducer_sectors = num_transducer_sectors[0] data_shape = self.parser_obj.ping_data_dict["complex"][ch].shape @@ -647,9 +640,7 @@ def merge_save(ds_combine, ds_type, group_name): ds_data = ds_data.expand_dims( { "frequency": [ - self.parser_obj.config_datagram["configuration"][ch][ - "transducer_frequency" - ] + self.parser_obj.config_datagram["configuration"][ch]["transducer_frequency"] ] } ) @@ -669,9 +660,7 @@ def merge_save(ds_combine, ds_type, group_name): if len(ds_complex) > 0: ds_beam = merge_save(ds_complex, "complex", group_name="/Sonar/Beam_group1") if len(ds_power) > 0: - ds_beam_power = merge_save( - ds_power, "power", group_name="/Sonar/Beam_group2" - ) + ds_beam_power = merge_save(ds_power, "power", group_name="/Sonar/Beam_group2") else: ds_beam = merge_save(ds_power, "power", group_name="/Sonar/Beam_group1") diff --git a/echopype/convert/utils/ek_date_conversion.py b/echopype/convert/utils/ek_date_conversion.py index 462abc78e..78dbc5747 100644 --- a/echopype/convert/utils/ek_date_conversion.py +++ b/echopype/convert/utils/ek_date_conversion.py @@ -130,9 +130,7 @@ def unix_to_datetime(unix_timestamp): unix_datetime = pytz_utc.normalize(unix_timestamp.astimezone(pytz_utc)) elif isinstance(unix_timestamp, float): - unix_datetime = pytz_utc.localize( - datetime.datetime.fromtimestamp(unix_timestamp) - ) + unix_datetime = pytz_utc.localize(datetime.datetime.fromtimestamp(unix_timestamp)) else: errstr = "Looking for a timestamp of type datetime.datetime or # of sec past unix epoch.\n" diff --git a/echopype/convert/utils/ek_raw_io.py b/echopype/convert/utils/ek_raw_io.py index 904195d3e..dc5d32dd8 100644 --- a/echopype/convert/utils/ek_raw_io.py +++ b/echopype/convert/utils/ek_raw_io.py @@ -318,9 +318,7 @@ def _read_next_dgram(self): dgram_size_check = self._read_dgram_size() except DatagramReadError as e: self._seek_bytes(old_file_pos, SEEK_SET) - e.message = ( - "Short read while getting trailing raw file datagram size for check" - ) + e.message = "Short read while getting trailing raw file datagram size for check" raise e # make sure they match @@ -649,9 +647,7 @@ def seek(self, offset, whence): self._current_dgram_offset = 0 elif whence == SEEK_END: if offset > 0: - raise ValueError( - "Use negative offsets when seeking backward from end of file" - ) + raise ValueError("Use negative offsets when seeking backward from end of file") # Do we need to generate the total number of datagrams w/in the file? try: diff --git a/echopype/convert/utils/ek_raw_parsers.py b/echopype/convert/utils/ek_raw_parsers.py index 75def1de6..881ef8f19 100644 --- a/echopype/convert/utils/ek_raw_parsers.py +++ b/echopype/convert/utils/ek_raw_parsers.py @@ -69,9 +69,7 @@ def validate_data_header(self, data): raise ValueError("Expected data of type %s, not %s" % (self._id, type_)) if version not in self._versions: - raise ValueError( - "No parser available for type %s version %d" % (self._id, version) - ) + raise ValueError("No parser available for type %s version %d" % (self._id, version)) return type_, version @@ -99,9 +97,7 @@ def _pack_contents(self, data={}, version=0): def finalize_datagram(cls, datagram_content_str): datagram_size = len(datagram_content_str) final_fmt = "=l%dsl" % (datagram_size) - return struct.pack( - final_fmt, datagram_size, datagram_content_str, datagram_size - ) + return struct.pack(final_fmt, datagram_size, datagram_content_str, datagram_size) class SimradDepthParser(_SimradDatagramParser): @@ -193,9 +189,7 @@ def _pack_contents(self, data, version): if len(set(lengths)) != 1: min_indx = min(lengths) - log.warning( - "Data lengths mismatched: d:%d, r:%d, u:%d, t:%d", *lengths - ) + log.warning("Data lengths mismatched: d:%d, r:%d, u:%d, t:%d", *lengths) log.warning(" Using minimum value: %d", min_indx) data["transceiver_count"] = min_indx @@ -272,9 +266,7 @@ def _unpack_contents(self, raw_string, bytes_read, version): depth_size = struct.calcsize(depth_fmt) buf_indx = self.header_size(version) data["depth"] = np.fromiter( - struct.unpack( - depth_fmt, raw_string[buf_indx : buf_indx + depth_size] - ), # noqa + struct.unpack(depth_fmt, raw_string[buf_indx : buf_indx + depth_size]), # noqa "float", ) @@ -859,9 +851,7 @@ def dict_to_dict(xml_dict, data_dict, parse_opts): # check if there are >1 transducer under a single transceiver channel if len(list(tcvr_ch)) > 1: - ValueError( - "Found >1 transducer under a single transceiver channel!" - ) + ValueError("Found >1 transducer under a single transceiver channel!") else: # should only have 1 transducer tcvr_ch_xducer = tcvr_ch.find( "Transducer" @@ -873,43 +863,25 @@ def dict_to_dict(xml_dict, data_dict, parse_opts): "frequency": np.array( [int(f.attrib["Frequency"]) for f in f_par] ), - "gain": np.array( - [float(f.attrib["Gain"]) for f in f_par] - ), + "gain": np.array([float(f.attrib["Gain"]) for f in f_par]), "impedance": np.array( [int(f.attrib["Impedance"]) for f in f_par] ), - "phase": np.array( - [float(f.attrib["Phase"]) for f in f_par] - ), + "phase": np.array([float(f.attrib["Phase"]) for f in f_par]), "beamwidth_alongship": np.array( - [ - float(f.attrib["BeamWidthAlongship"]) - for f in f_par - ] + [float(f.attrib["BeamWidthAlongship"]) for f in f_par] ), "beamwidth_athwartship": np.array( - [ - float(f.attrib["BeamWidthAthwartship"]) - for f in f_par - ] + [float(f.attrib["BeamWidthAthwartship"]) for f in f_par] ), "angle_offset_alongship": np.array( - [ - float(f.attrib["AngleOffsetAlongship"]) - for f in f_par - ] + [float(f.attrib["AngleOffsetAlongship"]) for f in f_par] ), "angle_offset_athwartship": np.array( - [ - float(f.attrib["AngleOffsetAthwartship"]) - for f in f_par - ] + [float(f.attrib["AngleOffsetAthwartship"]) for f in f_par] ), } - data["configuration"][channel_id][ - "calibration" - ] = cal_par + data["configuration"][channel_id]["calibration"] = cal_par # add the transducer data to the config dict dict_to_dict( tcvr_ch_xducer.attrib, @@ -927,9 +899,7 @@ def dict_to_dict(xml_dict, data_dict, parse_opts): # built occurrence lookup table for transducer name xducer_name_list = [] for xducer_ch in xducer.iter("Transducer"): - xducer_name_list.append( - xducer_ch.attrib["TransducerName"] - ) + xducer_name_list.append(xducer_ch.attrib["TransducerName"]) # find matching transducer for this channel_id match_found = False @@ -948,8 +918,7 @@ def dict_to_dict(xml_dict, data_dict, parse_opts): == tcvr_ch_xducer.attrib["SerialNumber"] ) match_tcvr = ( - tcvr_ch_num - in xducer_ch.attrib["TransducerCustomName"] + tcvr_ch_num in xducer_ch.attrib["TransducerCustomName"] ) # if find match add the transducer mounting details @@ -963,9 +932,7 @@ def dict_to_dict(xml_dict, data_dict, parse_opts): # only check sn and transceiver unique number match_found = match_sn or match_tcvr else: - match_found = ( - match_name or match_sn or match_tcvr - ) + match_found = match_name or match_sn or match_tcvr # add transducer mounting details if match_found: @@ -989,9 +956,7 @@ def dict_to_dict(xml_dict, data_dict, parse_opts): for h in root.iter("Channel"): parm_xml = h.attrib # add the data to the environment dict - dict_to_dict( - parm_xml, data["parameter"], self.parameter_parsing_options - ) + dict_to_dict(parm_xml, data["parameter"], self.parameter_parsing_options) elif data["subtype"] == "environment": @@ -999,9 +964,7 @@ def dict_to_dict(xml_dict, data_dict, parse_opts): for h in root.iter("Environment"): env_xml = h.attrib # add the data to the environment dict - dict_to_dict( - env_xml, data["environment"], self.environment_parsing_options - ) + dict_to_dict(env_xml, data["environment"], self.environment_parsing_options) for h in root.iter("Transducer"): transducer_xml = h.attrib @@ -1422,9 +1385,7 @@ def _unpack_contents(self, raw_string, bytes_read, version): txcvr["spare2"] = txcvr["spare2"].strip("\x00") txcvr["spare3"] = txcvr["spare3"].strip("\x00") txcvr["spare4"] = txcvr["spare4"].strip("\x00") - txcvr["gpt_software_version"] = txcvr["gpt_software_version"].strip( - "\x00" - ) + txcvr["gpt_software_version"] = txcvr["gpt_software_version"].strip("\x00") buf_indx += txcvr_header_size @@ -1442,9 +1403,7 @@ def _pack_contents(self, data, version): if version == 0: if data["transceiver_count"] != len(data["transceivers"]): - log.warning( - "Mismatch between 'transceiver_count' and actual # of transceivers" - ) + log.warning("Mismatch between 'transceiver_count' and actual # of transceivers") data["transceiver_count"] = len(data["transceivers"]) sounder_name = data["sounder_name"] @@ -1495,9 +1454,7 @@ def _pack_contents(self, data, version): txcvr_contents.extend(txcvr["sa_correction_table"]) txcvr_contents.append(txcvr["spare3"]) - txcvr_contents.extend( - [txcvr["gpt_software_version"], txcvr["spare4"]] - ) + txcvr_contents.extend([txcvr["gpt_software_version"], txcvr["spare4"]]) txcvr_contents_str = struct.pack(txcvr_header_fmt, *txcvr_contents) @@ -1716,17 +1673,13 @@ def _pack_contents(self, data, version): if version == 0: if data["count"] > 0: - if (int(data["mode"]) & 0x1) and ( - len(data.get("power", [])) != data["count"] - ): + if (int(data["mode"]) & 0x1) and (len(data.get("power", [])) != data["count"]): log.warning( "Data 'count' = %d, but contains %d power samples. Ignoring power." ) data["mode"] &= ~(1 << 0) - if (int(data["mode"]) & 0x2) and ( - len(data.get("angle", [])) != data["count"] - ): + if (int(data["mode"]) & 0x2) and (len(data.get("angle", [])) != data["count"]): log.warning( "Data 'count' = %d, but contains %d angle samples. Ignoring angle." ) diff --git a/echopype/echodata/combine.py b/echopype/echodata/combine.py index 85fbc7317..5fd3b8121 100644 --- a/echopype/echodata/combine.py +++ b/echopype/echodata/combine.py @@ -115,15 +115,11 @@ def combine_echodata(echodatas: List[EchoData], combine_attrs="override") -> Ech sonar_model = None for echodata in echodatas: if echodata.sonar_model is None: - raise ValueError( - "all EchoData objects must have non-None sonar_model values" - ) + raise ValueError("all EchoData objects must have non-None sonar_model values") elif sonar_model is None: sonar_model = echodata.sonar_model elif echodata.sonar_model != sonar_model: - raise ValueError( - "all EchoData objects must have the same sonar_model value" - ) + raise ValueError("all EchoData objects must have the same sonar_model value") # ping time before reversal correction old_ping_time = None @@ -175,9 +171,7 @@ def combine_echodata(echodatas: List[EchoData], combine_attrs="override") -> Ech [concat_dim], data_vars=concat_data_vars, coords="minimal", - combine_attrs="drop" - if combine_attrs == "overwrite_conflicts" - else combine_attrs, + combine_attrs="drop" if combine_attrs == "overwrite_conflicts" else combine_attrs, ) if combine_attrs == "overwrite_conflicts": combined_group.attrs.update(union_attrs(group_datasets)) @@ -187,16 +181,12 @@ def combine_echodata(echodatas: List[EchoData], combine_attrs="override") -> Ech combined_group["transceiver_software_version"] = combined_group[ "transceiver_software_version" ].astype(" Ech " (see https://github.com/OSOceanAcoustics/echopype/pull/297)" ) old_location_time = combined_group["location_time"] - coerce_increasing_time( - combined_group, time_name="location_time" - ) + coerce_increasing_time(combined_group, time_name="location_time") new_location_time = combined_group["location_time"] else: combined_group["location_time"] = new_location_time if sonar_model == "EK80": - if "mru_time" in combined_group and exist_reversed_time( - combined_group, "mru_time" - ): + if "mru_time" in combined_group and exist_reversed_time(combined_group, "mru_time"): if old_mru_time is None: warnings.warn( f"{sonar_model} mru_time reversal detected; the mru times will be corrected" # noqa diff --git a/echopype/echodata/convention/conv.py b/echopype/echodata/convention/conv.py index 59e65b750..7283f3c45 100644 --- a/echopype/echodata/convention/conv.py +++ b/echopype/echodata/convention/conv.py @@ -21,9 +21,7 @@ def yaml_dict(self): if self._yaml_dict: # Data has already been read, return it directly return self._yaml_dict - with resources.open_text( - package=convention, resource=f"{self.version}.yml" - ) as fid: + with resources.open_text(package=convention, resource=f"{self.version}.yml") as fid: convention_yaml = yaml.load(fid, Loader=yaml.SafeLoader) self._yaml_dict = convention_yaml diff --git a/echopype/echodata/echodata.py b/echopype/echodata/echodata.py index d195c0f6e..a87950d85 100644 --- a/echopype/echodata/echodata.py +++ b/echopype/echodata/echodata.py @@ -54,9 +54,7 @@ def __init__( self.storage_options: Dict[str, str] = ( storage_options if storage_options is not None else {} ) - self.open_kwargs: Dict[str, str] = ( - open_kwargs if open_kwargs is not None else {} - ) + self.open_kwargs: Dict[str, str] = open_kwargs if open_kwargs is not None else {} self.source_file: Optional["PathHint"] = source_file self.xml_path: Optional["PathHint"] = xml_path self.sonar_model: Optional["SonarModelsHint"] = sonar_model @@ -235,19 +233,14 @@ def squeeze_non_scalar(n): if "sound_speed" in env_params: sound_speed = squeeze_non_scalar(env_params["sound_speed"]) - elif all( - [param in env_params for param in ("temperature", "salinity", "pressure")] - ): + elif all([param in env_params for param in ("temperature", "salinity", "pressure")]): sound_speed = calc_sound_speed( squeeze_non_scalar(env_params["temperature"]), squeeze_non_scalar(env_params["salinity"]), squeeze_non_scalar(env_params["pressure"]), formula_source="AZFP" if self.sonar_model == "AZFP" else "Mackenzie", ) - elif ( - self.sonar_model in ("EK60", "EK80") - and "sound_speed_indicative" in self.environment - ): + elif self.sonar_model in ("EK60", "EK80") and "sound_speed_indicative" in self.environment: sound_speed = squeeze_non_scalar(self.environment["sound_speed_indicative"]) else: raise ValueError( @@ -260,9 +253,7 @@ def squeeze_non_scalar(n): if self.sonar_model == "AZFP": cal_type = azfp_cal_type if cal_type is None: - raise ValueError( - "azfp_cal_type must be specified when sonar_model is AZFP" - ) + raise ValueError("azfp_cal_type must be specified when sonar_model is AZFP") # Notation below follows p.86 of user manual N = self.vendor["number_of_samples_per_average_bin"] # samples per bin @@ -297,9 +288,7 @@ def squeeze_non_scalar(n): # if it is not indexed by ping_time then echopype.preprocess.compute_MVBS will fail # because it expects the range included in the Sv/Sp dataset # to be indexed by ping_time - range_meter = range_meter.expand_dims( - {"ping_time": self.beam["ping_time"]}, axis=1 - ) + range_meter = range_meter.expand_dims({"ping_time": self.beam["ping_time"]}, axis=1) return range_meter @@ -317,9 +306,7 @@ def squeeze_non_scalar(n): # EK80 needs waveform_mode specification if self.sonar_model == "EK80" and waveform_mode is None: - raise ValueError( - "ek_waveform_mode must be specified when sonar_model is EK80" - ) + raise ValueError("ek_waveform_mode must be specified when sonar_model is EK80") # TVG correction factor changes depending when the echo recording starts # wrt when the transmit signal is sent out. @@ -346,15 +333,11 @@ def squeeze_non_scalar(n): elif waveform_mode == "BB": beam = self.beam # always use the Beam group # TODO: bug: right now only first ping_time has non-nan range - shift = beam[ - "transmit_duration_nominal" - ] # based on Lar Anderson's Matlab code + shift = beam["transmit_duration_nominal"] # based on Lar Anderson's Matlab code # TODO: once we allow putting in arbitrary sound_speed, # change below to use linearly-interpolated values range_meter = ( - (beam.range_sample * beam["sample_interval"] - shift) - * sound_speed - / 2 + (beam.range_sample * beam["sample_interval"] - shift) * sound_speed / 2 ) # TODO: Lar Anderson's code include a slicing by minRange with a default of 0.02 m, # need to ask why and see if necessary here @@ -362,18 +345,12 @@ def squeeze_non_scalar(n): raise ValueError("Input waveform_mode not recognized!") # make order of dims conform with the order of backscatter data - range_meter = range_meter.transpose( - "frequency", "ping_time", "range_sample" - ) - range_meter = range_meter.where( - range_meter > 0, 0 - ) # set negative ranges to 0 + range_meter = range_meter.transpose("frequency", "ping_time", "range_sample") + range_meter = range_meter.where(range_meter > 0, 0) # set negative ranges to 0 # set entries with NaN backscatter data to NaN if "quadrant" in beam["backscatter_r"].dims: - valid_idx = ( - ~beam["backscatter_r"].isel(quadrant=0).drop("quadrant").isnull() - ) + valid_idx = ~beam["backscatter_r"].isel(quadrant=0).drop("quadrant").isnull() else: valid_idx = ~beam["backscatter_r"].isnull() range_meter = range_meter.where(valid_idx) @@ -441,8 +418,7 @@ def update_platform( for coordvar in extra_platform_data.coords: if ( "cf_role" in extra_platform_data[coordvar].attrs - and extra_platform_data[coordvar].attrs["cf_role"] - == "trajectory_id" + and extra_platform_data[coordvar].attrs["cf_role"] == "trajectory_id" ): trajectory_var = coordvar @@ -469,9 +445,7 @@ def update_platform( ) # fmt: on max_index = min( - np.searchsorted( - sorted_external_time, self.beam["ping_time"].max(), side="right" - ), + np.searchsorted(sorted_external_time, self.beam["ping_time"].max(), side="right"), len(sorted_external_time) - 1, ) extra_platform_data = extra_platform_data.sel( @@ -487,21 +461,15 @@ def update_platform( platform_vars_attrs = {var: platform[var].attrs for var in platform.variables} platform = platform.drop_dims(["location_time"], errors="ignore") # drop_dims is also dropping latitude, longitude and sentence_type why? - platform = platform.assign_coords( - location_time=extra_platform_data[time_dim].values - ) - history_attr = ( - f"{datetime.datetime.utcnow()} +00:00. Added from external platform data" - ) + platform = platform.assign_coords(location_time=extra_platform_data[time_dim].values) + history_attr = f"{datetime.datetime.utcnow()} +00:00. Added from external platform data" if extra_platform_data_file_name: history_attr += ", from file " + extra_platform_data_file_name location_time_attrs = { **self._varattrs["platform_coord_default"]["location_time"], **{"history": history_attr}, } - platform["location_time"] = platform["location_time"].assign_attrs( - **location_time_attrs - ) + platform["location_time"] = platform["location_time"].assign_attrs(**location_time_attrs) dropped_vars_target = [ "pitch", diff --git a/echopype/metrics/summary_statistics.py b/echopype/metrics/summary_statistics.py index d345318d1..991e9d1dd 100644 --- a/echopype/metrics/summary_statistics.py +++ b/echopype/metrics/summary_statistics.py @@ -85,9 +85,7 @@ def center_of_mass(ds: xr.Dataset, range_label="echo_range") -> xr.DataArray: """ dz = delta_z(ds, range_label=range_label) sv = convert_to_linear(ds, "Sv") - return (ds[range_label] * sv * dz).sum(dim="range_sample") / (sv * dz).sum( - dim="range_sample" - ) + return (ds[range_label] * sv * dz).sum(dim="range_sample") / (sv * dz).sum(dim="range_sample") def dispersion(ds: xr.Dataset, range_label="echo_range") -> xr.DataArray: @@ -108,9 +106,9 @@ def dispersion(ds: xr.Dataset, range_label="echo_range") -> xr.DataArray: dz = delta_z(ds, range_label=range_label) sv = convert_to_linear(ds, "Sv") cm = center_of_mass(ds) - return ((ds[range_label] - cm) ** 2 * sv * dz).sum(dim="range_sample") / ( - sv * dz - ).sum(dim="range_sample") + return ((ds[range_label] - cm) ** 2 * sv * dz).sum(dim="range_sample") / (sv * dz).sum( + dim="range_sample" + ) def evenness(ds: xr.Dataset, range_label="echo_range") -> xr.DataArray: @@ -131,9 +129,7 @@ def evenness(ds: xr.Dataset, range_label="echo_range") -> xr.DataArray: """ dz = delta_z(ds, range_label=range_label) sv = convert_to_linear(ds, "Sv") - return ((sv * dz).sum(dim="range_sample")) ** 2 / (sv ** 2 * dz).sum( - dim="range_sample" - ) + return ((sv * dz).sum(dim="range_sample")) ** 2 / (sv ** 2 * dz).sum(dim="range_sample") def aggregation(ds: xr.Dataset, range_label="echo_range") -> xr.DataArray: diff --git a/echopype/preprocess/api.py b/echopype/preprocess/api.py index ae0c8d2c5..5063efba5 100644 --- a/echopype/preprocess/api.py +++ b/echopype/preprocess/api.py @@ -61,12 +61,8 @@ def _freq_MVBS(ds, rint, pbin): return 10 * np.log10(sv_groupby_bins) # Groupby freq in case of different echo_range (from different sampling intervals) - range_interval = np.arange( - 0, ds_Sv["echo_range"].max() + range_meter_bin, range_meter_bin - ) - MVBS = ds_Sv.groupby("frequency").apply( - _freq_MVBS, args=(range_interval, ping_time_bin) - ) + range_interval = np.arange(0, ds_Sv["echo_range"].max() + range_meter_bin, range_meter_bin) + MVBS = ds_Sv.groupby("frequency").apply(_freq_MVBS, args=(range_interval, ping_time_bin)) # Attach attributes MVBS.attrs = { @@ -153,9 +149,7 @@ def estimate_noise(ds_Sv, ping_num, range_sample_num, noise_max=None): ------- A DataArray containing noise estimated from the input ``ds_Sv`` """ - noise_obj = NoiseEst( - ds_Sv=ds_Sv.copy(), ping_num=ping_num, range_sample_num=range_sample_num - ) + noise_obj = NoiseEst(ds_Sv=ds_Sv.copy(), ping_num=ping_num, range_sample_num=range_sample_num) noise_obj.estimate_noise(noise_max=noise_max) return noise_obj.Sv_noise @@ -188,9 +182,7 @@ def remove_noise(ds_Sv, ping_num, range_sample_num, noise_max=None, SNR_threshol The input dataset with additional variables, including the corrected Sv (``Sv_corrected``) and the noise estimates (``Sv_noise``) """ - noise_obj = NoiseEst( - ds_Sv=ds_Sv.copy(), ping_num=ping_num, range_sample_num=range_sample_num - ) + noise_obj = NoiseEst(ds_Sv=ds_Sv.copy(), ping_num=ping_num, range_sample_num=range_sample_num) noise_obj.remove_noise(noise_max=noise_max, SNR_threshold=SNR_threshold) return noise_obj.ds_Sv diff --git a/echopype/qc/api.py b/echopype/qc/api.py index 1fa4680f6..ec79d0c76 100644 --- a/echopype/qc/api.py +++ b/echopype/qc/api.py @@ -17,9 +17,7 @@ def _clean_ping_time(ping_time_old, local_win_len=100): ( ping_time_old[: ni + 1], ping_time_old[ni] - + np.cumsum( - np.hstack((local_pt_median, ping_time_old_diff[(ni + 1) :])) - ), + + np.cumsum(np.hstack((local_pt_median, ping_time_old_diff[(ni + 1) :]))), ) ) return _clean_ping_time(ping_time_new, local_win_len=local_win_len) diff --git a/echopype/utils/io.py b/echopype/utils/io.py index 938e49057..cf999469b 100644 --- a/echopype/utils/io.py +++ b/echopype/utils/io.py @@ -171,9 +171,7 @@ def validate_output_path( if not out_dir.exists(): out_dir.mkdir(parents=True) - warnings.warn( - f"Resulting converted file(s) will be available at {str(out_dir)}" - ) + warnings.warn(f"Resulting converted file(s) will be available at {str(out_dir)}") out_path = str(out_dir / (Path(source_file).stem + file_ext)) elif not isinstance(save_path, Path) and not isinstance(save_path, str): raise TypeError("save_path must be a string or Path") @@ -201,13 +199,9 @@ def validate_output_path( if isinstance(sanitized_path, Path): check_file_permissions(sanitized_path.parent) final_path = sanitized_path - out_path = str( - final_path.parent.joinpath(final_path.stem + file_ext).absolute() - ) + out_path = str(final_path.parent.joinpath(final_path.stem + file_ext).absolute()) else: - path_dir = fsspec.get_mapper( - os.path.dirname(save_path), **output_storage_options - ) + path_dir = fsspec.get_mapper(os.path.dirname(save_path), **output_storage_options) check_file_permissions(path_dir) final_path = Path(save_path) out_path = save_path @@ -218,9 +212,7 @@ def validate_output_path( return out_path -def check_file_existence( - file_path: "PathHint", storage_options: Dict[str, str] = {} -) -> bool: +def check_file_existence(file_path: "PathHint", storage_options: Dict[str, str] = {}) -> bool: """ Checks if file exists in the specified path @@ -271,9 +263,7 @@ def check_file_permissions(FILE_DIR): FILE_DIR = Path(FILE_DIR) if not FILE_DIR.exists(): - warnings.warn( - f"{str(FILE_DIR)} does not exist. Attempting to create it." - ) + warnings.warn(f"{str(FILE_DIR)} does not exist. Attempting to create it.") FILE_DIR.mkdir(exist_ok=True, parents=True) TEST_FILE = FILE_DIR.joinpath(Path(".permission_test")) TEST_FILE.write_text("testing\n") diff --git a/echopype/utils/repr.py b/echopype/utils/repr.py index 804e266a2..a99fa2630 100644 --- a/echopype/utils/repr.py +++ b/echopype/utils/repr.py @@ -11,8 +11,7 @@ def _load_static_files(): Clone from xarray.core.formatted_html_template. """ return [ - pkg_resources.resource_string("echopype", fname).decode("utf8") - for fname in STATIC_FILES + pkg_resources.resource_string("echopype", fname).decode("utf8") for fname in STATIC_FILES ] diff --git a/echopype/utils/uwa.py b/echopype/utils/uwa.py index 3fe088a89..e8f0588b5 100644 --- a/echopype/utils/uwa.py +++ b/echopype/utils/uwa.py @@ -4,9 +4,7 @@ import numpy as np -def calc_sound_speed( - temperature=27, salinity=35, pressure=10, formula_source="Mackenzie" -): +def calc_sound_speed(temperature=27, salinity=35, pressure=10, formula_source="Mackenzie"): """ Calculate sound speed in [m/s]. @@ -45,10 +43,7 @@ def calc_sound_speed( + 2.374e-4 * temperature ** 3 ) ss += 1.340 * (salinity - 35) + 1.630e-2 * pressure + 1.675e-7 * pressure ** 2 - ss += ( - -1.025e-2 * temperature * (salinity - 35) - - 7.139e-13 * temperature * pressure ** 3 - ) + ss += -1.025e-2 * temperature * (salinity - 35) - 7.139e-13 * temperature * pressure ** 3 elif formula_source == "AZFP": z = temperature / 10 ss = ( @@ -121,11 +116,7 @@ def calc_absorption( f1 = 2.8 * np.sqrt(salinity / 35) * 10 ** (4 - 1245 / (temperature + 273)) A2 = 21.44 * salinity / c * (1 + 0.025 * temperature) P2 = 1.0 - 1.37e-4 * pressure + 6.2e-9 * pressure ** 2 - f2 = ( - 8.17 - * 10 ** (8 - 1990 / (temperature + 273)) - / (1 + 0.0018 * (salinity - 35)) - ) + f2 = 8.17 * 10 ** (8 - 1990 / (temperature + 273)) / (1 + 0.0018 * (salinity - 35)) P3 = 1.0 - 3.83e-5 * pressure + 4.9e-10 * pressure ** 2 if temperature < 20: A3 = ( @@ -153,12 +144,7 @@ def calc_absorption( D = pressure / 1000 f1 = 0.78 * np.sqrt(salinity / 35) * np.exp(temperature / 26) f2 = 42 * np.exp(temperature / 17) - a1 = ( - 0.106 - * (f1 * (freq ** 2)) - / ((f1 ** 2) + (freq ** 2)) - * np.exp((pH - 8) / 0.56) - ) + a1 = 0.106 * (f1 * (freq ** 2)) / ((f1 ** 2) + (freq ** 2)) * np.exp((pH - 8) / 0.56) a2 = ( 0.52 * (1 + temperature / 43) @@ -186,11 +172,7 @@ def calc_absorption( ) c = ( 4.86e-13 - * ( - 1 - + temperature - * (-0.042 + temperature * (8.53e-4 - temperature * 6.23e-6)) - ) + * (1 + temperature * (-0.042 + temperature * (8.53e-4 - temperature * 6.23e-6))) * (1 + k * (-3.84e-4 + k * 7.57e-8)) ) if salinity == 0: From 56b8adcab625f93613b21d98776b0692cda5d0e8 Mon Sep 17 00:00:00 2001 From: lsetiawan Date: Wed, 30 Mar 2022 11:18:07 -0700 Subject: [PATCH 8/8] Revert back to dynamically generate deps --- docs/source/contributing.rst | 4 +++- requirements.txt | 4 ---- setup.cfg | 12 ------------ setup.py | 6 +++++- 4 files changed, 8 insertions(+), 18 deletions(-) diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index 83c2d70fc..e5dc18abf 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -81,7 +81,9 @@ Create a `conda `_ environment for echopype development # ipykernel is recommended, in order to use with JupyterLab and IPython # to aid with development. We recommend you install JupyterLab separately conda install -c conda-forge ipykernel - pip install -e .[all] + # plot is an extra set of requirements that can be used for plotting. + # the command will install all the dependencies along with plotting dependencies. + pip install -e .[plot] See the :doc:`installation` page to simply install the latest echopype release from conda or PyPI. diff --git a/requirements.txt b/requirements.txt index 0bba1ef8a..b79f2c9fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,3 @@ -# This file is redundant with setup.cfg; -# it exists to let GitHub build the repository dependency graph -# https://help.github.com/en/github/visualizing-repository-data-with-graphs/listing-the-packages-that-a-repository-depends-on - dask[array] netCDF4 numpy diff --git a/setup.cfg b/setup.cfg index 5185bf6c0..136e1acb8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,21 +29,9 @@ platforms = any py_modules = _echopype_version include_package_data = True -install_requires = - dask[array] - netCDF4 - pynmea2 - pytz - scipy - xarray - zarr - fsspec - s3fs python_requires = >=3.8 [options.extras_require] plot = matplotlib cmocean -all = - %(plot)s diff --git a/setup.py b/setup.py index ff1b148d6..a64e4ada3 100644 --- a/setup.py +++ b/setup.py @@ -2,5 +2,9 @@ from setuptools import setup +# Dynamically read dependencies from requirements file +with open("requirements.txt") as f: + requirements = f.readlines() + if __name__ == "__main__": - setup() + setup(install_requires=requirements)