CHANGELOG.md

@@ -3,6 +3,13 @@
 All notable changes to this project will be documented in this file. See
 [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+## [0.4.0](https://git.gfz-potsdam.de/gnss/pynex/compare/v0.3.2...v0.4.0) (2023-03-31)
+
+
+### Features
+
+* added hatanaka decompression support ([65e48f5](https://git.gfz-potsdam.de/gnss/pynex/commit/65e48f5d88ed3799e384883a92049d147b128397))
+
 ## [0.3.2](https://git.gfz-potsdam.de/gnss/pynex/compare/v0.3.1...v0.3.2) (2023-03-30)
 
 

poetry.lock

@@ -556,28 +556,6 @@ python-dateutil = ">=2.8.1"
 [package.extras]
 dev = ["flake8", "markdown", "twine", "wheel"]
 
-[[package]]
-name = "hatanaka"
-version = "2.8.0"
-description = "Effortlessly compress / decompress any RINEX file"
-category = "main"
-optional = false
-python-versions = ">=3.6"
-files = [
-    {file = "hatanaka-2.8.0-py3-none-macosx_10_15_x86_64.whl", hash = "sha256:0e095d35ed4f607eb77ae47ecb310e4c25f5a6267037b703ea258ed03e5c47da"},
-    {file = "hatanaka-2.8.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:ce1628029c6b50c142a8fc5f15453c4cf2a3fd88a7128075415aeb5c9a2727d0"},
-    {file = "hatanaka-2.8.0-py3-none-win_amd64.whl", hash = "sha256:ccf8be554deee2fc70be52bd2f1d3d4dd370001caa74333bf041933d69a19023"},
-    {file = "hatanaka-2.8.0.tar.gz", hash = "sha256:84faa953b4f641a6d3cf8187f1775ba7e7f8d815f7bcd48cfb18553b766cbc41"},
-]
-
-[package.dependencies]
-importlib-resources = "*"
-ncompress = "*"
-
-[package.extras]
-dev = ["pytest"]
-tests = ["pytest"]
-
 [[package]]
 name = "identify"
 version = "2.5.21"
@@ -1098,55 +1076,6 @@ files = [
     {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"},
 ]
 
-[[package]]
-name = "ncompress"
-version = "1.0.0"
-description = "LZW compression and decompression"
-category = "main"
-optional = false
-python-versions = "*"
-files = [
-    {file = "ncompress-1.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0349d7de11edd70a7efea9ce9dc67f0e47b5774832dd063f7ae68a9e3e36ea31"},
-    {file = "ncompress-1.0.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:af0011bae90e44121f4e4edbff3dccdce7e4c5fc5e354db7eb48410d71f496df"},
-    {file = "ncompress-1.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e6f5bf381412e9d3847b76e8b6bd1f84dfadcd3d9c25903c8592facb437909a0"},
-    {file = "ncompress-1.0.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e0ebd71990ef7909b6627b5341a2fe1977dcce61dd3760a29e19e3f9e4c6a275"},
-    {file = "ncompress-1.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8b9acc46cf36bb998ed215d6e76a94e2bd1e827b9a4cb5362982b7004b5a7620"},
-    {file = "ncompress-1.0.0-cp310-cp310-win32.whl", hash = "sha256:2a104803fbe3ab0a96edb14927fa22c8142be838aabe7e938b4a52a4e82db56e"},
-    {file = "ncompress-1.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:5a2ae8a9170fa1f45df7efa292eb8c437b18c225b63d4adca4f50f9da0e8e0c7"},
-    {file = "ncompress-1.0.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:7608fbda43d04d9f476be2dbf4ef3c96e72d83b9557a48b07fbc9ff3ad29cdd2"},
-    {file = "ncompress-1.0.0-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8322482e72ac2802d1dca1007ec06aa281a4d5cf1cf9f8c75bb51e917382b756"},
-    {file = "ncompress-1.0.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3590e66313041721ae81e72ece06b7048c9293321bb30900358638673608e264"},
-    {file = "ncompress-1.0.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:736dbae078107742cf6ac7ccc11ae9c5eab77ef2c02aab3ef64802877bb01cab"},
-    {file = "ncompress-1.0.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:5336a8831a7e587829ce54e9e27d1fb2e04ddbc7d2d983693e35a3a03ac3ce79"},
-    {file = "ncompress-1.0.0-cp36-cp36m-win32.whl", hash = "sha256:9cd040ad73a3b0e917e01cdfba507e10e0bb56849daaac3ac3d86382d4d7ad82"},
-    {file = "ncompress-1.0.0-cp36-cp36m-win_amd64.whl", hash = "sha256:8eb4a55cbeaeb238a3b412952077be6b3f37b3416cd0211cc22776391ff2fef7"},
-    {file = "ncompress-1.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:916671d62167191af58d6b4a17b1c09c647e349dcff1fc0b7d764aa64c3773ee"},
-    {file = "ncompress-1.0.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:15f10fbfa11345ff0af090e3e6ae13a1fe2b52a2bb39d4f2373c2d6aeac75e5d"},
-    {file = "ncompress-1.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe0a671a2f7dc1ee0438d278ef30ab425a969536100c4352b5cb6bc0b6210818"},
-    {file = "ncompress-1.0.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:9d89acf209858e7940223cf35324e1b2effec119bb009a41f039e2ea4db22177"},
-    {file = "ncompress-1.0.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:66d991155a1655ccd98e8433c4a7e811d63eb649adb55f47d8f9528a30cc4b7a"},
-    {file = "ncompress-1.0.0-cp37-cp37m-win32.whl", hash = "sha256:34c6496168fd4dbc13f1fc0c0fcbadded1957639956f8cbc6894c39999817e36"},
-    {file = "ncompress-1.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:94b3f4e851f5b37e1d4cf2d8da911fa10783a59cba3d7f1f2ae5bd2842558077"},
-    {file = "ncompress-1.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:aaa18a509d9fc173b4b47d53c834e43ced1eda63d2aa7d4613dc59b2f802a31a"},
-    {file = "ncompress-1.0.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6540556d47670a8fb93878a44d0206bbdc87f32e4c5b57d6fe63691efafbb982"},
-    {file = "ncompress-1.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9da7c81313aed4b6c6e8020442ed8d03d04bff72947f9380ea1ce2c63ffb8ad1"},
-    {file = "ncompress-1.0.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d45ec59a8a3ce00613df0c81e5567854574dbbbf01ecd1a5a0929cd8fb04844d"},
-    {file = "ncompress-1.0.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:393cc3c126b9451fb32fe2bc07773264c90e73afbd37da0df472ac23bfd1a2d5"},
-    {file = "ncompress-1.0.0-cp38-cp38-win32.whl", hash = "sha256:78674f246878938387b6f82b10d1aa2192e02544d214541943d12ef1a45e66c6"},
-    {file = "ncompress-1.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:da216a53db7cd4c0247376f87367dd71df457443567e55310f6d3d23a9aff2f2"},
-    {file = "ncompress-1.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:34754041d9bac2d6908ae0d07ba541e4d6d606cca222ddd53f3a57e15f386b0a"},
-    {file = "ncompress-1.0.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ab9fc62baaa55faf8ed8ac67f2c64a7295fec91d7c1f306ac46aa894ca4edf91"},
-    {file = "ncompress-1.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:070044eab19586a45d1855c3e50e000ce86d6075b122a5ec8cffd480713dffac"},
-    {file = "ncompress-1.0.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f9ba6ab2aadd6fd90365fdad5219e4dc7bc2459b94f1e900a733dddaf4e9b2e6"},
-    {file = "ncompress-1.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b031e06b42037b181e3514261e1e85a9eae4af990c12b9348a9f22b8042201ff"},
-    {file = "ncompress-1.0.0-cp39-cp39-win32.whl", hash = "sha256:13fa26ec8000d786a8079bb265508b5df4b445a4f460481a13549b4bd3c83824"},
-    {file = "ncompress-1.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:d11df815d280985dfa660974df11dbe051a1a18dca2f91f9d30fbd6548237b8f"},
-    {file = "ncompress-1.0.0.tar.gz", hash = "sha256:e7bbf10cca1376f4f17ae2c447e33a9d4067525abb0c71d488c9a5ced50552f1"},
-]
-
-[package.extras]
-tests = ["pytest"]
-
 [[package]]
 name = "nodeenv"
 version = "1.7.0"
@@ -2053,4 +1982,4 @@ test = ["black", "flake8", "flake8-docstrings", "isort", "mypy", "pytest", "pyte
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.8.1,<3.12"
-content-hash = "78d34e0270c8883b088c1a6d6740f5c16f1ccfbe91635229560ef534b60254a5"
+content-hash = "1925e3168685e8a24f0b766102a139de70205aa0aa94dbf0fb00422fdc4bb592"

pynex/__init__.py

-"""Top-level package for pynex."""
-
-__author__ = """Markus Bradke"""
-__email__ = 'markus.bradke@gfz-potsdam.de'
-__version__ = '0.0.1'

pynex/pynex.py

@@ -144,10 +144,16 @@ class Pynex(Base):
 
         if '.parquet' in suffixes:
             data = pd.read_parquet(file)
-            data.attrs = co.get_attributes_from_filename(filename=file)
+            attrs = co.get_attributes_from_filename(filename=file)
+
+            if attrs:
+                data.attrs = attrs
         elif '.csv' in suffixes:
             data = pd.read_csv(file)
-            data.attrs = co.get_attributes_from_filename(filename=file)
+            attrs = co.get_attributes_from_filename(filename=file)
+
+            if attrs:
+                data.attrs = attrs
         else:  # try RINEX
             info = rinex_info(file)
 

pynex/rinex/hatanaka.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: 2023 Markus Bradke <markus.bradke@gfz-potsdam.de>
+# SPDX-FileCopyrightText: 2023 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences
+#
+# SPDX-License-Identifier: EUPL-1.2
+
+"""Hatanaka decompression module for pynex."""
+import io
+import os
+import platform
+import subprocess
+from subprocess import PIPE
+from typing import AnyStr, TextIO, Union
+
+
+def crx2rnx(crx_content: Union[AnyStr, TextIO], *, skip_strange_epochs: bool = False) -> AnyStr:
+    """Decompress RINEX observation file from a Hatanaka compressed RINEX file.
+
+    Args:
+        crx_content (Union[AnyStr, TextIO]): Compact RINEX observation file content.
+        skip_strange_epochs (bool, optional): _description_. Defaults to False.
+
+    Returns:
+        AnyStr: _description_
+    """
+    extra_args = []
+
+    if skip_strange_epochs:
+        extra_args = ['-s']
+
+    return _run('CRX2RNX', crx_content, extra_args)
+
+
+def _is_binary(f: TextIO) -> bool:
+    """Check if file is binary.
+
+    Args:
+        f (TextIO): File.
+
+    Returns:
+        bool: True.
+    """
+    return isinstance(f.read(0), bytes)
+
+
+def _run(program: str, content: Union[AnyStr, TextIO], extra_args: list = []):
+    """Run the program.
+
+    Args:
+        program (str): Program.
+        content (Union[AnyStr, TextIO]): _description_
+        extra_args (list, optional): Command line arguments. Defaults to [].
+
+    Returns:
+        _type_: _description_
+    """
+    encoding = None
+    errors = None
+
+    if isinstance(content, io.IOBase):
+        if not _is_binary(content):
+            encoding = 'ascii'
+            errors = 'ignore'
+
+        proc = _call(
+            program, ['-'] + extra_args, stdout=PIPE, stderr=PIPE, stdin=content, encoding=encoding, errors=errors
+        )
+        stdout, stderr = proc.communicate()
+    else:
+        if isinstance(content, str):
+            encoding = 'ascii'
+            errors = 'ignore'
+
+        proc = _call(
+            program, ['-'] + extra_args, stdout=PIPE, stderr=PIPE, stdin=PIPE, encoding=encoding, errors=errors
+        )
+        stdout, stderr = proc.communicate(content)
+
+    retcode = proc.poll()
+
+    _check(retcode, stderr)
+
+    return stdout
+
+
+def _check(retcode: int, stderr: str):
+    """Check return code.
+
+    Args:
+        retcode (int): Return code.
+        stderr (str): STDERR.
+    """
+    if isinstance(stderr, bytes):
+        stderr = stderr.decode('ascii', errors='backslashreplace').strip()
+
+    if retcode == 1:
+        raise Exception(stderr)
+
+
+def _call(program: str, args: AnyStr, **kwargs) -> subprocess.Popen:
+    """Call the program.
+
+    Args:
+        program (str): Program name.
+        args (AnyStr): Arguments.
+
+    Returns:
+        subprocess.Popen: Subprocess.
+    """
+    version = platform.architecture()[0]
+    executable = os.path.abspath('bin')
+    executable = executable + '/' + platform.system()  # add platform
+
+    if version == '32bit':
+        executable = executable + '_32bit'
+
+    executable = executable + '/' + program  # add program name
+
+    if platform.system() == 'Windows':
+        executable += '.exe'
+
+    if not os.path.isfile(executable):
+        raise FileNotFoundError(f"Program {executable} not found.")
+
+    if not os.access(executable, os.X_OK):
+        raise Exception(f"Program {executable} is not executable.")
+
+    return subprocess.Popen([str(executable)] + args, **kwargs)

pynex/rinex/rinex_info.py

@@ -17,7 +17,8 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Iterator, TextIO
 
-# from hatanaka import crx2rnx
+from pynex.rinex.hatanaka import crx2rnx
+
 # from ncompress import decompress as unlzw
 
 
@@ -37,7 +38,7 @@ def pynex_open(filename: TextIO | Path, header: bool = False) -> Iterator[TextIO
     """File handle for ASCII and compressed files.
 
     Args:
-        filename (TextIO | Path): _description_
+        filename (TextIO | Path): Filename.
         header (bool, optional): _description_. Defaults to False.
 
     Yields:
@@ -61,18 +62,18 @@ def pynex_open(filename: TextIO | Path, header: bool = False) -> Iterator[TextIO
                 _, is_crinex = rinex_version(first_nonblank_line(f))
                 f.seek(0)
 
-                # if is_crinex and not header:
-                #     # Conversion to string is necessary because of a quirk where gzip.open()
-                #     # even with 'rt' doesn't decompress until read.
-                #     f = io.StringIO(crx2rnx(f.read()))
+                if is_crinex and not header:
+                    # Conversion to string is necessary because of a quirk where gzip.open()
+                    # even with 'rt' doesn't decompress until read.
+                    f = io.StringIO(crx2rnx(f.read()))
                 yield f
         elif suffix == ".bz2":
             with bz2.open(filename, "rt") as f:
                 _, is_crinex = rinex_version(first_nonblank_line(f))
                 f.seek(0)
 
-                # if is_crinex and not header:
-                #     f = io.StringIO(crx2rnx(f.read()))
+                if is_crinex and not header:
+                    f = io.StringIO(crx2rnx(f.read()))
                 yield f
         elif suffix == ".zip":
             with zipfile.ZipFile(filename, "r") as z:
@@ -90,11 +91,11 @@ def pynex_open(filename: TextIO | Path, header: bool = False) -> Iterator[TextIO
                 _, is_crinex = rinex_version(first_nonblank_line(f))
                 f.seek(0)
 
-                # if is_crinex and not header:
-                #     f = io.StringIO(crx2rnx(f))
+                if is_crinex and not header:
+                    f = io.StringIO(crx2rnx(f))
                 yield f
     else:
-        raise OSError(f"Unsure what to do with input of type: {type(filename)}")
+        raise OSError(f"Input of type {type(filename)} is not supported.")
 
 
 def first_nonblank_line(file: TextIO, max_lines: int = 10) -> str:
@@ -111,7 +112,7 @@ def first_nonblank_line(file: TextIO, max_lines: int = 10) -> str:
     _i = None
 
     if max_lines < 1:
-        raise ValueError("Must read at least one line")
+        raise ValueError("Must read at least one line.")
 
     for _i in range(max_lines):
         line = file.readline(81)
@@ -129,7 +130,7 @@ def rinex_info(file: TextIO | Path) -> RinexInfo:
     """Verify RINEX version.
 
     Args:
-        f (TextIO | Path): _description_
+        file (TextIO | Path): File.
 
     Returns:
         RinexInfo: _description_

pynex/utils/common.py

@@ -428,11 +428,11 @@ def get_attributes_from_filename(filename: str) -> dict:
         filename (str): Filename.
 
     Returns:
-        dict: _description_
+        dict: Station information.
     """
     filename = os.path.basename(filename)
 
-    m = re.search('^(\\S{9})_(R|S)_(\\d{11})_(\\d{2}\\w)_(\\d{2}\\w)*_(\\w)(\\w)', filename)
+    m = re.search('^(\\S{9})_(R|S)_(\\d{11})_(\\d{2}\\w)_(\\d{2}\\w)*_*(\\w)(\\w)', filename)
 
     if m:
         file_type = m.group(7)
@@ -443,6 +443,8 @@ def get_attributes_from_filename(filename: str) -> dict:
 
         return {'station': m.group(1), 'satellites': m.group(6), 'file_type': file_type, 'rinex_type': rinex_type}
 
+    return None
+
 
 def get_software_name_and_version() -> str:
     """Get software name and version.

pyproject.toml

 [tool]
 [tool.poetry]
 name = "pynex"
-version = "0.3.2"
+version = "0.4.0"
 homepage = "https://git.gfz-potsdam.de/gnss/pynex"
 description = "RINEX to Pandas DataFrame conversion."
 authors = ["Markus Bradke <markus.bradke@gfz-potsdam.de>"]
@@ -9,7 +9,7 @@ keywords = ["RINEX"]
 readme = "README.md"
 license = "EUPL-1.2"
 classifiers=[
-    'Development Status :: 2 - Pre-Alpha',
+    'Development Status :: 3 - Alpha',
     'Intended Audience :: Developers',
     'License :: OSI Approved :: EUPL Software License',
     'Natural Language :: English',
@@ -27,7 +27,7 @@ packages = [
 python = ">=3.8.1,<3.12"
 click = "8.0.1"
 
-virtualenv  = { version = "^20.2.2", optional = true}
+virtualenv = { version = "^20.2.2", optional = true}
 black = "^23.1.0"
 isort = "^5.12.0"
 flake8 = "^6.0.0"
@@ -48,7 +48,6 @@ toml = "^0.10.2"
 bump2version = "^1.0.1"
 pre-commit = "^3.2.0"
 loguru = "^0.6.0"
-hatanaka = "^2.8.0"
 pandas = "^1.5.3"
 python-dotenv = "^1.0.0"