Revised dependencies. Updated minimal version of arosics and geoarray.

Signed-off-by: Daniel Scheffler <danschef@gfz-potsdam.de>

.gitlab-ci.yml

@@ -22,8 +22,11 @@ test_gms_preprocessing:
     - export GMS_index_host=${GMS_INDEX_HOST}
     - export GMS_index_port=${GMS_INDEX_PORT}
 
+    - conda install -c conda-forge cartopy  # TODO remove as soon as docker container is rebuilt
+    - pip install "geoarray>=0.9.0"  # TODO remove as soon as docker container is rebuilt
+    - pip install "arosics>=1.0.0"  # TODO remove as soon as docker container is rebuilt
+
     # update sicor
-    # - conda install -y -q -c conda-forge basemap
     # - rm -rf context/sicor
     # - git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git ./context/sicor
     # - cd ./context/sicor
@@ -34,8 +37,7 @@ test_gms_preprocessing:
     - pip install -U sicor
     # - python -c 'from sicor.tables import get_tables; get_tables(sensor="s2"); get_tables(sensor="l8"); get_tables(sensor="enmap", optional_downloads=("ch4",))'
 
-    - pip install 'scipy==1.1.0'  # TODO remove as soon as pinning is included in SICOR requirements or SICOR issue #63 is fixed
-    - pip install "geoarray>=0.8.33"  # TODO remove as soon as docker container is rebuilt
+    - pip check
 
     # run tests
     - make nosetests
@@ -65,7 +67,7 @@ test_gms_preprocessing_install:
     # - conda config --set channel_priority strict  # otherwise gdal or libgdal may be installed from defaults channel
 
     # install some dependencies that cause trouble when installed via pip
-    - conda install -y -c conda-forge numpy gdal scikit-image pyproj lxml geopandas ipython matplotlib basemap scikit-learn=0.23.2 shapely pyhdf python-fmask holoviews
+    - conda install -y -c conda-forge numpy gdal scikit-image pyproj geopandas ipython matplotlib cartopy scikit-learn=0.23.2 shapely pyhdf python-fmask holoviews
 
     # install not pip-installable deps of arosics
     - conda install -y -c conda-forge pyfftw pykrige
@@ -76,6 +78,7 @@ test_gms_preprocessing_install:
 
     # install gms_preprocessing
     - pip install -e .
+    - pip check
     - cd ..
     - pwd
     - ls

README.rst

@@ -137,7 +137,7 @@ Using conda_, the recommended approach is:
   source activate gms_preprocessing
 
   # install some dependencies that cause trouble when installed via pip
-  conda install -c conda-forge numpy gdal scikit-image pyproj lxml geopandas ipython matplotlib basemap scikit-learn=0.23.2 shapely pyhdf python-fmask holoviews
+  conda install -c conda-forge numpy gdal scikit-image pyproj geopandas ipython matplotlib cartopy scikit-learn=0.23.2 shapely pyhdf python-fmask holoviews
 
   # install not pip-installable deps of arosics
   conda install -c conda-forge pyfftw pykrige

requirements.txt

-py_tools_ds>=0.12.4
-geoarray>=0.8.33
-arosics>=0.8.5
-sicor
-spechomo
-pyrsr>=0.3.1
-matplotlib
-numpy
-scikit-learn
-scipy
-gdal
-pyproj
-shapely
-ephem
-pyorbital
+arosics>=1.0.0
+cerberus
 dill
-pytz
-pandas
-spectral
-# pyhdf # not pip installable
+ephem
+# fmask # not pip installable
+gdal
+geoarray>=0.9.0
+geoalchemy2
 geopandas
 iso8601
-pyinstrument
-geoalchemy2
-sqlalchemy
-psycopg2
-# fmask # not pip installable
-six
-tqdm
 jsmin
-cerberus
+matplotlib
+natsort
 nested_dict
-openpyxl
-timeout_decorator
+numpy
+pandas
+psycopg2
+# pyhdf # not pip installable
+pyinstrument
+pyorbital
+pyproj
+pyrsr>=0.3.1
+py_tools_ds>=0.12.4
+psutil
+pytz
 redis
-retools
 redis-semaphore
-psutil
-natsort
+retools
+scikit-learn
+scipy
+shapely
+spectral>=0.16
+spechomo
+sqlalchemy
+timeout_decorator
+tqdm

requirements_pip.txt

@@ -9,12 +9,10 @@ iso8601
 pyinstrument
 geoalchemy2
 sqlalchemy
-six
 tqdm
 jsmin
 cerberus
 nested_dict
-openpyxl
 timeout_decorator
 redis
 retools

setup.py

@@ -40,15 +40,43 @@ with open("gms_preprocessing/version.py") as version_file:
     exec(version_file.read(), version)
 
 requirements = [
-    'matplotlib', 'numpy', 'scikit-learn', 'scipy', 'gdal', 'pyproj', 'shapely', 'ephem', 'pyorbital', 'dill', 'pytz',
-    'pandas', 'spectral>=0.16', 'geopandas', 'iso8601', 'pyinstrument', 'geoalchemy2', 'sqlalchemy', 'natsort',
-    'psycopg2', 'py_tools_ds>=0.12.4', 'geoarray>=0.8.33', 'arosics>=0.8.7', 'six', 'tqdm', 'jsmin', 'cerberus',
-    'nested_dict', 'openpyxl', 'timeout_decorator', 'redis', 'retools', 'redis-semaphore', 'psutil', 'pyrsr>=0.3.1',
-    'spechomo', 'sicor'
-    # spectral<0.16  has some problems with writing signed integer 8bit data
+    'arosics>=1.0.0',
+    'cerberus',
+    'dill',
+    'ephem',
+    'gdal',
+    'geoalchemy2',
+    'geoarray>=0.9.0',
+    'geopandas',
+    'iso8601',
+    'jsmin',
+    'matplotlib',
+    'natsort',
+    'nested_dict',
+    'numpy',
+    'pandas',
+    'psutil',
+    'psycopg2',
+    'pyinstrument',
+    'pyorbital',
+    'pyproj',
+    'pyrsr>=0.3.1',
+    'py_tools_ds>=0.12.4',
+    'pytz',
+    'redis',
+    'redis-semaphore',
+    'retools',
+    'scikit-learn',
+    'scipy',
+    'shapely',
+    'sicor',
+    'spechomo',
+    'spectral>=0.16',  # spectral<0.16  has some problems with writing signed integer 8bit data
+    'sqlalchemy',
+    'timeout_decorator',
+    'tqdm',
     # fmask # conda install -c conda-forge python-fmask
     # 'pyhdf', # conda install --yes -c conda-forge pyhdf
-    # 'sicor', # pip install git+https://gitext.gfz-potsdam.de/EnMAP/sicor.git
 ]
 setup_requirements = []  # TODO(danschef): put setup requirements (distutils extensions, etc.) here
 test_requirements = requirements + ['coverage', 'nose', 'nose2', 'nose-htmloutput', 'rednose']

tests/CI_docker/context/environment_gms_preprocessing.yml

@@ -6,78 +6,74 @@ channels:
 dependencies:
   - python>=3.7.*
   - pip
-  - numpy
+  - cartopy
   - gdal
-  - scikit-image
-  - pyproj
-  - lxml
   - geopandas
+  - holoviews
   - ipython
+  - pyhdf
+  - python-fmask
   - matplotlib
-  - basemap
+  - numpy
+  - pyproj
+  - scikit-image
   - scikit-learn=0.23.2  # pinned due to spechomo classifiers created with this version
   - shapely
-  - pyhdf
-  - python-fmask
-  - holoviews
 
   # arosics
   - pyfftw
   - pykrige
 
   # sicor
-  - glymur
-  - pygrib
   - cachetools
-  - pyhdf
+  - glymur
   - h5py
-  - pytables
-  - numba
   - llvmlite
+  - numba
+  - pygrib
+  - pytables
 
   - pip:
-    - dicttoxml
-    - jsmin
+    - arosics>=1.0.0
     - cerberus
-    - pyprind
-    - pint
+    - dill
+    - ecmwf-api-client
+    - ephem
+    - geoalchemy2
+    - geoarray>=0.9.0
+    - jsmin
     - iso8601
-    - tqdm
-    - mpld3
-    - sphinx-argparse
-    - six
+    - natsort
+    - nested_dict
+    - pandas
+    - psutil
+    - psycopg2
+    - pyinstrument
+    - pyorbital
+    - pyrsr>=0.3.1
+    - py_tools_ds>=0.12.4
+    - pytz
+    - redis
+    - retools
+    - redis-semaphore
+    - scipy
+    - sicor
+    - spechomo>=0.16
     - spectral
+    - sqlalchemy
+    - timeout_decorator
+    - tqdm
+
+    # test and docs requirements
+    - coverage
     - flake8
-    - pylint
     - pydocstyle
+    - pylint
     - nose
     - nose2
     - nose-htmloutput
-    - coverage
     - rednose
-    - scipy
-    - ephem
-    - pyorbital
-    - dill
-    - pytz
-    - pyinstrument
-    - geoalchemy2
+    - sphinx-argparse
     - sqlalchemy
     - psycopg2
-    - pandas
-    - jsmin
-    - cerberus
-    - nested_dict
-    - openpyxl
-    - timeout_decorator
-    - redis
-    - retools
-    - redis-semaphore
-    - psutil
-    - ecmwf-api-client
-    - natsort
-    - py_tools_ds>=0.12.4
-    - geoarray>=0.8.33
-    - arosics>=0.8.7
-    - pyrsr>=0.3.1
-    - spechomo
+