Merge branch 'enhancement/revise_docker_env' into 'master'

Enhancement/revise docker env

See merge request !11

.gitlab-ci.yml

@@ -57,7 +57,7 @@ test_gms_preprocessing_install:
     - source /root/miniconda3/bin/activate
     - conda update conda
     - conda create -y -q -c conda-forge --name gms_test python=3
-    - source activate gms_test
+    - conda activate gms_test
 
     # update conda and python
     - conda update -n base -c conda-forge conda
@@ -74,7 +74,6 @@ test_gms_preprocessing_install:
 
     # install not pip-installable deps of sicor
     - conda install -y -q -c conda-forge glymur pygrib cachetools pyhdf h5py pytables llvmlite numba
-    - pip install ecmwf-api-client  # FIXME remove as long as this is no more needed for the hyperspectral part of SICOR
 
     # install gms_preprocessing
     - pip install -e .

HISTORY.rst

@@ -2,6 +2,16 @@
 History
 =======
 
+0.18.7 (2020-10-13)
+-------------------
+
+Added missing requirement 'openpyxl'.
+Replaced deprecated 'source activate' by 'conda activate'.
+Removed ecmwf-api-client installation as this is now included in SICOR.
+Pinned scipy to 1.1.0 in environment_gms_preprocessing.yml to make CI work for now.
+Revised docker container setup files.
+
+
 0.18.6 (2020-10-12)
 -------------------
 

README.rst

@@ -134,7 +134,7 @@ Using conda_, the recommended approach is:
 
   # create virtual environment for gms_preprocessing, this is optional
   conda create -c conda-forge --name gms_preprocessing python=3
-  source activate gms_preprocessing
+  conda activate gms_preprocessing
 
   # install some dependencies that cause trouble when installed via pip
   conda install -c conda-forge numpy gdal scikit-image pyproj geopandas ipython matplotlib cartopy scikit-learn=0.23.2 shapely pyhdf python-fmask holoviews
@@ -144,7 +144,6 @@ Using conda_, the recommended approach is:
 
   # install not pip-installable deps of sicor
   conda install -c conda-forge glymur pygrib cachetools pyhdf h5py pytables llvmlite numba
-  pip install ecmwf-api-client
 
   # install gms_preprocessing
   git clone https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing.git

gms_preprocessing/version.py

@@ -24,5 +24,5 @@
 # You should have received a copy of the GNU Lesser General Public License along
 # with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-__version__ = '0.18.6'
-__versionalias__ = '20201012.01'
+__version__ = '0.18.7'
+__versionalias__ = '20201013.01'

requirements.txt

@@ -13,6 +13,7 @@ matplotlib
 natsort
 nested_dict
 numpy
+openpyxl  # implicitly needed by pandas to write GMS job summaries as excel files
 pandas
 psycopg2
 # pyhdf # not pip installable

requirements_pip.txt

-scipy
-ephem
-pyorbital
+cerberus
 dill
-pytz
-spectral
+ephem
+geoalchemy2
 geopandas
 iso8601
-pyinstrument
-geoalchemy2
-sqlalchemy
-tqdm
 jsmin
-cerberus
+natsort
 nested_dict
-timeout_decorator
-redis
-retools
-redis-semaphore
+openpyxl
 psutil
+pyinstrument
+pyorbital
 pyrsr>=0.3.1
-natsort
+pytz
+redis
+redis-semaphore
+retools
+scipy
+spectral
+sqlalchemy
+timeout_decorator
+tqdm

setup.py

@@ -54,6 +54,7 @@ requirements = [
     'natsort',
     'nested_dict',
     'numpy',
+    'openpyxl',  # implicitly needed by pandas to write GMS job summaries as excel files
     'pandas',
     'psutil',
     'psycopg2',

tests/CI_docker/build_testsuite_image.sh

@@ -2,13 +2,9 @@
 
 context_dir="./context"
 dockerfile="gms_preprocessing_ci.docker"
-tag="gms_preprocessing_ci:0.18.3"
+tag="gms_preprocessing_ci:0.18.5"
 gitlab_runner="gms_preprocessing_gitlab_CI_runner"
 
-# get sicor project
-rm -rf context/sicor
-git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git ./context/sicor
-
 # download sicor cache (fastens SICOR CI tests a lot, but cache needs to be updated manually using a local sicor repo:
 # 1. clone a fresh copy of sicor or delete sicor/sicor/aerosol_0_ch4_34d3778719cc87188787de09bb8f870d16050078.pkl.zip
 # 2. run a sicor test including sicor_ac or enmap_ac (recreates cache file) -> upload newly created cache file

tests/CI_docker/context/environment_gms_preprocessing.yml

@@ -6,59 +6,45 @@ channels:
 dependencies:
   - python>=3.7.*
   - pip
-  - cartopy
+
+  - arosics>=1.0.0
   - gdal
+  - geoarray>=0.9.0
   - geopandas
-  - holoviews
   - ipython
   - pyhdf
   - python-fmask
   - matplotlib
   - numpy
   - pyproj
+  - pyrsr>=0.3.1
+  - py-tools-ds>=0.12.4
   - scikit-image
   - scikit-learn=0.23.2  # pinned due to spechomo classifiers created with this version
   - shapely
-
-  # arosics
-  - pyfftw
-  - pykrige
-
-  # sicor
-  - cachetools
-  - glymur
-  - h5py
-  - llvmlite
-  - numba
-  - pygrib
-  - pytables
+  - scipy<=1.1.0  # pinned due to https://gitext.gfz-potsdam.de/EnMAP/sicor/-/issues/63
+  - sicor
+  - spechomo
 
   - pip:
-    - arosics>=1.0.0
     - cerberus
     - dill
-    - ecmwf-api-client
     - ephem
     - geoalchemy2
-    - geoarray>=0.9.0
     - jsmin
     - iso8601
     - natsort
     - nested_dict
+    - openpyxl
     - pandas
     - psutil
     - psycopg2
     - pyinstrument
     - pyorbital
-    - pyrsr>=0.3.1
-    - py_tools_ds>=0.12.4
     - pytz
     - redis
     - retools
     - redis-semaphore
-    - scipy
-    - sicor
-    - spechomo>=0.16
     - spectral
     - sqlalchemy
     - timeout_decorator
@@ -74,6 +60,3 @@ dependencies:
     - nose-htmloutput
     - rednose
     - sphinx-argparse
-    - sqlalchemy
-    - psycopg2
-

tests/CI_docker/context/gms_preprocessing_ci.docker

@@ -2,6 +2,7 @@ FROM ci_base_centos:0.1
 
 # copy some needed stuff to /root
 COPY *.yml /root/
+COPY *.sh /root/
 
 # update the ci_env environment (that already contains all packages installed via 'docker_pyenvs' repo)
 RUN /bin/bash -i -c "\
@@ -10,14 +11,8 @@ RUN /bin/bash -i -c "\
     source activate ci_env; \
     conda env update -n ci_env -f /root/environment_gms_preprocessing.yml"
 
-# copy sicor code to /tmp
-COPY sicor /tmp/sicor
-
-RUN bash -i -c "\
-    source /root/miniconda3/bin/activate ci_env; \
-    cd /tmp/sicor/ ; \
-    make install"
-
-# copy sicor cache files to sicor root directory (speeds up SICOR CI tests because table subsets dont have to be created each time)
-# -> sicor root directory is the default directory of these cache files if sicor_cache_dir is not set in EnPT options
-COPY *.zip /tmp/sicor/sicor/
+# download SICOR atmospheric lookup tables and copy already created cache files to SICOR root dir to speed up SICOR in CI jobs
+RUN mkdir /tmp/sicor_cache
+COPY *.zip /tmp/sicor_cache/
+RUN /bin/bash -i -c "\
+    bash /root/setup_sicor_tables_and_cachefiles.sh"

tests/CI_docker/context/setup_sicor_tables_and_cachefiles.sh

+#!/usr/bin/env bash
+
+conda /root/miniconda3/bin/activate ci_env
+
+# copy sicor cache files to sicor root directory (speeds up SICOR CI tests because table subsets dont have to be created each time)
+# -> sicor root directory is the default directory of these cache files if sicor_cache_dir is not set in EnPT options
+SICOR_PATH=$(python -c "from sicor import __path__; print(__path__[0])")
+echo Copying SICOR cache files to ${SICOR_PATH}/...
+cp /tmp/sicor_cache/*.zip ${SICOR_PATH}/
+
+# download atmospheric lookup tables to save time when running CI jobs later
+echo Downloading SICOR atmospheric lookup tables...
+python -c "from sicor.tables import get_tables; get_tables(sensor=\"s2\"); get_tables(sensor=\"l8\")"