Commit c54c93b5 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Merge branch 'enhancement/revise_docker_env' into 'master'

Enhancement/revise docker env

See merge request !11
parents 6ecc002b d7f8936d
Pipeline #14518 failed with stage
in 12 minutes and 21 seconds
...@@ -57,7 +57,7 @@ test_gms_preprocessing_install: ...@@ -57,7 +57,7 @@ test_gms_preprocessing_install:
- source /root/miniconda3/bin/activate - source /root/miniconda3/bin/activate
- conda update conda - conda update conda
- conda create -y -q -c conda-forge --name gms_test python=3 - conda create -y -q -c conda-forge --name gms_test python=3
- source activate gms_test - conda activate gms_test
# update conda and python # update conda and python
- conda update -n base -c conda-forge conda - conda update -n base -c conda-forge conda
...@@ -74,7 +74,6 @@ test_gms_preprocessing_install: ...@@ -74,7 +74,6 @@ test_gms_preprocessing_install:
# install not pip-installable deps of sicor # install not pip-installable deps of sicor
- conda install -y -q -c conda-forge glymur pygrib cachetools pyhdf h5py pytables llvmlite numba - conda install -y -q -c conda-forge glymur pygrib cachetools pyhdf h5py pytables llvmlite numba
- pip install ecmwf-api-client # FIXME remove as long as this is no more needed for the hyperspectral part of SICOR
# install gms_preprocessing # install gms_preprocessing
- pip install -e . - pip install -e .
......
...@@ -2,6 +2,16 @@ ...@@ -2,6 +2,16 @@
History History
======= =======
0.18.7 (2020-10-13)
-------------------
Added missing requirement 'openpyxl'.
Replaced deprecated 'source activate' by 'conda activate'.
Removed ecmwf-api-client installation as this is now included in SICOR.
Pinned scipy to 1.1.0 in environment_gms_preprocessing.yml to make CI work for now.
Revised docker container setup files.
0.18.6 (2020-10-12) 0.18.6 (2020-10-12)
------------------- -------------------
......
...@@ -134,7 +134,7 @@ Using conda_, the recommended approach is: ...@@ -134,7 +134,7 @@ Using conda_, the recommended approach is:
# create virtual environment for gms_preprocessing, this is optional # create virtual environment for gms_preprocessing, this is optional
conda create -c conda-forge --name gms_preprocessing python=3 conda create -c conda-forge --name gms_preprocessing python=3
source activate gms_preprocessing conda activate gms_preprocessing
# install some dependencies that cause trouble when installed via pip # install some dependencies that cause trouble when installed via pip
conda install -c conda-forge numpy gdal scikit-image pyproj geopandas ipython matplotlib cartopy scikit-learn=0.23.2 shapely pyhdf python-fmask holoviews conda install -c conda-forge numpy gdal scikit-image pyproj geopandas ipython matplotlib cartopy scikit-learn=0.23.2 shapely pyhdf python-fmask holoviews
...@@ -144,7 +144,6 @@ Using conda_, the recommended approach is: ...@@ -144,7 +144,6 @@ Using conda_, the recommended approach is:
# install not pip-installable deps of sicor # install not pip-installable deps of sicor
conda install -c conda-forge glymur pygrib cachetools pyhdf h5py pytables llvmlite numba conda install -c conda-forge glymur pygrib cachetools pyhdf h5py pytables llvmlite numba
pip install ecmwf-api-client
# install gms_preprocessing # install gms_preprocessing
git clone https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing.git git clone https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing.git
......
...@@ -24,5 +24,5 @@ ...@@ -24,5 +24,5 @@
# You should have received a copy of the GNU Lesser General Public License along # You should have received a copy of the GNU Lesser General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>. # with this program. If not, see <http://www.gnu.org/licenses/>.
__version__ = '0.18.6' __version__ = '0.18.7'
__versionalias__ = '20201012.01' __versionalias__ = '20201013.01'
...@@ -13,6 +13,7 @@ matplotlib ...@@ -13,6 +13,7 @@ matplotlib
natsort natsort
nested_dict nested_dict
numpy numpy
openpyxl # implicitly needed by pandas to write GMS job summaries as excel files
pandas pandas
psycopg2 psycopg2
# pyhdf # not pip installable # pyhdf # not pip installable
......
scipy cerberus
ephem
pyorbital
dill dill
pytz ephem
spectral geoalchemy2
geopandas geopandas
iso8601 iso8601
pyinstrument
geoalchemy2
sqlalchemy
tqdm
jsmin jsmin
cerberus natsort
nested_dict nested_dict
timeout_decorator openpyxl
redis
retools
redis-semaphore
psutil psutil
pyinstrument
pyorbital
pyrsr>=0.3.1 pyrsr>=0.3.1
natsort pytz
redis
redis-semaphore
retools
scipy
spectral
sqlalchemy
timeout_decorator
tqdm
...@@ -54,6 +54,7 @@ requirements = [ ...@@ -54,6 +54,7 @@ requirements = [
'natsort', 'natsort',
'nested_dict', 'nested_dict',
'numpy', 'numpy',
'openpyxl', # implicitly needed by pandas to write GMS job summaries as excel files
'pandas', 'pandas',
'psutil', 'psutil',
'psycopg2', 'psycopg2',
......
...@@ -2,13 +2,9 @@ ...@@ -2,13 +2,9 @@
context_dir="./context" context_dir="./context"
dockerfile="gms_preprocessing_ci.docker" dockerfile="gms_preprocessing_ci.docker"
tag="gms_preprocessing_ci:0.18.3" tag="gms_preprocessing_ci:0.18.5"
gitlab_runner="gms_preprocessing_gitlab_CI_runner" gitlab_runner="gms_preprocessing_gitlab_CI_runner"
# get sicor project
rm -rf context/sicor
git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git ./context/sicor
# download sicor cache (fastens SICOR CI tests a lot, but cache needs to be updated manually using a local sicor repo: # download sicor cache (fastens SICOR CI tests a lot, but cache needs to be updated manually using a local sicor repo:
# 1. clone a fresh copy of sicor or delete sicor/sicor/aerosol_0_ch4_34d3778719cc87188787de09bb8f870d16050078.pkl.zip # 1. clone a fresh copy of sicor or delete sicor/sicor/aerosol_0_ch4_34d3778719cc87188787de09bb8f870d16050078.pkl.zip
# 2. run a sicor test including sicor_ac or enmap_ac (recreates cache file) -> upload newly created cache file # 2. run a sicor test including sicor_ac or enmap_ac (recreates cache file) -> upload newly created cache file
......
...@@ -6,59 +6,45 @@ channels: ...@@ -6,59 +6,45 @@ channels:
dependencies: dependencies:
- python>=3.7.* - python>=3.7.*
- pip - pip
- cartopy
- arosics>=1.0.0
- gdal - gdal
- geoarray>=0.9.0
- geopandas - geopandas
- holoviews
- ipython - ipython
- pyhdf - pyhdf
- python-fmask - python-fmask
- matplotlib - matplotlib
- numpy - numpy
- pyproj - pyproj
- pyrsr>=0.3.1
- py-tools-ds>=0.12.4
- scikit-image - scikit-image
- scikit-learn=0.23.2 # pinned due to spechomo classifiers created with this version - scikit-learn=0.23.2 # pinned due to spechomo classifiers created with this version
- shapely - shapely
- scipy<=1.1.0 # pinned due to https://gitext.gfz-potsdam.de/EnMAP/sicor/-/issues/63
# arosics - sicor
- pyfftw - spechomo
- pykrige
# sicor
- cachetools
- glymur
- h5py
- llvmlite
- numba
- pygrib
- pytables
- pip: - pip:
- arosics>=1.0.0
- cerberus - cerberus
- dill - dill
- ecmwf-api-client
- ephem - ephem
- geoalchemy2 - geoalchemy2
- geoarray>=0.9.0
- jsmin - jsmin
- iso8601 - iso8601
- natsort - natsort
- nested_dict - nested_dict
- openpyxl
- pandas - pandas
- psutil - psutil
- psycopg2 - psycopg2
- pyinstrument - pyinstrument
- pyorbital - pyorbital
- pyrsr>=0.3.1
- py_tools_ds>=0.12.4
- pytz - pytz
- redis - redis
- retools - retools
- redis-semaphore - redis-semaphore
- scipy
- sicor
- spechomo>=0.16
- spectral - spectral
- sqlalchemy - sqlalchemy
- timeout_decorator - timeout_decorator
...@@ -74,6 +60,3 @@ dependencies: ...@@ -74,6 +60,3 @@ dependencies:
- nose-htmloutput - nose-htmloutput
- rednose - rednose
- sphinx-argparse - sphinx-argparse
- sqlalchemy
- psycopg2
...@@ -2,6 +2,7 @@ FROM ci_base_centos:0.1 ...@@ -2,6 +2,7 @@ FROM ci_base_centos:0.1
# copy some needed stuff to /root # copy some needed stuff to /root
COPY *.yml /root/ COPY *.yml /root/
COPY *.sh /root/
# update the ci_env environment (that already contains all packages installed via 'docker_pyenvs' repo) # update the ci_env environment (that already contains all packages installed via 'docker_pyenvs' repo)
RUN /bin/bash -i -c "\ RUN /bin/bash -i -c "\
...@@ -10,14 +11,8 @@ RUN /bin/bash -i -c "\ ...@@ -10,14 +11,8 @@ RUN /bin/bash -i -c "\
source activate ci_env; \ source activate ci_env; \
conda env update -n ci_env -f /root/environment_gms_preprocessing.yml" conda env update -n ci_env -f /root/environment_gms_preprocessing.yml"
# copy sicor code to /tmp # download SICOR atmospheric lookup tables and copy already created cache files to SICOR root dir to speed up SICOR in CI jobs
COPY sicor /tmp/sicor RUN mkdir /tmp/sicor_cache
COPY *.zip /tmp/sicor_cache/
RUN bash -i -c "\ RUN /bin/bash -i -c "\
source /root/miniconda3/bin/activate ci_env; \ bash /root/setup_sicor_tables_and_cachefiles.sh"
cd /tmp/sicor/ ; \
make install"
# copy sicor cache files to sicor root directory (speeds up SICOR CI tests because table subsets dont have to be created each time)
# -> sicor root directory is the default directory of these cache files if sicor_cache_dir is not set in EnPT options
COPY *.zip /tmp/sicor/sicor/
#!/usr/bin/env bash
conda /root/miniconda3/bin/activate ci_env
# copy sicor cache files to sicor root directory (speeds up SICOR CI tests because table subsets dont have to be created each time)
# -> sicor root directory is the default directory of these cache files if sicor_cache_dir is not set in EnPT options
SICOR_PATH=$(python -c "from sicor import __path__; print(__path__[0])")
echo Copying SICOR cache files to ${SICOR_PATH}/...
cp /tmp/sicor_cache/*.zip ${SICOR_PATH}/
# download atmospheric lookup tables to save time when running CI jobs later
echo Downloading SICOR atmospheric lookup tables...
python -c "from sicor.tables import get_tables; get_tables(sensor=\"s2\"); get_tables(sensor=\"l8\")"
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment