Merged master into spectral_homogenization.

Former-commit-id: 36b7aab6

.gitlab-ci.yml

@@ -10,8 +10,8 @@ stages:
 test_gms_preprocessing:
   stage: test
   script:
-    - source /root/anaconda3/bin/activate
-    - export GDAL_DATA=/root/anaconda3/share/gdal
+    - source /root/miniconda3/bin/activate
+    - export GDAL_DATA=/root/miniconda3/share/gdal
     - export PYTHONPATH=$PYTHONPATH:/root  # /root <- directory needed later
     # update sicor
     # - conda install -y -q -c conda-forge basemap
@@ -36,8 +36,8 @@ test_gms_preprocessing:
 test_styles:
   stage: test
   script:
-    - source /root/anaconda3/bin/activate
-    - export GDAL_DATA=/root/anaconda3/share/gdal
+    - source /root/miniconda3/bin/activate
+    - export GDAL_DATA=/root/miniconda3/share/gdal
     - export PYTHONPATH=$PYTHONPATH:/root  # /root <- directory needed later
     - make lint
   artifacts:

gms_preprocessing/__init__.py

@@ -13,8 +13,8 @@ from .processing.process_controller import process_controller  # noqa: E402
 
 __author__ = """Daniel Scheffler"""
 __email__ = 'daniel.scheffler@gfz-potsdam.de'
-__version__ = '0.9.8'
-__versionalias__ = '20171104.01'
+__version__ = '0.9.9'
+__versionalias__ = '20171120.01'
 __all__ = ['algorithms',
            'io',
            'misc',

requirements.txt

-py_tools_ds>=0.10.0
+py_tools_ds>=0.12.4
 geoarray>=0.7.1
 arosics>=0.6.6
 git+https://gitext.gfz-potsdam.de/EnMAP/sicor.git

setup.py

@@ -14,7 +14,7 @@ with open('HISTORY.rst') as history_file:
 requirements = [
     'matplotlib', 'numpy', 'scikit-learn', 'scipy', 'gdal', 'pyproj', 'shapely', 'ephem', 'pyorbital', 'dill', 'pytz',
     'pandas', 'numba', 'spectral>=0.16', 'geopandas', 'iso8601', 'pyinstrument', 'geoalchemy2', 'sqlalchemy',
-    'psycopg2', 'py_tools_ds>=0.10.0', 'geoarray>=0.7.1', 'arosics>=0.6.6', 'six', 'tqdm'
+    'psycopg2', 'py_tools_ds>=0.12.4', 'geoarray>=0.7.1', 'arosics>=0.6.6', 'six', 'tqdm'
     # spectral<0.16  has some problems with writing signed integer 8bit data
     # fmask # conda install -c conda-forge python-fmask
     # 'pyhdf', # conda install --yes -c conda-forge pyhdf
@@ -25,7 +25,7 @@ test_requirements = requirements + ['coverage', 'nose', 'nose2', 'nose-htmloutpu
 
 setup(
     name='gms_preprocessing',
-    version='0.9.8',
+    version='0.9.9',
     description="GeoMultiSens - Scalable Multi-Sensor Analysis of Remote Sensing Data",
     long_description=readme + '\n\n' + history,
     author="Daniel Scheffler",

tests/CI_docker/build_testsuite_image.sh

@@ -2,10 +2,10 @@
 
 context_dir="./context"
 dockerfile="gms_preprocessing_ci.docker"
-tag="gms_preprocessing_ci:0.9.7"
+tag="gms_preprocessing_ci:0.9.9"
 gitlab_runner="gms_preprocessing_gitlab_CI_runner"
 
-# get sicor project # TODO move this to setup.py as soon as package is public
+# get sicor project
 rm -rf context/sicor
 git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git ./context/sicor
 

tests/CI_docker/context/environment_gms_preprocessing.yml

+name: gms_preprocessing
+
+channels: &id1
+  - http://conda.anaconda.org/ioam  # only for holoviews
+  - http://conda.anaconda.org/conda-forge
+
+dependencies:
+  - python=3.*.*
+  - pyqt
+  - numpy
+  - gdal
+  - scikit-image
+  - rasterio
+  - pyproj
+  - lxml
+  - geopandas
+  - ipython
+  - matplotlib
+  - basemap
+  - scikit-learn
+  - shapely
+  - pyhdf
+  - python-fmask
+  - holoviews
+  - bokeh
+
+  # arosics
+  - pyfftw
+  - pykrige
+
+  # sicor
+  - glymur
+  - pygrib
+  - cachetools
+  - pyhdf
+  - h5py
+  - pytables
+
+
+  - pip:
+    - dicttoxml
+    - jsmin
+    - cerberus
+    - pyprind
+    - pint
+    - iso8601
+    - tqdm
+    - mpld3
+    - sphinx-argparse
+    - six
+    - spectral
+    - flake8
+    - pycodestyle
+    - pylint
+    - pydocstyle
+    - nose
+    - nose2
+    - nose-htmloutput
+    - coverage
+    - rednose
+    - scipy
+    - ephem
+    - pyorbital
+    - dill
+    - pytz
+    - numba
+    - pyinstrument
+    - geoalchemy2
+    - sqlalchemy
+    - psycopg2
+    - pandas
+    - py_tools_ds>=0.12.4
+    - geoarray>=0.7.0
+    - arosics>=0.6.6

tests/CI_docker/context/gms_preprocessing_ci.docker

-FROM centos:7
-RUN yum update -y && \
-    yum install -y wget vim bzip2 gcc gcc-c++ make libgl1-mesa-glx mesa-libGL qt5-qtbase-gui git gdb
-
-ENV anaconda_dl 'Anaconda3-5.0.0-Linux-x86_64.sh'
-RUN /bin/bash -i -c "cd /root; wget https://repo.continuum.io/archive/$anaconda_dl ; \
-    bash -i /root/$anaconda_dl -b ; \
-    rm -f /root/$anaconda_dl"
-
-# 'conda install --yes -c conda-forge 'icu=58.*' lxml' fixing a bug for conda-forge gdal build, as of 06/2017
-# holoviews and bokeh is only needed for geoarray because its not installable via pip during geoarray.setup.py
-RUN /bin/bash -i -c "source /root/anaconda3/bin/activate ; \
-    conda install --yes pyqt numba; \
-    conda install --yes -c conda-forge pyfftw=0.10.4 ; \
-    conda install --yes -c ioam holoviews bokeh ; \
-    conda install --yes -c conda-forge numpy gdal scikit-image scikit-learn matplotlib pyproj rasterio shapely basemap \
-                                       pykrige glymur pygrib pyproj cachetools pyhdf ephem python-fmask scipy ; \
-    conda install --yes -c conda-forge 'icu=58.*' lxml ; \
-    pip install pandas geopandas dicttoxml jsmin cerberus pyprind pint iso8601 mpld3 sphinx-argparse dill pytz \
-                spectral>0.16 psycopg2 pyorbital pyinstrument geoalchemy2 sqlalchemy py_tools_ds>=0.10.0 \
-                geoarray>=0.7.1 arosics>=0.6.6 flake8 pycodestyle pylint pydocstyle nose nose2 nose-htmloutput \
-                coverage rednose six tqdm"   # must include all the requirements needed to build the docs!
+FROM gms_base_centos:pinned_v0.1
 
 # copy some needed stuff to /root
-#COPY *.pkl /root/ # EXAMPLE
-#COPY *.h5 /root/  # EXAMPLE
+COPY *.yml /root/
+
+RUN /bin/bash -i -c "\
+    source /root/miniconda3/bin/activate ; \
+    conda env update -n root -f /root/environment_gms_preprocessing.yml"
 
 # copy sicor code to /tmp
 COPY sicor /tmp/sicor
 
-# install git lfs
-RUN /bin/bash -i -c "curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.rpm.sh | bash"
-
-# installing git large file support, see here: https://git-lfs.github.com/ , change version number below for future upgrades
-ENV git_lfs_v='2.1.1'
-RUN /bin/bash -i -c "wget https://github.com/git-lfs/git-lfs/releases/download/v${git_lfs_v}/git-lfs-linux-amd64-${git_lfs_v}.tar.gz; \
-    tar -zxvf git-lfs-linux-amd64-${git_lfs_v}.tar.gz; \
-    cd git-lfs-${git_lfs_v}; \
-    bash ./install.sh"
-
-# set matplotlib backend to Agg as default, Qt won't work in headless docker containers
-RUN bash -i -c 'source /root/anaconda3/bin/activate; \
-    sed -i "/backend      : tkagg/c\backend      : Agg" $(python -c "import matplotlib; print(matplotlib.matplotlib_fname())"); \
-    sed -i "/backend      : Qt4Agg/c\backend      : Agg" $(python -c "import matplotlib; print(matplotlib.matplotlib_fname())"); \
-    sed -i "/backend      : qt5agg/c\backend      : Agg" $(python -c "import matplotlib; print(matplotlib.matplotlib_fname())")'
-
-RUN bash -i -c "source /root/anaconda3/bin/activate; \
-    cd /tmp/sicor/; make install; \
-    " # TODO move this to setup.py as soon as package is public
+RUN bash -i -c "source /root/miniconda3/bin/activate; \
+    cd /tmp/sicor/; make install; "