Updated CI setup files. sensormapgeo_ci.docker now inherits from ci_base_centos:0.1.

Signed-off-by: Daniel Scheffler <danschef@gfz-potsdam.de>

.gitlab-ci.yml

@@ -11,8 +11,6 @@ test_sensormapgeo:
   stage: test
   script:
     - source /root/miniconda3/bin/activate ci_env
-    - export GDAL_DATA=/root/miniconda3/envs/ci_env/share/gdal
-    - export PYTHONPATH=$PYTHONPATH:/root  # /root <- directory needed later
 
     # run nosetests
     - make nosetests
@@ -34,8 +32,6 @@ test_styles:
   stage: test
   script:
     - source /root/miniconda3/bin/activate ci_env
-    - export GDAL_DATA=/root/miniconda3/envs/ci_env/share/gdal
-    - export PYTHONPATH=$PYTHONPATH:/root  # /root <- directory needed later
     - make lint
   artifacts:
     paths:
@@ -49,12 +45,18 @@ test_sensormapgeo_install:
   stage: test
   script:
     - source /root/miniconda3/bin/activate
+
+    # update conda and python
+    - conda update -n base -c conda-forge conda
+    # - conda update -c conda-forge python
+
+    # create sensormapgeo environment
     - conda create -y -q --name sensormapgeo_testinstall python=3
     - source activate sensormapgeo_testinstall
 
     # resolve some requirements with conda
-    - conda config --set channel_priority strict  # otherwise gdal or libgdal may be installed from defaults channel
-    - conda install --yes -q -c conda-forge numpy gdal pyresample pyqt scikit-image rasterio pyproj lxml geopandas ipython
+    # - conda config --set channel_priority strict  # otherwise gdal or libgdal may be installed from defaults channel
+    - conda install --yes -q -c conda-forge numpy gdal pyresample pyqt scikit-image pyproj lxml geopandas ipython
 
     # run installer
     - python setup.py install
@@ -67,7 +69,6 @@ test_sensormapgeo_install:
     - python -c "from sensormapgeo import SensorMapGeometryTransformer, SensorMapGeometryTransformer3D"
   only:
     - master
-    - feature/add_CI
 
 
 pages:  # this job must be called 'pages' to advise GitLab to upload content to GitLab Pages
@@ -81,12 +82,16 @@ pages:  # this job must be called 'pages' to advise GitLab to upload content to
     - mkdir -p public/doc
     - mkdir -p public/coverage
     - mkdir -p public/nosetests_reports
+
     # Copy over the docs
     - cp -r docs/_build/html/* public/doc/
+
     # Copy over the coverage reports
     - cp -r htmlcov/* public/coverage/
+
     # Copy over the nosetests reports
     - cp nosetests.* public/nosetests_reports/
+
     # Check if everything is working great
     - ls -al public
     - ls -al public/doc
@@ -105,8 +110,7 @@ deploy_pypi:
   dependencies:
     - test_sensormapgeo
   script:
-    - source /root/miniconda3/bin/activate
-    - source activate ci_env
+    - source /root/miniconda3/bin/activate ci_env
     - pip install -U twine
     - python setup.py sdist
     - twine upload dist/*  # requires creds as environment variables

tests/CI_docker/build_testsuite_image.sh

@@ -2,30 +2,52 @@
 
 context_dir="./context"
 dockerfile="sensormapgeo_ci.docker"
-tag="sensormapgeo_ci:0.2.0"
+tag="sensormapgeo_ci:0.4.3"
 gitlab_runner="sensormapgeo_gitlab_CI_runner"
 
 echo "#### Build runner docker image"
-sudo docker rmi ${tag}
-sudo docker build -f ${context_dir}/${dockerfile} -m 20G -t ${tag} ${context_dir}
+docker rmi ${tag}
+docker build ${context_dir} \
+    --no-cache \
+    -f ${context_dir}/${dockerfile} \
+    -m 20G \
+    -t ${tag}
 
+# create the gitlab-runner docker container for the current project
+# NOTE: The 'gitlab-runner' and 'gitlab-ci-multi-runner' services will run within this container.
+#       The runner uses a 'config.toml' configuration file at /etc/gitlab-runner within the container which can be
+#       modified through additional parameters of the 'gitlab-runner register' command.
 echo "#### Create gitlab-runner (daemon) container with tag; ${tag}"
-sudo docker stop ${gitlab_runner}
-sudo docker rm ${gitlab_runner}
-sudo docker run -d --name ${gitlab_runner} --restart always \
--v /var/run/docker.sock:/var/run/docker.sock gitlab/gitlab-runner:latest
+docker stop ${gitlab_runner}
+docker rm ${gitlab_runner}
+docker run \
+    -d \
+    --name ${gitlab_runner} \
+    --restart always \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    gitlab/gitlab-runner:latest
 
+# register the runner at the corresponding GitLab repository via a registration-token
 echo "#### Register container at gitlab, get token here https://gitext.gfz-potsdam.de/EnMAP/sensormapgeo/settings/ci_cd"
 read -p "Please enter gitlab token: " token
 echo ""
 read -p "Please enter gitlab runner name: " runner_name
 echo "New gitlab runner image will named  ${gitlab_runner}"
-sudo docker exec -it ${gitlab_runner} /bin/bash -c "export RUNNER_EXECUTOR=docker && gitlab-ci-multi-runner register -n \
+# NOTE: In case of locally stored images (like here), the docker pull policy 'never' must be used
+#       (see https://docs.gitlab.com/runner/executors/docker.html#how-pull-policies-work).
+docker exec -it ${gitlab_runner} /bin/bash -c "\
+export RUNNER_EXECUTOR=docker && \
+gitlab-ci-multi-runner register \
+  --non-interactive \
+  --executor 'docker' \
+  --docker-image '${tag}' \
   --url 'https://gitext.gfz-potsdam.de/ci' \
   --registration-token '${token}' \
-  --run-untagged=true \
-  --locked=true \
-  --tag-list  sensormapgeo_ci_client \
   --description '${runner_name}' \
-  --docker-image '${tag}' "
+  --tag-list sensormapgeo_ci_client \
+  --run-untagged='true' \
+  --locked='true' \
+  --access-level='not_protected' \
+  --docker-pull-policy='never'
+  "
 ls

tests/CI_docker/context/environment_sensormapgeo.yml

 name: sensormapgeo
 
 channels: &id1
-  - http://conda.anaconda.org/conda-forge
+  - conda-forge
 
 dependencies:
   - python=3.*.*
   - pip  # avoids that conda uses the wrong pip
   - numpy
   - gdal
-  - conda-forge::libgdal  # force to use conda-forge for libgdal to avoid package version incompatiblies due to mixed channels (libkea.so.1.4.7: cannot open shared object file: No such file or directory)
   - pyresample>=1.11.0
   - pyproj>=2.2
 
   # py_tools_ds
   - pyqt
   - scikit-image
-  - rasterio
   - lxml
   - geopandas
   - ipython

tests/CI_docker/context/sensormapgeo_ci.docker

-FROM gms_base_centos:0.3
+FROM ci_base_centos:0.1
 
 # copy some needed stuff to /root
 COPY *.yml /root/
@@ -6,8 +6,7 @@ COPY *.yml /root/
 # update the ci_env environment (that already contains all packages installed via 'docker_pyenvs' repo)
 # NOTE: We update conda and Python to avoid version incompatibilities.
 RUN /bin/bash -i -c "\
-    source /root/miniconda3/bin/activate ; \
+    source /root/miniconda3/bin/activate; \
+    conda update -n base -c conda-forge conda ; \
     source activate ci_env; \
-    conda update -n base -c defaults conda ; \
-    conda update -c conda-forge python ; \
     conda env update -n ci_env -f /root/environment_sensormapgeo.yml"