Updated CI setup files and .gitlab-ci.yml.

Signed-off-by: Daniel Scheffler <danschef@gfz-potsdam.de>

.gitlab-ci.yml

@@ -10,11 +10,9 @@ stages:
 test_pyrsr:
   stage: test
   script:
-    - source /root/miniconda3/bin/activate
-    - source activate ci_env
-    - export GDAL_DATA=/root/miniconda3/envs/ci_env/share/gdal
-    - export PYTHONPATH=$PYTHONPATH:/root  # /root <- directory needed later
+    - source /root/miniconda3/bin/activate ci_env
     - make nosetests
+
     # create the docs
     - pip install sphinx_rtd_theme  # Read-the-docs theme for SPHINX documentation
     - pip install sphinx-autodoc-typehints
@@ -31,10 +29,7 @@ test_pyrsr:
 test_styles:
   stage: test
   script:
-    - source /root/miniconda3/bin/activate
-    - source activate ci_env
-    - export GDAL_DATA=/root/miniconda3/envs/ci_env/share/gdal
-    - export PYTHONPATH=$PYTHONPATH:/root  # /root <- directory needed later
+    - source /root/miniconda3/bin/activate ci_env
     - make lint
   artifacts:
     paths:
@@ -48,12 +43,19 @@ test_pyrsr_install:
   stage: test
   script:
     - source /root/miniconda3/bin/activate
-    - conda create -y -q --name pyrsr_testinstall python=3
+
+    - conda update -n base -c conda-forge conda
+    # - conda update -c conda-forge python
+
+    - conda create -y -q -c conda-forge --name pyrsr_testinstall python=3
     - source activate pyrsr_testinstall
+
      # resolve some requirements with conda
-    - conda install --yes -q -c conda-forge numpy matplotlib pandas
+    - conda install --y -q -c conda-forge numpy matplotlib pandas
+
     # run installer
     - python setup.py install
+
     # test if its importable
     - cd ..
     - pwd
@@ -75,12 +77,16 @@ pages:  # this job must be called 'pages' to advise GitLab to upload content to
     - mkdir -p public/doc
     - mkdir -p public/coverage
     - mkdir -p public/nosetests_reports
+
     # Copy over the docs
     - cp -r docs/_build/html/* public/doc/
+
     # Copy over the coverage reports
     - cp -r htmlcov/* public/coverage/
+
     # Copy over the nosetests reports
     - cp nosetests.* public/nosetests_reports/
+
     # Check if everything is working great
     - ls -al public
     - ls -al public/doc

tests/CI_docker/build_testsuite_image.sh

@@ -2,30 +2,52 @@
 
 context_dir="./context"
 dockerfile="pyrsr_ci.docker"
-tag="pyrsr_ci:0.2.0"
+tag="pyrsr_ci:0.3.3"
 gitlab_runner="pyrsr_gitlab_CI_runner"
 
 echo "#### Build runner docker image"
-sudo docker rmi ${tag}
-sudo docker build -f ${context_dir}/${dockerfile} -m 20G -t ${tag} ${context_dir}
-# sudo docker build -f ./context/pyrsr_ci.docker -m 20G -t pyrsr_ci:0.2.0 ./context --no-cache
+docker rmi ${tag}
+docker build ${context_dir} \
+    --no-cache \
+    -f ${context_dir}/${dockerfile} \
+    -m 20G \
+    -t ${tag}
 
+# create the gitlab-runner docker container for the current project
+# NOTE: The 'gitlab-runner' and 'gitlab-ci-multi-runner' services will run within this container.
+#       The runner uses a 'config.toml' configuration file at /etc/gitlab-runner within the container which can be
+#       modified through additional parameters of the 'gitlab-runner register' command.
 echo "#### Create gitlab-runner (daemon) container with tag; ${tag}"
-sudo docker stop ${gitlab_runner}
-sudo docker rm ${gitlab_runner}
-sudo docker run -d --name ${gitlab_runner} --restart always \
--v /var/run/docker.sock:/var/run/docker.sock gitlab/gitlab-runner:latest
+docker stop ${gitlab_runner}
+docker rm ${gitlab_runner}
+docker run \
+    -d \
+    --name ${gitlab_runner} \
+    --restart always \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    gitlab/gitlab-runner:latest
 
+# register the runner at the corresponding GitLab repository via a registration-token
 echo "#### Register container at gitlab, get token here https://gitext.gfz-potsdam.de/geomultisens/pyrsr/settings/ci_cd"
 read -p "Please enter gitlab token: " token
 echo ""
 read -p "Please enter gitlab runner name: " runner_name
 echo "New gitlab runner image will named  ${gitlab_runner}"
-sudo docker exec -it ${gitlab_runner} /bin/bash -c "export RUNNER_EXECUTOR=docker && gitlab-ci-multi-runner register -n \
+# NOTE: In case of locally stored images (like here), the docker pull policy 'never' must be used
+#       (see https://docs.gitlab.com/runner/executors/docker.html#how-pull-policies-work).
+docker exec -it ${gitlab_runner} /bin/bash -c "\
+export RUNNER_EXECUTOR=docker && \
+gitlab-ci-multi-runner register \
+  --non-interactive \
+  --executor 'docker' \
+  --docker-image '${tag}' \
   --url 'https://gitext.gfz-potsdam.de/ci' \
   --registration-token '${token}' \
-  --run-untagged=true \
-  --locked=true \
-  --tag-list pyrsr_client \
   --description '${runner_name}' \
-  --docker-image '${tag}' "
+  --tag-list pyrsr_ci_client \
+  --run-untagged='true' \
+  --locked='true' \
+  --access-level='not_protected' \
+  --docker-pull-policy='never'
+  "
+ls

tests/CI_docker/context/environment_pyrsr.yml

 name: pyrsr
 
 channels: &id1
-  - http://conda.anaconda.org/conda-forge
+  - conda-forge
 
 dependencies:
   - python=3.*.*
+  - pip
   - matplotlib
   - numpy
   - pandas

tests/CI_docker/context/pyrsr_ci.docker

-FROM gms_base_centos:0.2
+FROM ci_base_centos:0.1
 
 # copy some needed stuff to /root
 COPY *.yml /root/
@@ -6,5 +6,6 @@ COPY *.yml /root/
 # update the ci_env environment (that already contains all packages installed via 'docker_pyenvs' repo)
 RUN /bin/bash -i -c "\
     source /root/miniconda3/bin/activate ; \
+    conda update -n base -c conda-forge conda;\
     source activate ci_env; \
     conda env update -n ci_env -f /root/environment_pyrsr.yml"