Added some comments to build_testsuite_image.sh and set concurrency to 4....

Added some comments to build_testsuite_image.sh and set concurrency to 4. Updated deprecated code in .gitlab_ci.yml.

Signed-off-by: Daniel Scheffler <danschef@gfz-potsdam.de>

.gitlab-ci.yml

@@ -20,11 +20,7 @@ stages:
 test_py_tools_ds:
   stage: test
   script:
-    - source /root/miniconda3/bin/activate
-    - source activate ci_env
-    - export GDAL_DATA=/root/miniconda3/envs/ci_env/share/gdal
-    - export PYTHONPATH=$PYTHONPATH:/root  # /root <- directory needed later
-    - conda install -c conda-forge 'pyproj>=2.1'  # FIXME remove as soon as docker container is rebuilt
+    - source /root/miniconda3/bin/activate ci_env
     - make nosetests
     - make docs
   artifacts:
@@ -39,10 +35,7 @@ test_py_tools_ds:
 test_styles:
   stage: test
   script:
-    - source /root/miniconda3/bin/activate
-    - source activate ci_env
-    - export GDAL_DATA=/root/miniconda3/envs/ci_env/share/gdal
-    - export PYTHONPATH=$PYTHONPATH:/root  # /root <- directory needed later
+    - source /root/miniconda3/bin/activate ci_env
     - make lint
   artifacts:
     paths:
@@ -61,10 +54,10 @@ test_py_tools_ds_install:
 
     # update conda and python
     - conda update -n base -c defaults conda
-#    - conda update -c conda-forge python
+    # - conda update -c conda-forge python
 
     # avoid package incompatibilities due to usage of wrong channels
-#    - conda config --set channel_priority strict  # otherwise gdal or libgdal may be installed from defaults channel
+    # - conda config --set channel_priority strict  # otherwise gdal or libgdal may be installed from defaults channel
 
     # resolve some requirements with conda
     - conda install --yes -q -c conda-forge numpy gdal 'pyproj>=2.1.0' shapely geopandas
@@ -93,12 +86,16 @@ pages:  # this job must be called 'pages' to advise GitLab to upload content to
     - mkdir -p public/doc
     - mkdir -p public/coverage
     - mkdir -p public/nosetests_reports
+
     # Copy over the docs
     - cp -r docs/_build/html/* public/doc/
+
     # Copy over the coverage reports
     - cp -r htmlcov/* public/coverage/
+
     # Copy over the nosetests reports
     - cp nosetests.* public/nosetests_reports/
+
     # Check if everything is working great
     - ls -al public
     - ls -al public/doc
@@ -117,8 +114,7 @@ deploy_pypi:
   dependencies:
     - test_py_tools_ds
   script:
-    - source /root/miniconda3/bin/activate
-    - source activate ci_env
+    - source /root/miniconda3/bin/activate ci_env
     - pip install -U twine
     - python setup.py sdist
     - twine upload dist/*  # requires creds as environment variables
@@ -133,8 +129,7 @@ deploy_anaconda:
   dependencies:
     - test_py_tools_ds
   script:
-    - source /root/miniconda3/bin/activate
-    - source activate ci_env
+    - source /root/miniconda3/bin/activate ci_env
     - conda install -y -q conda-build conda-build-all anaconda-client
     - yum install -y patch # https://stackoverflow.com/questions/40392161/building-conda-skeleton-from-pypi-package-throws-error
     # - conda skeleton pypi py_tools_ds --output-dir public/conda.recipe/py_tools_ds # create recipe

tests/CI_docker/build_testsuite_image.sh

@@ -12,6 +12,10 @@ sudo docker build ${context_dir} \
     -m 20G \
     -t ${tag}
 
+# create the gitlab-runner docker container for the current project
+# NOTE: The 'gitlab-runner' and 'gitlab-ci-multi-runner' services will run within this container.
+#       The runner uses a 'config.toml' configuration file at /etc/gitlab-runner within the container which can be
+#       modified through additional parameters of the 'gitlab-runner register' command.
 echo "#### Create gitlab-runner (daemon) container with tag; ${tag}"
 sudo docker stop ${gitlab_runner}
 sudo docker rm ${gitlab_runner}
@@ -21,13 +25,15 @@ sudo docker run \
     --restart always \
     -v /var/run/docker.sock:/var/run/docker.sock \
     gitlab/gitlab-runner:latest
-# -v gitlab-runner-config:/etc/gitlab-runner \
 
+# register the runner at the corresponding GitLab repository via a registration-token
 echo "#### Register container at gitlab, get token here https://gitext.gfz-potsdam.de/danschef/py_tools_ds/settings/ci_cd"
 read -p "Please enter gitlab token: " token
 echo ""
 read -p "Please enter gitlab runner name: " runner_name
 echo "New gitlab runner image will named  ${gitlab_runner}"
+# NOTE: In case of locally stored images (like here), the docker pull policy 'never' must be used
+#       (see https://docs.gitlab.com/runner/executors/docker.html#how-pull-policies-work).
 sudo docker exec -it ${gitlab_runner} /bin/bash -c "\
 export RUNNER_EXECUTOR=docker && \
 gitlab-ci-multi-runner register \
@@ -41,5 +47,6 @@ gitlab-ci-multi-runner register \
   --run-untagged='true' \
   --locked='true' \
   --access-level='not_protected' \
-  --docker-pull-policy='never'"
+  --docker-pull-policy='never' \
+  --request-concurrency=4"
 ls