Commit f9aed261 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Added some comments to build_testsuite_image.sh and set concurrency to 4....


Added some comments to build_testsuite_image.sh and set concurrency to 4. Updated deprecated code in .gitlab_ci.yml.
Signed-off-by: Daniel Scheffler's avatarDaniel Scheffler <danschef@gfz-potsdam.de>
parent e4ab9c3a
Pipeline #12214 passed with stage
in 21 seconds
...@@ -20,11 +20,7 @@ stages: ...@@ -20,11 +20,7 @@ stages:
test_py_tools_ds: test_py_tools_ds:
stage: test stage: test
script: script:
- source /root/miniconda3/bin/activate - source /root/miniconda3/bin/activate ci_env
- source activate ci_env
- export GDAL_DATA=/root/miniconda3/envs/ci_env/share/gdal
- export PYTHONPATH=$PYTHONPATH:/root # /root <- directory needed later
- conda install -c conda-forge 'pyproj>=2.1' # FIXME remove as soon as docker container is rebuilt
- make nosetests - make nosetests
- make docs - make docs
artifacts: artifacts:
...@@ -39,10 +35,7 @@ test_py_tools_ds: ...@@ -39,10 +35,7 @@ test_py_tools_ds:
test_styles: test_styles:
stage: test stage: test
script: script:
- source /root/miniconda3/bin/activate - source /root/miniconda3/bin/activate ci_env
- source activate ci_env
- export GDAL_DATA=/root/miniconda3/envs/ci_env/share/gdal
- export PYTHONPATH=$PYTHONPATH:/root # /root <- directory needed later
- make lint - make lint
artifacts: artifacts:
paths: paths:
...@@ -61,10 +54,10 @@ test_py_tools_ds_install: ...@@ -61,10 +54,10 @@ test_py_tools_ds_install:
# update conda and python # update conda and python
- conda update -n base -c defaults conda - conda update -n base -c defaults conda
# - conda update -c conda-forge python # - conda update -c conda-forge python
# avoid package incompatibilities due to usage of wrong channels # avoid package incompatibilities due to usage of wrong channels
# - conda config --set channel_priority strict # otherwise gdal or libgdal may be installed from defaults channel # - conda config --set channel_priority strict # otherwise gdal or libgdal may be installed from defaults channel
# resolve some requirements with conda # resolve some requirements with conda
- conda install --yes -q -c conda-forge numpy gdal 'pyproj>=2.1.0' shapely geopandas - conda install --yes -q -c conda-forge numpy gdal 'pyproj>=2.1.0' shapely geopandas
...@@ -93,12 +86,16 @@ pages: # this job must be called 'pages' to advise GitLab to upload content to ...@@ -93,12 +86,16 @@ pages: # this job must be called 'pages' to advise GitLab to upload content to
- mkdir -p public/doc - mkdir -p public/doc
- mkdir -p public/coverage - mkdir -p public/coverage
- mkdir -p public/nosetests_reports - mkdir -p public/nosetests_reports
# Copy over the docs # Copy over the docs
- cp -r docs/_build/html/* public/doc/ - cp -r docs/_build/html/* public/doc/
# Copy over the coverage reports # Copy over the coverage reports
- cp -r htmlcov/* public/coverage/ - cp -r htmlcov/* public/coverage/
# Copy over the nosetests reports # Copy over the nosetests reports
- cp nosetests.* public/nosetests_reports/ - cp nosetests.* public/nosetests_reports/
# Check if everything is working great # Check if everything is working great
- ls -al public - ls -al public
- ls -al public/doc - ls -al public/doc
...@@ -117,8 +114,7 @@ deploy_pypi: ...@@ -117,8 +114,7 @@ deploy_pypi:
dependencies: dependencies:
- test_py_tools_ds - test_py_tools_ds
script: script:
- source /root/miniconda3/bin/activate - source /root/miniconda3/bin/activate ci_env
- source activate ci_env
- pip install -U twine - pip install -U twine
- python setup.py sdist - python setup.py sdist
- twine upload dist/* # requires creds as environment variables - twine upload dist/* # requires creds as environment variables
...@@ -133,8 +129,7 @@ deploy_anaconda: ...@@ -133,8 +129,7 @@ deploy_anaconda:
dependencies: dependencies:
- test_py_tools_ds - test_py_tools_ds
script: script:
- source /root/miniconda3/bin/activate - source /root/miniconda3/bin/activate ci_env
- source activate ci_env
- conda install -y -q conda-build conda-build-all anaconda-client - conda install -y -q conda-build conda-build-all anaconda-client
- yum install -y patch # https://stackoverflow.com/questions/40392161/building-conda-skeleton-from-pypi-package-throws-error - yum install -y patch # https://stackoverflow.com/questions/40392161/building-conda-skeleton-from-pypi-package-throws-error
# - conda skeleton pypi py_tools_ds --output-dir public/conda.recipe/py_tools_ds # create recipe # - conda skeleton pypi py_tools_ds --output-dir public/conda.recipe/py_tools_ds # create recipe
......
...@@ -12,6 +12,10 @@ sudo docker build ${context_dir} \ ...@@ -12,6 +12,10 @@ sudo docker build ${context_dir} \
-m 20G \ -m 20G \
-t ${tag} -t ${tag}
# create the gitlab-runner docker container for the current project
# NOTE: The 'gitlab-runner' and 'gitlab-ci-multi-runner' services will run within this container.
# The runner uses a 'config.toml' configuration file at /etc/gitlab-runner within the container which can be
# modified through additional parameters of the 'gitlab-runner register' command.
echo "#### Create gitlab-runner (daemon) container with tag; ${tag}" echo "#### Create gitlab-runner (daemon) container with tag; ${tag}"
sudo docker stop ${gitlab_runner} sudo docker stop ${gitlab_runner}
sudo docker rm ${gitlab_runner} sudo docker rm ${gitlab_runner}
...@@ -21,13 +25,15 @@ sudo docker run \ ...@@ -21,13 +25,15 @@ sudo docker run \
--restart always \ --restart always \
-v /var/run/docker.sock:/var/run/docker.sock \ -v /var/run/docker.sock:/var/run/docker.sock \
gitlab/gitlab-runner:latest gitlab/gitlab-runner:latest
# -v gitlab-runner-config:/etc/gitlab-runner \
# register the runner at the corresponding GitLab repository via a registration-token
echo "#### Register container at gitlab, get token here https://gitext.gfz-potsdam.de/danschef/py_tools_ds/settings/ci_cd" echo "#### Register container at gitlab, get token here https://gitext.gfz-potsdam.de/danschef/py_tools_ds/settings/ci_cd"
read -p "Please enter gitlab token: " token read -p "Please enter gitlab token: " token
echo "" echo ""
read -p "Please enter gitlab runner name: " runner_name read -p "Please enter gitlab runner name: " runner_name
echo "New gitlab runner image will named ${gitlab_runner}" echo "New gitlab runner image will named ${gitlab_runner}"
# NOTE: In case of locally stored images (like here), the docker pull policy 'never' must be used
# (see https://docs.gitlab.com/runner/executors/docker.html#how-pull-policies-work).
sudo docker exec -it ${gitlab_runner} /bin/bash -c "\ sudo docker exec -it ${gitlab_runner} /bin/bash -c "\
export RUNNER_EXECUTOR=docker && \ export RUNNER_EXECUTOR=docker && \
gitlab-ci-multi-runner register \ gitlab-ci-multi-runner register \
...@@ -41,5 +47,6 @@ gitlab-ci-multi-runner register \ ...@@ -41,5 +47,6 @@ gitlab-ci-multi-runner register \
--run-untagged='true' \ --run-untagged='true' \
--locked='true' \ --locked='true' \
--access-level='not_protected' \ --access-level='not_protected' \
--docker-pull-policy='never'" --docker-pull-policy='never' \
--request-concurrency=4"
ls ls
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment