Commit 559b216e authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Updated CI setup files and .gitlab-ci.yml.


Signed-off-by: Daniel Scheffler's avatarDaniel Scheffler <danschef@gfz-potsdam.de>
parent ab15981e
Pipeline #12264 passed with stages
in 1 minute and 38 seconds
...@@ -10,11 +10,9 @@ stages: ...@@ -10,11 +10,9 @@ stages:
test_pyrsr: test_pyrsr:
stage: test stage: test
script: script:
- source /root/miniconda3/bin/activate - source /root/miniconda3/bin/activate ci_env
- source activate ci_env
- export GDAL_DATA=/root/miniconda3/envs/ci_env/share/gdal
- export PYTHONPATH=$PYTHONPATH:/root # /root <- directory needed later
- make nosetests - make nosetests
# create the docs # create the docs
- pip install sphinx_rtd_theme # Read-the-docs theme for SPHINX documentation - pip install sphinx_rtd_theme # Read-the-docs theme for SPHINX documentation
- pip install sphinx-autodoc-typehints - pip install sphinx-autodoc-typehints
...@@ -31,10 +29,7 @@ test_pyrsr: ...@@ -31,10 +29,7 @@ test_pyrsr:
test_styles: test_styles:
stage: test stage: test
script: script:
- source /root/miniconda3/bin/activate - source /root/miniconda3/bin/activate ci_env
- source activate ci_env
- export GDAL_DATA=/root/miniconda3/envs/ci_env/share/gdal
- export PYTHONPATH=$PYTHONPATH:/root # /root <- directory needed later
- make lint - make lint
artifacts: artifacts:
paths: paths:
...@@ -48,12 +43,19 @@ test_pyrsr_install: ...@@ -48,12 +43,19 @@ test_pyrsr_install:
stage: test stage: test
script: script:
- source /root/miniconda3/bin/activate - source /root/miniconda3/bin/activate
- conda create -y -q --name pyrsr_testinstall python=3
- conda update -n base -c conda-forge conda
# - conda update -c conda-forge python
- conda create -y -q -c conda-forge --name pyrsr_testinstall python=3
- source activate pyrsr_testinstall - source activate pyrsr_testinstall
# resolve some requirements with conda # resolve some requirements with conda
- conda install --yes -q -c conda-forge numpy matplotlib pandas - conda install --y -q -c conda-forge numpy matplotlib pandas
# run installer # run installer
- python setup.py install - python setup.py install
# test if its importable # test if its importable
- cd .. - cd ..
- pwd - pwd
...@@ -75,12 +77,16 @@ pages: # this job must be called 'pages' to advise GitLab to upload content to ...@@ -75,12 +77,16 @@ pages: # this job must be called 'pages' to advise GitLab to upload content to
- mkdir -p public/doc - mkdir -p public/doc
- mkdir -p public/coverage - mkdir -p public/coverage
- mkdir -p public/nosetests_reports - mkdir -p public/nosetests_reports
# Copy over the docs # Copy over the docs
- cp -r docs/_build/html/* public/doc/ - cp -r docs/_build/html/* public/doc/
# Copy over the coverage reports # Copy over the coverage reports
- cp -r htmlcov/* public/coverage/ - cp -r htmlcov/* public/coverage/
# Copy over the nosetests reports # Copy over the nosetests reports
- cp nosetests.* public/nosetests_reports/ - cp nosetests.* public/nosetests_reports/
# Check if everything is working great # Check if everything is working great
- ls -al public - ls -al public
- ls -al public/doc - ls -al public/doc
......
...@@ -2,30 +2,52 @@ ...@@ -2,30 +2,52 @@
context_dir="./context" context_dir="./context"
dockerfile="pyrsr_ci.docker" dockerfile="pyrsr_ci.docker"
tag="pyrsr_ci:0.2.0" tag="pyrsr_ci:0.3.3"
gitlab_runner="pyrsr_gitlab_CI_runner" gitlab_runner="pyrsr_gitlab_CI_runner"
echo "#### Build runner docker image" echo "#### Build runner docker image"
sudo docker rmi ${tag} docker rmi ${tag}
sudo docker build -f ${context_dir}/${dockerfile} -m 20G -t ${tag} ${context_dir} docker build ${context_dir} \
# sudo docker build -f ./context/pyrsr_ci.docker -m 20G -t pyrsr_ci:0.2.0 ./context --no-cache --no-cache \
-f ${context_dir}/${dockerfile} \
-m 20G \
-t ${tag}
# create the gitlab-runner docker container for the current project
# NOTE: The 'gitlab-runner' and 'gitlab-ci-multi-runner' services will run within this container.
# The runner uses a 'config.toml' configuration file at /etc/gitlab-runner within the container which can be
# modified through additional parameters of the 'gitlab-runner register' command.
echo "#### Create gitlab-runner (daemon) container with tag; ${tag}" echo "#### Create gitlab-runner (daemon) container with tag; ${tag}"
sudo docker stop ${gitlab_runner} docker stop ${gitlab_runner}
sudo docker rm ${gitlab_runner} docker rm ${gitlab_runner}
sudo docker run -d --name ${gitlab_runner} --restart always \ docker run \
-v /var/run/docker.sock:/var/run/docker.sock gitlab/gitlab-runner:latest -d \
--name ${gitlab_runner} \
--restart always \
-v /var/run/docker.sock:/var/run/docker.sock \
gitlab/gitlab-runner:latest
# register the runner at the corresponding GitLab repository via a registration-token
echo "#### Register container at gitlab, get token here https://gitext.gfz-potsdam.de/geomultisens/pyrsr/settings/ci_cd" echo "#### Register container at gitlab, get token here https://gitext.gfz-potsdam.de/geomultisens/pyrsr/settings/ci_cd"
read -p "Please enter gitlab token: " token read -p "Please enter gitlab token: " token
echo "" echo ""
read -p "Please enter gitlab runner name: " runner_name read -p "Please enter gitlab runner name: " runner_name
echo "New gitlab runner image will named ${gitlab_runner}" echo "New gitlab runner image will named ${gitlab_runner}"
sudo docker exec -it ${gitlab_runner} /bin/bash -c "export RUNNER_EXECUTOR=docker && gitlab-ci-multi-runner register -n \ # NOTE: In case of locally stored images (like here), the docker pull policy 'never' must be used
# (see https://docs.gitlab.com/runner/executors/docker.html#how-pull-policies-work).
docker exec -it ${gitlab_runner} /bin/bash -c "\
export RUNNER_EXECUTOR=docker && \
gitlab-ci-multi-runner register \
--non-interactive \
--executor 'docker' \
--docker-image '${tag}' \
--url 'https://gitext.gfz-potsdam.de/ci' \ --url 'https://gitext.gfz-potsdam.de/ci' \
--registration-token '${token}' \ --registration-token '${token}' \
--run-untagged=true \
--locked=true \
--tag-list pyrsr_client \
--description '${runner_name}' \ --description '${runner_name}' \
--docker-image '${tag}' " --tag-list pyrsr_ci_client \
--run-untagged='true' \
--locked='true' \
--access-level='not_protected' \
--docker-pull-policy='never'
"
ls
name: pyrsr name: pyrsr
channels: &id1 channels: &id1
- http://conda.anaconda.org/conda-forge - conda-forge
dependencies: dependencies:
- python=3.*.* - python=3.*.*
- pip
- matplotlib - matplotlib
- numpy - numpy
- pandas - pandas
......
FROM gms_base_centos:0.2 FROM ci_base_centos:0.1
# copy some needed stuff to /root # copy some needed stuff to /root
COPY *.yml /root/ COPY *.yml /root/
...@@ -6,5 +6,6 @@ COPY *.yml /root/ ...@@ -6,5 +6,6 @@ COPY *.yml /root/
# update the ci_env environment (that already contains all packages installed via 'docker_pyenvs' repo) # update the ci_env environment (that already contains all packages installed via 'docker_pyenvs' repo)
RUN /bin/bash -i -c "\ RUN /bin/bash -i -c "\
source /root/miniconda3/bin/activate ; \ source /root/miniconda3/bin/activate ; \
conda update -n base -c conda-forge conda;\
source activate ci_env; \ source activate ci_env; \
conda env update -n ci_env -f /root/environment_pyrsr.yml" conda env update -n ci_env -f /root/environment_pyrsr.yml"
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment