From a1a4dc6185346351b6a7ef39c792f524c9900f9c Mon Sep 17 00:00:00 2001 From: Romulo Goncalves Date: Fri, 19 Feb 2021 11:14:12 +0100 Subject: [PATCH 1/4] Add runner --- tests/CI_docker/build_hasa_testsuite_image.sh | 33 ++++++++++++++ .../context/build_runner_image.docker | 45 +++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 tests/CI_docker/build_hasa_testsuite_image.sh create mode 100644 tests/CI_docker/context/build_runner_image.docker diff --git a/tests/CI_docker/build_hasa_testsuite_image.sh b/tests/CI_docker/build_hasa_testsuite_image.sh new file mode 100644 index 0000000..15392fd --- /dev/null +++ b/tests/CI_docker/build_hasa_testsuite_image.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +context_dir="./context" +dockerfile="build_runner_image.docker" +runner_version="latest" +runner_tag="hasa_ci:$runner_version" +gitlab_runner="hasa_gitlab_CI_runner" + +echo "#### Build runner docker image" +docker rmi ${runner_tag} +docker build -f ${context_dir}/${dockerfile} -m 20G -t ${runner_tag} ${context_dir} + +echo "#### Create gitlab-runner (daemon) container" +docker stop ${gitlab_runner} +docker rm ${gitlab_runner} +docker run -d --name ${gitlab_runner} --network host --restart always -v /var/run/docker.sock:/var/run/docker.sock gitlab/gitlab-runner:latest + +echo "#### Register container at gitlab" +read -p "Please enter gitlab token: " token +echo "" +read -p "Please enter gitlab runner name: " runner_name +echo "${runner_tag}" +docker exec -it ${gitlab_runner} /bin/bash -c "export RUNNER_EXECUTOR=docker && gitlab-ci-multi-runner register -n \ + --url 'https://git.gfz-potsdam.de' \ + --registration-token '${token}' \ + --run-untagged=true \ + --docker-network-mode='host' \ + --locked=true \ + --tag-list hasa \ + --docker-pull-policy='never' \ + --description '${runner_name}' \ + --docker-image '${runner_tag}' " + diff --git a/tests/CI_docker/context/build_runner_image.docker b/tests/CI_docker/context/build_runner_image.docker new file mode 100644 index 0000000..0fe2ce2 --- /dev/null +++ b/tests/CI_docker/context/build_runner_image.docker @@ -0,0 +1,45 @@ +FROM ubuntu:20.04 +RUN apt-get update -y && \ + echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections && \ + apt-get install -y -q dialog apt-utils && \ + apt-get install libudunits2-dev build-essential bash-completion automake -y && \ + apt-get install cmake zlib1g-dev libpng-dev libjpeg-dev -y && \ + apt-get install wget vim zip unzip curl git jq apt-rdepends -y && \ + apt-get install strace bzip2 cron make gcc -y +# add user hasa +RUN /bin/bash -i -c "\ + groupadd -g 1000 hasa && \ + useradd -g 1000 -u 1000 -m -s $(which bash) hasa && \ + cat /root/.bashrc > /home/hasa/.bashrc \ +" +# install dependencies +RUN apt-get update -y && \ + apt-get install -y libjq-dev protobuf-compiler libprotobuf-dev proj-bin gdal-bin libgdal-dev jq libv8-dev pandoc +# install R +RUN /bin/bash -i -c "\ + apt-get install -y dirmngr gnupg apt-transport-https ca-certificates software-properties-common && \ + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ + add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/' && \ + apt-get install -y r-base \ +" +# install Python and pip +RUN /bin/bash -i -c "\ + apt-get install -y software-properties-common && \ + add-apt-repository ppa:deadsnakes/ppa && \ + apt-get update -y && \ + apt-get install -y python3.8 && \ + apt-get install -y python3-pip \ +" +# install Python dependencies +RUN /bin/bash -i -c "\ + pip3 install geojson \ +" +# install jupyter-lab and IRkernel +RUN /bin/bash -i -c "\ + pip3 install jupyterlab \ +" +# install IRkernel and HaSa +COPY install.R /home/hasa/install.R +RUN /bin/bash -i -c "\ + Rscript /home/hasa/install.R \ +" -- GitLab From a9461c0c23521753d3469e46768f3e905b6c31c8 Mon Sep 17 00:00:00 2001 From: Romulo Goncalves Date: Fri, 19 Feb 2021 11:18:10 +0100 Subject: [PATCH 2/4] Script to build and register an hasa testsuite image --- docker/build_register_hasa_testsuite_image.sh | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100755 docker/build_register_hasa_testsuite_image.sh diff --git a/docker/build_register_hasa_testsuite_image.sh b/docker/build_register_hasa_testsuite_image.sh new file mode 100755 index 0000000..db22c6e --- /dev/null +++ b/docker/build_register_hasa_testsuite_image.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +context_dir="./context" +dockerfile="hasa.docker" +runner_version="latest" +runner_tag="hasa_ci:$runner_version" +gitlab_runner="hasa_gitlab_CI_runner" + +echo "#### Build runner docker image" +docker rmi ${runner_tag} +docker build -f ${context_dir}/${dockerfile} -m 20G -t ${runner_tag} ${context_dir} + +echo "#### Create gitlab-runner (daemon) container" +docker stop ${gitlab_runner} +docker rm ${gitlab_runner} +docker run -d --name ${gitlab_runner} --network host --restart always -v /var/run/docker.sock:/var/run/docker.sock gitlab/gitlab-runner:latest + +echo "#### Register container at gitlab" +read -p "Please enter gitlab token: " token +echo "" +read -p "Please enter gitlab runner name: " runner_name +echo "${runner_tag}" +docker exec -it ${gitlab_runner} /bin/bash -c "export RUNNER_EXECUTOR=docker && gitlab-ci-multi-runner register -n \ + --url 'https://git.gfz-potsdam.de' \ + --registration-token '${token}' \ + --run-untagged=true \ + --docker-network-mode='host' \ + --locked=true \ + --tag-list hasa \ + --docker-pull-policy='never' \ + --description '${runner_name}' \ + --docker-image '${runner_tag}' " + -- GitLab From 82833230b696ad15f02f02921249105173d0f799 Mon Sep 17 00:00:00 2001 From: Romulo Goncalves Date: Fri, 19 Feb 2021 11:19:16 +0100 Subject: [PATCH 3/4] Let's use a single docker image for testing and deployment. --- tests/CI_docker/build_hasa_testsuite_image.sh | 33 -------------- .../context/build_runner_image.docker | 45 ------------------- 2 files changed, 78 deletions(-) delete mode 100644 tests/CI_docker/build_hasa_testsuite_image.sh delete mode 100644 tests/CI_docker/context/build_runner_image.docker diff --git a/tests/CI_docker/build_hasa_testsuite_image.sh b/tests/CI_docker/build_hasa_testsuite_image.sh deleted file mode 100644 index 15392fd..0000000 --- a/tests/CI_docker/build_hasa_testsuite_image.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -context_dir="./context" -dockerfile="build_runner_image.docker" -runner_version="latest" -runner_tag="hasa_ci:$runner_version" -gitlab_runner="hasa_gitlab_CI_runner" - -echo "#### Build runner docker image" -docker rmi ${runner_tag} -docker build -f ${context_dir}/${dockerfile} -m 20G -t ${runner_tag} ${context_dir} - -echo "#### Create gitlab-runner (daemon) container" -docker stop ${gitlab_runner} -docker rm ${gitlab_runner} -docker run -d --name ${gitlab_runner} --network host --restart always -v /var/run/docker.sock:/var/run/docker.sock gitlab/gitlab-runner:latest - -echo "#### Register container at gitlab" -read -p "Please enter gitlab token: " token -echo "" -read -p "Please enter gitlab runner name: " runner_name -echo "${runner_tag}" -docker exec -it ${gitlab_runner} /bin/bash -c "export RUNNER_EXECUTOR=docker && gitlab-ci-multi-runner register -n \ - --url 'https://git.gfz-potsdam.de' \ - --registration-token '${token}' \ - --run-untagged=true \ - --docker-network-mode='host' \ - --locked=true \ - --tag-list hasa \ - --docker-pull-policy='never' \ - --description '${runner_name}' \ - --docker-image '${runner_tag}' " - diff --git a/tests/CI_docker/context/build_runner_image.docker b/tests/CI_docker/context/build_runner_image.docker deleted file mode 100644 index 0fe2ce2..0000000 --- a/tests/CI_docker/context/build_runner_image.docker +++ /dev/null @@ -1,45 +0,0 @@ -FROM ubuntu:20.04 -RUN apt-get update -y && \ - echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections && \ - apt-get install -y -q dialog apt-utils && \ - apt-get install libudunits2-dev build-essential bash-completion automake -y && \ - apt-get install cmake zlib1g-dev libpng-dev libjpeg-dev -y && \ - apt-get install wget vim zip unzip curl git jq apt-rdepends -y && \ - apt-get install strace bzip2 cron make gcc -y -# add user hasa -RUN /bin/bash -i -c "\ - groupadd -g 1000 hasa && \ - useradd -g 1000 -u 1000 -m -s $(which bash) hasa && \ - cat /root/.bashrc > /home/hasa/.bashrc \ -" -# install dependencies -RUN apt-get update -y && \ - apt-get install -y libjq-dev protobuf-compiler libprotobuf-dev proj-bin gdal-bin libgdal-dev jq libv8-dev pandoc -# install R -RUN /bin/bash -i -c "\ - apt-get install -y dirmngr gnupg apt-transport-https ca-certificates software-properties-common && \ - apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ - add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/' && \ - apt-get install -y r-base \ -" -# install Python and pip -RUN /bin/bash -i -c "\ - apt-get install -y software-properties-common && \ - add-apt-repository ppa:deadsnakes/ppa && \ - apt-get update -y && \ - apt-get install -y python3.8 && \ - apt-get install -y python3-pip \ -" -# install Python dependencies -RUN /bin/bash -i -c "\ - pip3 install geojson \ -" -# install jupyter-lab and IRkernel -RUN /bin/bash -i -c "\ - pip3 install jupyterlab \ -" -# install IRkernel and HaSa -COPY install.R /home/hasa/install.R -RUN /bin/bash -i -c "\ - Rscript /home/hasa/install.R \ -" -- GitLab From ccd0f64004254b8a758debce782b6df2a93de25b Mon Sep 17 00:00:00 2001 From: Romulo Goncalves Date: Fri, 19 Feb 2021 11:23:18 +0100 Subject: [PATCH 4/4] We need to use the host network. --- docker/build_docker.sh | 2 +- docker/build_register_hasa_testsuite_image.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/build_docker.sh b/docker/build_docker.sh index 9e713d5..0825936 100755 --- a/docker/build_docker.sh +++ b/docker/build_docker.sh @@ -13,7 +13,7 @@ input_data_folder="/tmp/hasa" if [ $(sudo docker images | grep ${runner_iname} | wc -l) == 0 ] then # build docker image - sudo docker build -f ${context_dir}/${dockerfile} -m 20G -t ${runner_tag} ${context_dir} + sudo docker build --network=host -f ${context_dir}/${dockerfile} -m 20G -t ${runner_tag} ${context_dir} else echo "It already exists a Docker image with the name ${runner_tag}!!!" fi diff --git a/docker/build_register_hasa_testsuite_image.sh b/docker/build_register_hasa_testsuite_image.sh index db22c6e..2081e6d 100755 --- a/docker/build_register_hasa_testsuite_image.sh +++ b/docker/build_register_hasa_testsuite_image.sh @@ -8,7 +8,7 @@ gitlab_runner="hasa_gitlab_CI_runner" echo "#### Build runner docker image" docker rmi ${runner_tag} -docker build -f ${context_dir}/${dockerfile} -m 20G -t ${runner_tag} ${context_dir} +docker build --network=host -f ${context_dir}/${dockerfile} -m 20G -t ${runner_tag} ${context_dir} echo "#### Create gitlab-runner (daemon) container" docker stop ${gitlab_runner} -- GitLab