Commit 0b4a1e55 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Merge branch 'feature/add_CI' into 'master'

Feature/add ci

See merge request !1
parents f6dbf2e4 442a3126
Pipeline #5487 passed with stages
in 3 minutes and 7 seconds
...@@ -129,76 +129,5 @@ dmypy.json ...@@ -129,76 +129,5 @@ dmypy.json
# Pyre type checker # Pyre type checker
.pyre/ .pyre/
# PyCharm settings
### JetBrains template .idea/
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
# User-specific stuff
.idea/**/workspace.xml
.idea/**/tasks.xml
.idea/**/usage.statistics.xml
.idea/**/dictionaries
.idea/**/shelf
# Generated files
.idea/**/contentModel.xml
# Sensitive or high-churn files
.idea/**/dataSources/
.idea/**/dataSources.ids
.idea/**/dataSources.local.xml
.idea/**/sqlDataSources.xml
.idea/**/dynamic.xml
.idea/**/uiDesigner.xml
.idea/**/dbnavigator.xml
# Gradle
.idea/**/gradle.xml
.idea/**/libraries
# Gradle and Maven with auto-import
# When using Gradle or Maven with auto-import, you should exclude module files,
# since they will be recreated, and may cause churn. Uncomment if using
# auto-import.
# .idea/artifacts
# .idea/compiler.xml
# .idea/modules.xml
# .idea/*.iml
# .idea/modules
# *.iml
# *.ipr
# CMake
cmake-build-*/
# Mongo Explorer plugin
.idea/**/mongoSettings.xml
# File-based project format
*.iws
# IntelliJ
out/
# mpeltonen/sbt-idea plugin
.idea_modules/
# JIRA plugin
atlassian-ide-plugin.xml
# Cursive Clojure plugin
.idea/replstate.xml
# Crashlytics plugin (for Android Studio and IntelliJ)
com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties
# Editor-based Rest Client
.idea/httpRequests
# Android studio 3.1+ serialized cache file
.idea/caches/build_file_checksums.ser
before_script:
- git lfs pull
stages:
- test
- deploy
test_sensormapgeo:
stage: test
script:
- source /root/miniconda3/bin/activate ci_env
- export GDAL_DATA=/root/miniconda3/envs/ci_env/share/gdal
- export PYTHONPATH=$PYTHONPATH:/root # /root <- directory needed later
# run nosetests
- make nosetests
# create the docs
- pip install sphinx_rtd_theme # Read-the-docs theme for SPHINX documentation
- pip install sphinx-autodoc-typehints
- make docs
artifacts:
paths:
- htmlcov/
- docs/_build/html/
- nosetests.html
- nosetests.xml
when: always
test_styles:
stage: test
script:
- source /root/miniconda3/bin/activate ci_env
- export GDAL_DATA=/root/miniconda3/envs/ci_env/share/gdal
- export PYTHONPATH=$PYTHONPATH:/root # /root <- directory needed later
- make lint
artifacts:
paths:
- tests/linting/flake8.log
- tests/linting/pycodestyle.log
- tests/linting/pydocstyle.log
when: always
test_sensormapgeo_install:
stage: test
script:
- source /root/miniconda3/bin/activate
- conda create -y -q --name sensormapgeo_testinstall python=3
- source activate sensormapgeo_testinstall
# resolve some requirements with conda
- conda install --yes -q -c conda-forge numpy gdal pyresample pyqt scikit-image rasterio pyproj lxml geopandas ipython
# run installer
- python setup.py install
# test if its importable
- cd ..
- pwd
- ls
- python -c "import sensormapgeo; print(sensormapgeo)"
- python -c "from sensormapgeo import SensorMapGeometryTransformer, SensorMapGeometryTransformer3D"
only:
- master
- feature/add_CI
pages: # this job must be called 'pages' to advise GitLab to upload content to GitLab Pages
stage: deploy
dependencies:
- test_sensormapgeo
script:
# Create the public directory
- rm -rf public
- mkdir public
- mkdir -p public/doc
- mkdir -p public/coverage
- mkdir -p public/nosetests_reports
# Copy over the docs
- cp -r docs/_build/html/* public/doc/
# Copy over the coverage reports
- cp -r htmlcov/* public/coverage/
# Copy over the nosetests reports
- cp nosetests.* public/nosetests_reports/
# Check if everything is working great
- ls -al public
- ls -al public/doc
- ls -al public/coverage
- ls -al public/nosetests_reports
artifacts:
paths:
- public
expire_in: 30 days
only:
- master
deploy_pypi:
stage: deploy
dependencies:
- test_sensormapgeo
script:
- source /root/miniconda3/bin/activate
- source activate ci_env
- pip install -U twine
- python setup.py sdist
- twine upload dist/* # requires creds as environment variables
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- dev
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="JavaScriptSettings">
<option name="languageLevel" value="ES6" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6.0 (~/python_GFZ/python/bin/python3.6)" project-jdk-type="Python SDK" />
<component name="PythonCompatibilityInspectionAdvertiser">
<option name="version" value="3" />
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/sensormapgeo.iml" filepath="$PROJECT_DIR$/.idea/sensormapgeo.iml" />
</modules>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="PackageRequirementsSettings">
<option name="requirementsPath" value="" />
</component>
<component name="PyDocumentationSettings">
<option name="format" value="PLAIN" />
<option name="myDocStringFormat" value="Plain" />
</component>
<component name="TestRunnerService">
<option name="PROJECT_TEST_RUNNER" value="Nosetests" />
</component>
</module>
\ No newline at end of file
...@@ -42,13 +42,17 @@ clean-pyc: ## remove Python file artifacts ...@@ -42,13 +42,17 @@ clean-pyc: ## remove Python file artifacts
find . -name '__pycache__' -exec rm -fr {} + find . -name '__pycache__' -exec rm -fr {} +
clean-test: ## remove test and coverage artifacts clean-test: ## remove test and coverage artifacts
coverage erase
rm -fr .tox/ rm -fr .tox/
rm -f .coverage rm -f .coverage
rm -fr htmlcov/ rm -fr htmlcov/
rm -fr .pytest_cache rm -fr nosetests.html
rm -fr nosetests.xml
lint: ## check style with flake8 lint: ## check style with flake8
flake8 sensormapgeo tests flake8 --max-line-length=120 sensormapgeo tests > ./tests/linting/flake8.log
pycodestyle sensormapgeo --exclude="*.ipynb,*.ipynb*" --max-line-length=120 > ./tests/linting/pycodestyle.log
-pydocstyle sensormapgeo > ./tests/linting/pydocstyle.log
test: ## run tests quickly with the default Python test: ## run tests quickly with the default Python
python setup.py test python setup.py test
...@@ -56,11 +60,18 @@ test: ## run tests quickly with the default Python ...@@ -56,11 +60,18 @@ test: ## run tests quickly with the default Python
test-all: ## run tests on every Python version with tox test-all: ## run tests on every Python version with tox
tox tox
coverage: ## check code coverage quickly with the default Python coverage: clean-test ## check code coverage quickly with the default Python
coverage run --source sensormapgeo setup.py test coverage run --source sensormapgeo setup.py test
coverage report -m coverage report -m
coverage html coverage html
$(BROWSER) htmlcov/index.html # $(BROWSER) htmlcov/index.html
nosetests: clean-test ## Runs nosetests with coverage, xUnit and nose-html-output
## - puts the coverage results in the folder 'htmlcov'
## - generates 'nosetests.html' (--with-html)
## - generates 'nosetests.xml' (--with-xunit) which is currently not visualizable by GitLab
nosetests -vv --with-coverage --cover-package=sensormapgeo --cover-erase --cover-html --cover-html-dir=htmlcov \
--with-html --with-xunit --rednose --force-color
docs: ## generate Sphinx HTML documentation, including API docs docs: ## generate Sphinx HTML documentation, including API docs
rm -f docs/sensormapgeo.rst rm -f docs/sensormapgeo.rst
...@@ -68,7 +79,9 @@ docs: ## generate Sphinx HTML documentation, including API docs ...@@ -68,7 +79,9 @@ docs: ## generate Sphinx HTML documentation, including API docs
sphinx-apidoc -o docs/ sensormapgeo sphinx-apidoc -o docs/ sensormapgeo
$(MAKE) -C docs clean $(MAKE) -C docs clean
$(MAKE) -C docs html $(MAKE) -C docs html
$(BROWSER) docs/_build/html/index.html # $(MAKE) -C docs latex
# $(MAKE) -C docs latexpdf
# $(BROWSER) docs/_build/html/index.html
servedocs: docs ## compile the docs watching for changes servedocs: docs ## compile the docs watching for changes
watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .
...@@ -82,4 +95,8 @@ dist: clean ## builds source and wheel package ...@@ -82,4 +95,8 @@ dist: clean ## builds source and wheel package
ls -l dist ls -l dist
install: clean ## install the package to the active Python's site-packages install: clean ## install the package to the active Python's site-packages
pip install -r requirements.txt
python setup.py install python setup.py install
gitlab_CI_docker: ## Build a docker image for CI use within gitlab
cd ./tests/CI_docker/; bash ./build_testsuite_image.sh
...@@ -26,6 +26,10 @@ ...@@ -26,6 +26,10 @@
from .sensormapgeo import SensorMapGeometryTransformer, SensorMapGeometryTransformer3D from .sensormapgeo import SensorMapGeometryTransformer, SensorMapGeometryTransformer3D
__all__ = [
'SensorMapGeometryTransformer',
'SensorMapGeometryTransformer3D'
]
__author__ = """Daniel Scheffler""" __author__ = """Daniel Scheffler"""
__email__ = 'daniel.scheffler@gfz-potsdam.de' __email__ = 'daniel.scheffler@gfz-potsdam.de'
__version__ = '0.1.0' __version__ = '0.1.0'
...@@ -22,6 +22,6 @@ ...@@ -22,6 +22,6 @@
# You should have received a copy of the GNU Lesser General Public License along # You should have received a copy of the GNU Lesser General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>. # with this program. If not, see <http://www.gnu.org/licenses/>.
__version__ = '0.0.1' __version__ = '0.1.1'
__versionalias__ = '20200106.01' __versionalias__ = '20200106.01'
__author__ = 'Daniel Scheffler' __author__ = 'Daniel Scheffler'
...@@ -37,7 +37,7 @@ version = {} ...@@ -37,7 +37,7 @@ version = {}
with open("sensormapgeo/version.py") as version_file: with open("sensormapgeo/version.py") as version_file:
exec(version_file.read(), version) exec(version_file.read(), version)
requirements = ['numpy', 'gdal', 'pyresample', 'py_tools_ds'] requirements = ['numpy', 'gdal', 'pyresample>=1.11.0', 'py_tools_ds']
setup_requirements = [] setup_requirements = []
...@@ -69,7 +69,7 @@ setup( ...@@ -69,7 +69,7 @@ setup(
setup_requires=setup_requirements, setup_requires=setup_requirements,
test_suite='tests', test_suite='tests',
tests_require=test_requirements, tests_require=test_requirements,
url='https://github.com/danschef/sensormapgeo', url='https://gitext.gfz-potsdam.de/EnMAP/sensormapgeo',
version=version['__version__'], version=version['__version__'],
zip_safe=False, zip_safe=False,
) )
#!/usr/bin/env bash
context_dir="./context"
dockerfile="sensormapgeo_ci.docker"
tag="sensormapgeo_ci:0.1.1"
gitlab_runner="sensormapgeo_gitlab_CI_runner"
echo "#### Build runner docker image"
sudo docker rmi ${tag}
sudo docker build -f ${context_dir}/${dockerfile} -m 20G -t ${tag} ${context_dir}
echo "#### Create gitlab-runner (daemon) container with tag; ${tag}"
sudo docker stop ${gitlab_runner}
sudo docker rm ${gitlab_runner}
sudo docker run -d --name ${gitlab_runner} --restart always \
-v /var/run/docker.sock:/var/run/docker.sock gitlab/gitlab-runner:latest
echo "#### Register container at gitlab, get token here https://gitext.gfz-potsdam.de/EnMAP/sensormapgeo/settings/ci_cd"
read -p "Please enter gitlab token: " token
echo ""
read -p "Please enter gitlab runner name: " runner_name
echo "New gitlab runner image will named ${gitlab_runner}"
sudo docker exec -it ${gitlab_runner} /bin/bash -c "export RUNNER_EXECUTOR=docker && gitlab-ci-multi-runner register -n \
--url 'https://gitext.gfz-potsdam.de/ci' \
--registration-token '${token}' \
--run-untagged=true \
--locked=true \
--tag-list sensormapgeo_ci_client \
--description '${runner_name}' \
--docker-image '${tag}' "
ls
name: sensormapgeo
channels: &id1
- http://conda.anaconda.org/conda-forge
dependencies:
- python=3.*.*
- numpy
- gdal
- pyresample>=1.11.0
# py_tools_ds
- pyqt
- scikit-image
- rasterio
- pyproj
- lxml
- geopandas
- ipython
- pip:
- py_tools_ds
- sphinx-argparse
- flake8
- pycodestyle
- pylint
- pydocstyle
- nose
- nose2
- nose-htmloutput
- coverage
- rednose
FROM gms_base_centos:0.3
# copy some needed stuff to /root
COPY *.yml /root/
# update the ci_env environment (that already contains all packages installed via 'docker_pyenvs' repo)
# NOTE: We update conda and Python to avoid version incompatibilities.
RUN /bin/bash -i -c "\
source /root/miniconda3/bin/activate ; \
source activate ci_env; \
conda update -n base -c defaults conda ; \
conda update -c conda-forge python ; \
conda env update -n ci_env -f /root/environment_sensormapgeo.yml"
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment