Merge branch 'feature/add_CI' into 'master'

Feature/add ci

See merge request !1

.gitignore

@@ -129,76 +129,5 @@ dmypy.json
 # Pyre type checker
 .pyre/
 
-
-### JetBrains template
-# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
-# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
-
-# User-specific stuff
-.idea/**/workspace.xml
-.idea/**/tasks.xml
-.idea/**/usage.statistics.xml
-.idea/**/dictionaries
-.idea/**/shelf
-
-# Generated files
-.idea/**/contentModel.xml
-
-# Sensitive or high-churn files
-.idea/**/dataSources/
-.idea/**/dataSources.ids
-.idea/**/dataSources.local.xml
-.idea/**/sqlDataSources.xml
-.idea/**/dynamic.xml
-.idea/**/uiDesigner.xml
-.idea/**/dbnavigator.xml
-
-# Gradle
-.idea/**/gradle.xml
-.idea/**/libraries
-
-# Gradle and Maven with auto-import
-# When using Gradle or Maven with auto-import, you should exclude module files,
-# since they will be recreated, and may cause churn.  Uncomment if using
-# auto-import.
-# .idea/artifacts
-# .idea/compiler.xml
-# .idea/modules.xml
-# .idea/*.iml
-# .idea/modules
-# *.iml
-# *.ipr
-
-# CMake
-cmake-build-*/
-
-# Mongo Explorer plugin
-.idea/**/mongoSettings.xml
-
-# File-based project format
-*.iws
-
-# IntelliJ
-out/
-
-# mpeltonen/sbt-idea plugin
-.idea_modules/
-
-# JIRA plugin
-atlassian-ide-plugin.xml
-
-# Cursive Clojure plugin
-.idea/replstate.xml
-
-# Crashlytics plugin (for Android Studio and IntelliJ)
-com_crashlytics_export_strings.xml
-crashlytics.properties
-crashlytics-build.properties
-fabric.properties
-
-# Editor-based Rest Client
-.idea/httpRequests
-
-# Android studio 3.1+ serialized cache file
-.idea/caches/build_file_checksums.ser
-
+# PyCharm settings
+.idea/

.gitlab-ci.yml

+before_script:
+  - git lfs pull
+
+
+stages:
+  - test
+  - deploy
+
+
+test_sensormapgeo:
+  stage: test
+  script:
+    - source /root/miniconda3/bin/activate ci_env
+    - export GDAL_DATA=/root/miniconda3/envs/ci_env/share/gdal
+    - export PYTHONPATH=$PYTHONPATH:/root  # /root <- directory needed later
+
+    # run nosetests
+    - make nosetests
+
+    # create the docs
+    - pip install sphinx_rtd_theme  # Read-the-docs theme for SPHINX documentation
+    - pip install sphinx-autodoc-typehints
+    - make docs
+  artifacts:
+    paths:
+    - htmlcov/
+    - docs/_build/html/
+    - nosetests.html
+    - nosetests.xml
+    when: always
+
+
+test_styles:
+  stage: test
+  script:
+    - source /root/miniconda3/bin/activate ci_env
+    - export GDAL_DATA=/root/miniconda3/envs/ci_env/share/gdal
+    - export PYTHONPATH=$PYTHONPATH:/root  # /root <- directory needed later
+    - make lint
+  artifacts:
+    paths:
+    - tests/linting/flake8.log
+    - tests/linting/pycodestyle.log
+    - tests/linting/pydocstyle.log
+    when: always
+
+
+test_sensormapgeo_install:
+  stage: test
+  script:
+    - source /root/miniconda3/bin/activate
+    - conda create -y -q --name sensormapgeo_testinstall python=3
+    - source activate sensormapgeo_testinstall
+
+    # resolve some requirements with conda
+    - conda install --yes -q -c conda-forge numpy gdal pyresample pyqt scikit-image rasterio pyproj lxml geopandas ipython
+
+    # run installer
+    - python setup.py install
+
+    # test if its importable
+    - cd ..
+    - pwd
+    - ls
+    - python -c "import sensormapgeo; print(sensormapgeo)"
+    - python -c "from sensormapgeo import SensorMapGeometryTransformer, SensorMapGeometryTransformer3D"
+  only:
+    - master
+    - feature/add_CI
+
+
+pages:  # this job must be called 'pages' to advise GitLab to upload content to GitLab Pages
+  stage: deploy
+  dependencies:
+    - test_sensormapgeo
+  script:
+      # Create the public directory
+    - rm -rf public
+    - mkdir public
+    - mkdir -p public/doc
+    - mkdir -p public/coverage
+    - mkdir -p public/nosetests_reports
+    # Copy over the docs
+    - cp -r docs/_build/html/* public/doc/
+    # Copy over the coverage reports
+    - cp -r htmlcov/* public/coverage/
+    # Copy over the nosetests reports
+    - cp nosetests.* public/nosetests_reports/
+    # Check if everything is working great
+    - ls -al public
+    - ls -al public/doc
+    - ls -al public/coverage
+    - ls -al public/nosetests_reports
+  artifacts:
+    paths:
+      - public
+    expire_in: 30 days
+  only:
+    - master
+
+
+deploy_pypi:
+  stage: deploy
+  dependencies:
+    - test_sensormapgeo
+  script:
+    - source /root/miniconda3/bin/activate
+    - source activate ci_env
+    - pip install -U twine
+    - python setup.py sdist
+    - twine upload dist/*  # requires creds as environment variables
+  only:
+    - /^v\d+\.\d+\.\d+([abc]\d*)?$/  # PEP-440 compliant version (tags)
+  except:
+    - dev

.idea/inspectionProfiles/profiles_settings.xml

-<component name="InspectionProjectProfileManager">
-  <settings>
-    <option name="USE_PROJECT_PROFILE" value="false" />
-    <version value="1.0" />
-  </settings>
-</component>
\ No newline at end of file

.idea/misc.xml

-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="JavaScriptSettings">
-    <option name="languageLevel" value="ES6" />
-  </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6.0 (~/python_GFZ/python/bin/python3.6)" project-jdk-type="Python SDK" />
-  <component name="PythonCompatibilityInspectionAdvertiser">
-    <option name="version" value="3" />
-  </component>
-</project>
\ No newline at end of file

.idea/modules.xml

-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectModuleManager">
-    <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/sensormapgeo.iml" filepath="$PROJECT_DIR$/.idea/sensormapgeo.iml" />
-    </modules>
-  </component>
-</project>
\ No newline at end of file

.idea/sensormapgeo.iml

-<?xml version="1.0" encoding="UTF-8"?>
-<module type="PYTHON_MODULE" version="4">
-  <component name="NewModuleRootManager">
-    <content url="file://$MODULE_DIR$" />
-    <orderEntry type="inheritedJdk" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-  <component name="PackageRequirementsSettings">
-    <option name="requirementsPath" value="" />
-  </component>
-  <component name="PyDocumentationSettings">
-    <option name="format" value="PLAIN" />
-    <option name="myDocStringFormat" value="Plain" />
-  </component>
-  <component name="TestRunnerService">
-    <option name="PROJECT_TEST_RUNNER" value="Nosetests" />
-  </component>
-</module>
\ No newline at end of file

Makefile

@@ -42,13 +42,17 @@ clean-pyc: ## remove Python file artifacts
 	find . -name '__pycache__' -exec rm -fr {} +
 
 clean-test: ## remove test and coverage artifacts
+	coverage erase
 	rm -fr .tox/
 	rm -f .coverage
 	rm -fr htmlcov/
-	rm -fr .pytest_cache
+	rm -fr nosetests.html
+	rm -fr nosetests.xml
 
 lint: ## check style with flake8
-	flake8 sensormapgeo tests
+	flake8 --max-line-length=120 sensormapgeo tests > ./tests/linting/flake8.log
+	pycodestyle sensormapgeo --exclude="*.ipynb,*.ipynb*" --max-line-length=120 > ./tests/linting/pycodestyle.log
+	-pydocstyle sensormapgeo > ./tests/linting/pydocstyle.log
 
 test: ## run tests quickly with the default Python
 	python setup.py test
@@ -56,11 +60,18 @@ test: ## run tests quickly with the default Python
 test-all: ## run tests on every Python version with tox
 	tox
 
-coverage: ## check code coverage quickly with the default Python
+coverage: clean-test ## check code coverage quickly with the default Python
 	coverage run --source sensormapgeo setup.py test
 	coverage report -m
 	coverage html
-	$(BROWSER) htmlcov/index.html
+	# $(BROWSER) htmlcov/index.html
+
+nosetests: clean-test ## Runs nosetests with coverage, xUnit and nose-html-output
+	## - puts the coverage results in the folder 'htmlcov'
+	## - generates 'nosetests.html' (--with-html)
+	## - generates 'nosetests.xml' (--with-xunit) which is currently not visualizable by GitLab
+	nosetests -vv --with-coverage --cover-package=sensormapgeo --cover-erase --cover-html --cover-html-dir=htmlcov \
+		--with-html --with-xunit --rednose --force-color
 
 docs: ## generate Sphinx HTML documentation, including API docs
 	rm -f docs/sensormapgeo.rst
@@ -68,7 +79,9 @@ docs: ## generate Sphinx HTML documentation, including API docs
 	sphinx-apidoc -o docs/ sensormapgeo
 	$(MAKE) -C docs clean
 	$(MAKE) -C docs html
-	$(BROWSER) docs/_build/html/index.html
+#	$(MAKE) -C docs latex
+#	$(MAKE) -C docs latexpdf
+	# $(BROWSER) docs/_build/html/index.html
 
 servedocs: docs ## compile the docs watching for changes
 	watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .
@@ -82,4 +95,8 @@ dist: clean ## builds source and wheel package
 	ls -l dist
 
 install: clean ## install the package to the active Python's site-packages
+	pip install -r requirements.txt
 	python setup.py install
+
+gitlab_CI_docker:  ## Build a docker image for CI use within gitlab
+	cd ./tests/CI_docker/; bash ./build_testsuite_image.sh

sensormapgeo/__init__.py

@@ -26,6 +26,10 @@
 
 from .sensormapgeo import SensorMapGeometryTransformer, SensorMapGeometryTransformer3D
 
+__all__ = [
+    'SensorMapGeometryTransformer',
+    'SensorMapGeometryTransformer3D'
+]
 __author__ = """Daniel Scheffler"""
 __email__ = 'daniel.scheffler@gfz-potsdam.de'
 __version__ = '0.1.0'

sensormapgeo/version.py

@@ -22,6 +22,6 @@
 # You should have received a copy of the GNU Lesser General Public License along
 # with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-__version__ = '0.0.1'
+__version__ = '0.1.1'
 __versionalias__ = '20200106.01'
 __author__ = 'Daniel Scheffler'

setup.py

@@ -37,7 +37,7 @@ version = {}
 with open("sensormapgeo/version.py") as version_file:
     exec(version_file.read(), version)
 
-requirements = ['numpy', 'gdal', 'pyresample', 'py_tools_ds']
+requirements = ['numpy', 'gdal', 'pyresample>=1.11.0', 'py_tools_ds']
 
 setup_requirements = []
 
@@ -69,7 +69,7 @@ setup(
     setup_requires=setup_requirements,
     test_suite='tests',
     tests_require=test_requirements,
-    url='https://github.com/danschef/sensormapgeo',
+    url='https://gitext.gfz-potsdam.de/EnMAP/sensormapgeo',
     version=version['__version__'],
     zip_safe=False,
 )

tests/CI_docker/build_testsuite_image.sh

+#!/usr/bin/env bash
+
+context_dir="./context"
+dockerfile="sensormapgeo_ci.docker"
+tag="sensormapgeo_ci:0.1.1"
+gitlab_runner="sensormapgeo_gitlab_CI_runner"
+
+echo "#### Build runner docker image"
+sudo docker rmi ${tag}
+sudo docker build -f ${context_dir}/${dockerfile} -m 20G -t ${tag} ${context_dir}
+
+echo "#### Create gitlab-runner (daemon) container with tag; ${tag}"
+sudo docker stop ${gitlab_runner}
+sudo docker rm ${gitlab_runner}
+sudo docker run -d --name ${gitlab_runner} --restart always \
+-v /var/run/docker.sock:/var/run/docker.sock gitlab/gitlab-runner:latest
+
+echo "#### Register container at gitlab, get token here https://gitext.gfz-potsdam.de/EnMAP/sensormapgeo/settings/ci_cd"
+read -p "Please enter gitlab token: " token
+echo ""
+read -p "Please enter gitlab runner name: " runner_name
+echo "New gitlab runner image will named  ${gitlab_runner}"
+sudo docker exec -it ${gitlab_runner} /bin/bash -c "export RUNNER_EXECUTOR=docker && gitlab-ci-multi-runner register -n \
+  --url 'https://gitext.gfz-potsdam.de/ci' \
+  --registration-token '${token}' \
+  --run-untagged=true \
+  --locked=true \
+  --tag-list  sensormapgeo_ci_client \
+  --description '${runner_name}' \
+  --docker-image '${tag}' "
+ls

tests/CI_docker/context/environment_sensormapgeo.yml

+name: sensormapgeo
+
+channels: &id1
+  - http://conda.anaconda.org/conda-forge
+
+dependencies:
+  - python=3.*.*
+  - numpy
+  - gdal
+  - pyresample>=1.11.0
+
+  # py_tools_ds
+  - pyqt
+  - scikit-image
+  - rasterio
+  - pyproj
+  - lxml
+  - geopandas
+  - ipython
+
+  - pip:
+    - py_tools_ds
+    - sphinx-argparse
+    - flake8
+    - pycodestyle
+    - pylint
+    - pydocstyle
+    - nose
+    - nose2
+    - nose-htmloutput
+    - coverage
+    - rednose

tests/CI_docker/context/sensormapgeo_ci.docker

+FROM gms_base_centos:0.3
+
+# copy some needed stuff to /root
+COPY *.yml /root/
+
+# update the ci_env environment (that already contains all packages installed via 'docker_pyenvs' repo)
+# NOTE: We update conda and Python to avoid version incompatibilities.
+RUN /bin/bash -i -c "\
+    source /root/miniconda3/bin/activate ; \
+    source activate ci_env; \
+    conda update -n base -c defaults conda ; \
+    conda update -c conda-forge python ; \
+    conda env update -n ci_env -f /root/environment_sensormapgeo.yml"