Commit 415485fd authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Merge branch 'master' into feature/implement_operators

# Conflicts:
#	geoarray/baseclasses.py
parents 8829a837 bf03bd02
Pipeline #19577 failed with stage
in 1 minute and 34 seconds
...@@ -2,12 +2,6 @@ before_script: ...@@ -2,12 +2,6 @@ before_script:
- git lfs pull - git lfs pull
# Advise GitLab that these environment vars should be loaded from the Variables config.
variables:
PYPI_USER: SECURE
PYPI_PASSWORD: SECURE
stages: stages:
- test - test
- deploy - deploy
...@@ -17,12 +11,13 @@ stages: ...@@ -17,12 +11,13 @@ stages:
test_geoarray: test_geoarray:
stage: test stage: test
script: script:
- source /root/miniconda3/bin/activate - source /root/miniconda3/bin/activate ci_env
- export GDAL_DATA=/root/miniconda3/share/gdal
- export PYTHONPATH=$PYTHONPATH:/root # /root <- directory needed later # update py_tools_ds
- export LD_LIBRARY_PATH=/root/anaconda3/lib:$LD_LIBRARY_PATH # /root <- directory needed later - pip install -U py_tools_ds -q
# update in-house dependencies
- pip install py_tools_ds>=0.9.0 - pip install parameterized -q # TODO remove as soon as CI runner is rebuilt
# run tests # run tests
- make nosetests - make nosetests
- make docs - make docs
...@@ -38,10 +33,7 @@ test_geoarray: ...@@ -38,10 +33,7 @@ test_geoarray:
test_styles: test_styles:
stage: test stage: test
script: script:
- source /root/miniconda3/bin/activate - source /root/miniconda3/bin/activate ci_env
- export GDAL_DATA=/root/miniconda3/share/gdal
- export PYTHONPATH=$PYTHONPATH:/root # /root <- directory needed later
- pip install "pycodestyle>=2.0.0,!=2.4.0" # TODO remove as soon as docker runner is recreated
- make lint - make lint
artifacts: artifacts:
paths: paths:
...@@ -51,17 +43,37 @@ test_styles: ...@@ -51,17 +43,37 @@ test_styles:
when: always when: always
test_urls:
stage: test
script:
- source /root/miniconda3/bin/activate ci_env
- pip install -U urlchecker
- make urlcheck
when: always
test_geoarray_install: test_geoarray_install:
stage: test stage: test
script: script:
- source /root/miniconda3/bin/activate - source /root/miniconda3/bin/activate
- export GDAL_DATA=/root/miniconda3/share/gdal
- conda create -y -q --name geoarray python=3 # update conda and python
- source activate geoarray - conda update -n base -c conda-forge conda
# resolve some requirements with conda # - conda update -c conda-forge python
- conda install --yes -q -c conda-forge numpy scikit-image matplotlib geopandas gdal rasterio pyproj basemap shapely
# create test environment
- conda create -y -q -c conda-forge --name geoarray_testinstall python=3
- conda activate geoarray_testinstall
# avoid package incompatibilities due to usage of wrong channels
# - conda config --set channel_priority strict # otherwise gdal or libgdal may be installed from defaults channel
# resolve some requirements with conda
- conda install --yes -q -c conda-forge numpy scikit-image matplotlib geopandas gdal pyproj cartopy shapely
# run installer # run installer
- python setup.py install - python setup.py install
# test if its importable # test if its importable
- cd .. - cd ..
- pwd - pwd
...@@ -83,12 +95,16 @@ pages: # this job must be called 'pages' to advise GitLab to upload content to ...@@ -83,12 +95,16 @@ pages: # this job must be called 'pages' to advise GitLab to upload content to
- mkdir -p public/doc - mkdir -p public/doc
- mkdir -p public/coverage - mkdir -p public/coverage
- mkdir -p public/nosetests_reports - mkdir -p public/nosetests_reports
# Copy over the docs # Copy over the docs
- cp -r docs/_build/html/* public/doc/ - cp -r docs/_build/html/* public/doc/
# Copy over the coverage reports # Copy over the coverage reports
- cp -r htmlcov/* public/coverage/ - cp -r htmlcov/* public/coverage/
# Copy over the nosetests reports # Copy over the nosetests reports
- cp nosetests.* public/nosetests_reports/ - cp nosetests.* public/nosetests_reports/
# Check if everything is working great # Check if everything is working great
- ls -al public - ls -al public
- ls -al public/doc - ls -al public/doc
...@@ -106,22 +122,12 @@ deploy_pypi: ...@@ -106,22 +122,12 @@ deploy_pypi:
stage: deploy stage: deploy
dependencies: dependencies:
- test_geoarray - test_geoarray
script: # Configure the PyPI credentials, then push the package, and cleanup the creds. script:
- source /root/miniconda3/bin/activate - source /root/miniconda3/bin/activate ci_env
- printf "[distutils]\nindex-servers =\n pypi\n\n" >> ~/.pypirc - pip install -U twine
- printf "[pypi]\n""repository:"" https://upload.pypi.org/legacy/\n" >> ~/.pypirc - python setup.py sdist
- printf "username= ${PYPI_USER}\n" >> ~/.pypirc - twine upload dist/* # requires creds as environment variables
- printf "password= ${PYPI_PASSWORD}\n" >> ~/.pypirc
- python setup.py check sdist upload -r pypi # This will fail if your creds are bad.
- echo "" > ~/.pypirc && rm ~/.pypirc # If the above fails, this won't run.
only: only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags) - /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except: except:
- dev - dev
cleanup_pypirc:
stage: cleanup
when: always # this is important; run even if preceding stages failed.
script:
- rm -vf ~/.pypirc # we don't want to leave these around, but GitLab may clean up anyway.
# Config file for automatic testing at travis-ci.org
# This file will be regenerated if you run travis_pypi_setup.py
language: python
python: 3.5
env:
- TOXENV=py35
- TOXENV=py34
- TOXENV=py33
- TOXENV=py27
- TOXENV=py26
- TOXENV=pypy
# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
install: pip install -U tox
# command to run tests, e.g. python setup.py test
script: tox -e ${TOXENV}
...@@ -15,7 +15,7 @@ Types of Contributions ...@@ -15,7 +15,7 @@ Types of Contributions
Report Bugs Report Bugs
~~~~~~~~~~~ ~~~~~~~~~~~
Report bugs at https://gitext.gfz-potsdam.de/danschef/geoarray/issues. Report bugs at https://git.gfz-potsdam.de/danschef/geoarray/issues.
If you are reporting a bug, please include: If you are reporting a bug, please include:
...@@ -45,7 +45,7 @@ articles, and such. ...@@ -45,7 +45,7 @@ articles, and such.
Submit Feedback Submit Feedback
~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~
The best way to send feedback is to file an issue at https://gitext.gfz-potsdam.de/danschef/geoarray/issues. The best way to send feedback is to file an issue at https://git.gfz-potsdam.de/danschef/geoarray/issues.
If you are proposing a feature: If you are proposing a feature:
...@@ -62,7 +62,7 @@ Ready to contribute? Here's how to set up `geoarray` for local development. ...@@ -62,7 +62,7 @@ Ready to contribute? Here's how to set up `geoarray` for local development.
1. Fork the `geoarray` repo on GitHub. 1. Fork the `geoarray` repo on GitHub.
2. Clone your fork locally:: 2. Clone your fork locally::
$ git clone https://gitext.gfz-potsdam.de/danschef/geoarray.git $ git clone https://git.gfz-potsdam.de/danschef/geoarray.git
3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: 3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development::
......
This diff is collapsed.
...@@ -6,7 +6,7 @@ include HISTORY.rst ...@@ -6,7 +6,7 @@ include HISTORY.rst
include LICENSE include LICENSE
include README.rst include README.rst
recursive-include tests * recursive-exclude tests *
recursive-exclude * __pycache__ recursive-exclude * __pycache__
recursive-exclude * *.py[co] recursive-exclude * *.py[co]
......
...@@ -42,9 +42,10 @@ clean-pyc: ## remove Python file artifacts ...@@ -42,9 +42,10 @@ clean-pyc: ## remove Python file artifacts
find . -name '__pycache__' -exec rm -fr {} + find . -name '__pycache__' -exec rm -fr {} +
clean-test: ## remove test and coverage artifacts clean-test: ## remove test and coverage artifacts
coverage erase ## don't call coverage erase here because make install calls make clean which calls make clean-test
## -> since make install should run without the test requirements we can't use coverage erase here
rm -fr .tox/ rm -fr .tox/
rm -f .coverage rm -fr .coverage.*
rm -fr htmlcov/ rm -fr htmlcov/
rm -fr nosetests.html rm -fr nosetests.html
rm -fr nosetests.xml rm -fr nosetests.xml
...@@ -54,6 +55,9 @@ lint: ## check style with flake8 ...@@ -54,6 +55,9 @@ lint: ## check style with flake8
pycodestyle geoarray --exclude="*.ipynb,*.ipynb*" --max-line-length=120 > ./tests/linting/pycodestyle.log pycodestyle geoarray --exclude="*.ipynb,*.ipynb*" --max-line-length=120 > ./tests/linting/pycodestyle.log
-pydocstyle geoarray > ./tests/linting/pydocstyle.log -pydocstyle geoarray > ./tests/linting/pydocstyle.log
urlcheck: ## check for dead URLs
urlchecker check . --file-types .py,.rst,.md,.json
test: ## run tests quickly with the default Python test: ## run tests quickly with the default Python
python setup.py test python setup.py test
...@@ -86,9 +90,8 @@ docs: ## generate Sphinx HTML documentation, including API docs ...@@ -86,9 +90,8 @@ docs: ## generate Sphinx HTML documentation, including API docs
servedocs: docs ## compile the docs watching for changes servedocs: docs ## compile the docs watching for changes
watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .
release: clean ## package and upload a release release: dist ## package and upload a release
python setup.py sdist upload twine upload dist/*
python setup.py bdist_wheel upload
dist: clean ## builds source and wheel package dist: clean ## builds source and wheel package
python setup.py sdist python setup.py sdist
......
...@@ -5,76 +5,102 @@ geoarray ...@@ -5,76 +5,102 @@ geoarray
Fast Python interface for geodata - either on disk or in memory. Fast Python interface for geodata - either on disk or in memory.
The geoarray package facilitates reading and writing of all GDAL compatible image file formats
and provides functions for geospatial processing.
* Free software: GNU General Public License v3
* Documentation: http://danschef.gitext.gfz-potsdam.de/geoarray/doc/ * Free software: GNU General Public License v3 or later (GPLv3+)
* Documentation: https://danschef.git-pages.gfz-potsdam.de/geoarray/doc/
Status Status
------ ------
.. .. image:: https://img.shields.io/travis/danschef/geoarray.svg .. image:: https://git.gfz-potsdam.de/danschef/geoarray/badges/master/pipeline.svg
:target: https://travis-ci.org/danschef/geoarray :target: https://git.gfz-potsdam.de/danschef/geoarray/commits/master
.. image:: https://git.gfz-potsdam.de/danschef/geoarray/badges/master/coverage.svg
.. .. image:: https://readthedocs.org/projects/geoarray/badge/?version=latest :target: https://danschef.git-pages.gfz-potsdam.de/geoarray/coverage/
:target: https://geoarray.readthedocs.io/en/latest/?badge=latest
:alt: Documentation Status
.. .. image:: https://pyup.io/repos/github/danschef/geoarray/shield.svg
:target: https://pyup.io/repos/github/danschef/geoarray/
:alt: Updates
.. image:: https://gitext.gfz-potsdam.de/danschef/geoarray/badges/master/build.svg
:target: https://gitext.gfz-potsdam.de/danschef/geoarray/commits/master
.. image:: https://gitext.gfz-potsdam.de/danschef/geoarray/badges/master/coverage.svg
:target: http://danschef.gitext.gfz-potsdam.de/geoarray/coverage/
.. image:: https://img.shields.io/pypi/v/geoarray.svg .. image:: https://img.shields.io/pypi/v/geoarray.svg
:target: https://pypi.python.org/pypi/geoarray :target: https://pypi.python.org/pypi/geoarray
.. image:: https://img.shields.io/conda/vn/conda-forge/geoarray.svg
:target: https://anaconda.org/conda-forge/geoarray
.. image:: https://img.shields.io/pypi/l/geoarray.svg .. image:: https://img.shields.io/pypi/l/geoarray.svg
:target: https://gitext.gfz-potsdam.de/danschef/geoarray/blob/master/LICENSE :target: https://git.gfz-potsdam.de/danschef/geoarray/blob/master/LICENSE
.. image:: https://img.shields.io/pypi/pyversions/geoarray.svg .. image:: https://img.shields.io/pypi/pyversions/geoarray.svg
:target: https://img.shields.io/pypi/pyversions/geoarray.svg :target: https://img.shields.io/pypi/pyversions/geoarray.svg
.. image:: https://img.shields.io/pypi/dm/geoarray.svg
:target: https://pypi.python.org/pypi/geoarray
See also the latest coverage_ report and the nosetests_ HTML report. See also the latest coverage_ report and the nosetests_ HTML report.
Features Features and usage
-------- ------------------
* TODO * There is an example notebook that shows how to use geoarray: here_.
Installation Installation
------------ ------------
geoarray depends on some open source packages which are usually installed without problems by the automatic install
routine. However, for some projects, we strongly recommend resolving the dependency before the automatic installer
is run. This approach avoids problems with conflicting versions of the same software.
Using conda_, the recommended approach is:
.. code-block:: console Using Anaconda or Miniconda (recommended)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Using conda_ (latest version recommended), geoarray is installed as follows:
1. Create virtual environment for geoarray (optional but recommended):
.. code-block:: bash
$ conda create -c conda-forge --name geoarray python=3
$ conda activate geoarray
2. Then install geoarray itself:
.. code-block:: bash
$ conda install -c conda-forge geoarray
This is the preferred method to install geoarray, as it always installs the most recent stable release and
automatically resolves all the dependencies.
Using pip (not recommended)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
# create virtual environment for geoarray, this is optional There is also a `pip`_ installer for geoarray. However, please note that geoarray depends on some
conda create -y -q --name geoarray python=3 open source packages that may cause problems when installed with pip. Therefore, we strongly recommend
source activate geoarray to resolve the following dependencies before the pip installer is run:
conda install -y -q -c conda-forge numpy gdal scikit-image matplotlib pandas rasterio pyproj basemap shapely geopandas
conda install --yes -c ioam bokeh holoviews # optional
* cartopy
* gdal >=2.1.0
* geopandas
* holoviews # optional, in case you want to use interactive plotting
* matplotlib
* numpy
* pandas
* pyproj >2.2.0
* scikit-image
* shapely
To install geoarray, use the pip installer: Then, the pip installer can be run by:
.. code-block:: console .. code-block:: bash
pip install geoarray $ pip install geoarray
If you don't have `pip`_ installed, this `Python installation guide`_ can guide you through the process.
* Or clone the repository via GIT and update the PATH environment variable:
.. code-block:: console History / Changelog
-------------------
cd /your/installation/folder You can find the protocol of recent changes in the geoarray package
git clone https://gitext.gfz-potsdam.de/danschef/geoarray.git `here <https://git.gfz-potsdam.de/danschef/geoarray/-/blob/master/HISTORY.rst>`__.
PATH=$PATH:~/path/to/your/installation/folder/geoarray
Credits Credits
...@@ -84,6 +110,9 @@ This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypack ...@@ -84,6 +110,9 @@ This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypack
.. _Cookiecutter: https://github.com/audreyr/cookiecutter .. _Cookiecutter: https://github.com/audreyr/cookiecutter
.. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage .. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage
.. _coverage: http://danschef.gitext.gfz-potsdam.de/geoarray/coverage/ .. _coverage: https://danschef.git-pages.gfz-potsdam.de/geoarray/coverage/
.. _nosetests: http://danschef.gitext.gfz-potsdam.de/geoarray/nosetests_reports/nosetests.html .. _nosetests: https://danschef.git-pages.gfz-potsdam.de/geoarray/nosetests_reports/nosetests.html
.. _conda: https://conda.io/docs/ .. _conda: https://conda.io/docs/
.. _here: https://git.gfz-potsdam.de/danschef/geoarray/-/blob/master/examples/notebooks/features_and_usage.ipynb
.. _pip: https://pip.pypa.io
.. _Python installation guide: http://docs.python-guide.org/en/latest/starting/installation/
...@@ -32,13 +32,13 @@ You can either clone the public repository: ...@@ -32,13 +32,13 @@ You can either clone the public repository:
.. code-block:: console .. code-block:: console
$ git clone https://gitext.gfz-potsdam.de/danschef/geoarray.git $ git clone https://git.gfz-potsdam.de/danschef/geoarray.git
Or download the `tarball`_: Or download the `tarball`_:
.. code-block:: console .. code-block:: console
$ curl -OL https://gitext.gfz-potsdam.de/danschef/geoarray/repository/archive.tar.gz?ref=master $ curl -OL https://git.gfz-potsdam.de/danschef/geoarray/repository/archive.tar.gz?ref=master
Once you have a copy of the source, you can install it with: Once you have a copy of the source, you can install it with:
...@@ -47,5 +47,5 @@ Once you have a copy of the source, you can install it with: ...@@ -47,5 +47,5 @@ Once you have a copy of the source, you can install it with:
$ python setup.py install $ python setup.py install
.. _GitLab repo: https://gitext.gfz-potsdam.de/danschef/geoarray .. _GitLab repo: https://git.gfz-potsdam.de/danschef/geoarray
.. _tarball: https://gitext.gfz-potsdam.de/danschef/geoarray/repository/archive.tar.gz?ref=master .. _tarball: https://git.gfz-potsdam.de/danschef/geoarray/repository/archive.tar.gz?ref=master
This diff is collapsed.
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# geoarray, A fast Python interface for image geodata - either on disk or in memory.
#
# Copyright (C) 2019 Daniel Scheffler (GFZ Potsdam, daniel.scheffler@gfz-potsdam.de)
#
# This software was developed within the context of the GeoMultiSens project funded
# by the German Federal Ministry of Education and Research
# (project grant code: 01 IS 14 010 A-C).
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
import os import os
if 'MPLBACKEND' not in os.environ: if 'MPLBACKEND' not in os.environ:
os.environ['MPLBACKEND'] = 'Agg' os.environ['MPLBACKEND'] = 'Agg'
......
This diff is collapsed.
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# geoarray, A fast Python interface for image geodata - either on disk or in memory.
#
# Copyright (C) 2019 Daniel Scheffler (GFZ Potsdam, daniel.scheffler@gfz-potsdam.de)
#
# This software was developed within the context of the GeoMultiSens project funded
# by the German Federal Ministry of Education and Research
# (project grant code: 01 IS 14 010 A-C).
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
import numpy as np import numpy as np
# internal imports # internal imports
...@@ -17,7 +38,7 @@ class BadDataMask(GeoArray): ...@@ -17,7 +38,7 @@ class BadDataMask(GeoArray):
if self.is_inmem: if self.is_inmem:
# validate input data - before converting to bool # validate input data - before converting to bool
self._validate_array_values(self.arr) self._validate_array_values(self.arr)
self.arr = self.arr.astype(np.bool) self.arr = self.arr.astype(bool)
# del self._mask_baddata, self.mask_baddata # TODO delete property (requires deleter) # del self._mask_baddata, self.mask_baddata # TODO delete property (requires deleter)
...@@ -29,7 +50,7 @@ class BadDataMask(GeoArray): ...@@ -29,7 +50,7 @@ class BadDataMask(GeoArray):
def arr(self, ndarray): def arr(self, ndarray):
assert isinstance(ndarray, np.ndarray), "'arr' can only be set to a numpy array!" assert isinstance(ndarray, np.ndarray), "'arr' can only be set to a numpy array!"
self._validate_array_values(ndarray) self._validate_array_values(ndarray)
self._arr = ndarray.astype(np.bool) self._arr = ndarray.astype(bool)
def _validate_array_values(self, maskarray): def _validate_array_values(self, maskarray):
pixelVals_in_mask = sorted(list(np.unique(maskarray))) pixelVals_in_mask = sorted(list(np.unique(maskarray)))
...@@ -50,7 +71,7 @@ class NoDataMask(GeoArray): ...@@ -50,7 +71,7 @@ class NoDataMask(GeoArray):
if self.is_inmem: if self.is_inmem:
# validate input data - before converting to bool # validate input data - before converting to bool
self._validate_array_values(self.arr) self._validate_array_values(self.arr)
self.arr = self.arr.astype(np.bool) self.arr = self.arr.astype(bool)
# del self._mask_nodata, self.mask_nodata # TODO delete property (requires deleter) # del self._mask_nodata, self.mask_nodata # TODO delete property (requires deleter)
# TODO disk-mode: init must check the numbers of bands, and ideally also the pixel values in mask # TODO disk-mode: init must check the numbers of bands, and ideally also the pixel values in mask
...@@ -63,7 +84,7 @@ class NoDataMask(GeoArray): ...@@ -63,7 +84,7 @@ class NoDataMask(GeoArray):
def arr(self, ndarray): def arr(self, ndarray):
assert isinstance(ndarray, np.ndarray), "'arr' can only be set to a numpy array!" assert isinstance(ndarray, np.ndarray), "'arr' can only be set to a numpy array!"
self._validate_array_values(ndarray) self._validate_array_values(ndarray)
self._arr = ndarray.astype(np.bool) self._arr = ndarray.astype(bool)
def _validate_array_values(self, maskarray): def _validate_array_values(self, maskarray):
pixelVals_in_mask = sorted(list(np.unique(maskarray))) pixelVals_in_mask = sorted(list(np.unique(maskarray)))
......
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# geoarray, A fast Python interface for image geodata - either on disk or in memory.
#
# Copyright (C) 2019 Daniel Scheffler (GFZ Potsdam, daniel.scheffler@gfz-potsdam.de)
#
# This software was developed within the context of the GeoMultiSens project funded
# by the German Federal Ministry of Education and Research
# (project grant code: 01 IS 14 010 A-C).
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT