Commit 415485fd authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Merge branch 'master' into feature/implement_operators

# Conflicts:
#	geoarray/baseclasses.py
parents 8829a837 bf03bd02
Pipeline #19577 failed with stage
in 1 minute and 34 seconds
......@@ -2,12 +2,6 @@ before_script:
- git lfs pull
# Advise GitLab that these environment vars should be loaded from the Variables config.
variables:
PYPI_USER: SECURE
PYPI_PASSWORD: SECURE
stages:
- test
- deploy
......@@ -17,12 +11,13 @@ stages:
test_geoarray:
stage: test
script:
- source /root/miniconda3/bin/activate
- export GDAL_DATA=/root/miniconda3/share/gdal
- export PYTHONPATH=$PYTHONPATH:/root # /root <- directory needed later
- export LD_LIBRARY_PATH=/root/anaconda3/lib:$LD_LIBRARY_PATH # /root <- directory needed later
# update in-house dependencies
- pip install py_tools_ds>=0.9.0
- source /root/miniconda3/bin/activate ci_env
# update py_tools_ds
- pip install -U py_tools_ds -q
- pip install parameterized -q # TODO remove as soon as CI runner is rebuilt
# run tests
- make nosetests
- make docs
......@@ -38,10 +33,7 @@ test_geoarray:
test_styles:
stage: test
script:
- source /root/miniconda3/bin/activate
- export GDAL_DATA=/root/miniconda3/share/gdal
- export PYTHONPATH=$PYTHONPATH:/root # /root <- directory needed later
- pip install "pycodestyle>=2.0.0,!=2.4.0" # TODO remove as soon as docker runner is recreated
- source /root/miniconda3/bin/activate ci_env
- make lint
artifacts:
paths:
......@@ -51,17 +43,37 @@ test_styles:
when: always
test_urls:
stage: test
script:
- source /root/miniconda3/bin/activate ci_env
- pip install -U urlchecker
- make urlcheck
when: always
test_geoarray_install:
stage: test
script:
- source /root/miniconda3/bin/activate
- export GDAL_DATA=/root/miniconda3/share/gdal
- conda create -y -q --name geoarray python=3
- source activate geoarray
# resolve some requirements with conda
- conda install --yes -q -c conda-forge numpy scikit-image matplotlib geopandas gdal rasterio pyproj basemap shapely
# update conda and python
- conda update -n base -c conda-forge conda
# - conda update -c conda-forge python
# create test environment
- conda create -y -q -c conda-forge --name geoarray_testinstall python=3
- conda activate geoarray_testinstall
# avoid package incompatibilities due to usage of wrong channels
# - conda config --set channel_priority strict # otherwise gdal or libgdal may be installed from defaults channel
# resolve some requirements with conda
- conda install --yes -q -c conda-forge numpy scikit-image matplotlib geopandas gdal pyproj cartopy shapely
# run installer
- python setup.py install
# test if its importable
- cd ..
- pwd
......@@ -83,12 +95,16 @@ pages: # this job must be called 'pages' to advise GitLab to upload content to
- mkdir -p public/doc
- mkdir -p public/coverage
- mkdir -p public/nosetests_reports
# Copy over the docs
- cp -r docs/_build/html/* public/doc/
# Copy over the coverage reports
- cp -r htmlcov/* public/coverage/
# Copy over the nosetests reports
- cp nosetests.* public/nosetests_reports/
# Check if everything is working great
- ls -al public
- ls -al public/doc
......@@ -106,22 +122,12 @@ deploy_pypi:
stage: deploy
dependencies:
- test_geoarray
script: # Configure the PyPI credentials, then push the package, and cleanup the creds.
- source /root/miniconda3/bin/activate
- printf "[distutils]\nindex-servers =\n pypi\n\n" >> ~/.pypirc
- printf "[pypi]\n""repository:"" https://upload.pypi.org/legacy/\n" >> ~/.pypirc
- printf "username= ${PYPI_USER}\n" >> ~/.pypirc
- printf "password= ${PYPI_PASSWORD}\n" >> ~/.pypirc
- python setup.py check sdist upload -r pypi # This will fail if your creds are bad.
- echo "" > ~/.pypirc && rm ~/.pypirc # If the above fails, this won't run.
script:
- source /root/miniconda3/bin/activate ci_env
- pip install -U twine
- python setup.py sdist
- twine upload dist/* # requires creds as environment variables
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- dev
cleanup_pypirc:
stage: cleanup
when: always # this is important; run even if preceding stages failed.
script:
- rm -vf ~/.pypirc # we don't want to leave these around, but GitLab may clean up anyway.
# Config file for automatic testing at travis-ci.org
# This file will be regenerated if you run travis_pypi_setup.py
language: python
python: 3.5
env:
- TOXENV=py35
- TOXENV=py34
- TOXENV=py33
- TOXENV=py27
- TOXENV=py26
- TOXENV=pypy
# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
install: pip install -U tox
# command to run tests, e.g. python setup.py test
script: tox -e ${TOXENV}
......@@ -15,7 +15,7 @@ Types of Contributions
Report Bugs
~~~~~~~~~~~
Report bugs at https://gitext.gfz-potsdam.de/danschef/geoarray/issues.
Report bugs at https://git.gfz-potsdam.de/danschef/geoarray/issues.
If you are reporting a bug, please include:
......@@ -45,7 +45,7 @@ articles, and such.
Submit Feedback
~~~~~~~~~~~~~~~
The best way to send feedback is to file an issue at https://gitext.gfz-potsdam.de/danschef/geoarray/issues.
The best way to send feedback is to file an issue at https://git.gfz-potsdam.de/danschef/geoarray/issues.
If you are proposing a feature:
......@@ -62,7 +62,7 @@ Ready to contribute? Here's how to set up `geoarray` for local development.
1. Fork the `geoarray` repo on GitHub.
2. Clone your fork locally::
$ git clone https://gitext.gfz-potsdam.de/danschef/geoarray.git
$ git clone https://git.gfz-potsdam.de/danschef/geoarray.git
3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development::
......
This diff is collapsed.
......@@ -6,7 +6,7 @@ include HISTORY.rst
include LICENSE
include README.rst
recursive-include tests *
recursive-exclude tests *
recursive-exclude * __pycache__
recursive-exclude * *.py[co]
......
......@@ -42,9 +42,10 @@ clean-pyc: ## remove Python file artifacts
find . -name '__pycache__' -exec rm -fr {} +
clean-test: ## remove test and coverage artifacts
coverage erase
## don't call coverage erase here because make install calls make clean which calls make clean-test
## -> since make install should run without the test requirements we can't use coverage erase here
rm -fr .tox/
rm -f .coverage
rm -fr .coverage.*
rm -fr htmlcov/
rm -fr nosetests.html
rm -fr nosetests.xml
......@@ -54,6 +55,9 @@ lint: ## check style with flake8
pycodestyle geoarray --exclude="*.ipynb,*.ipynb*" --max-line-length=120 > ./tests/linting/pycodestyle.log
-pydocstyle geoarray > ./tests/linting/pydocstyle.log
urlcheck: ## check for dead URLs
urlchecker check . --file-types .py,.rst,.md,.json
test: ## run tests quickly with the default Python
python setup.py test
......@@ -86,9 +90,8 @@ docs: ## generate Sphinx HTML documentation, including API docs
servedocs: docs ## compile the docs watching for changes
watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .
release: clean ## package and upload a release
python setup.py sdist upload
python setup.py bdist_wheel upload
release: dist ## package and upload a release
twine upload dist/*
dist: clean ## builds source and wheel package
python setup.py sdist
......
......@@ -5,76 +5,102 @@ geoarray
Fast Python interface for geodata - either on disk or in memory.
The geoarray package facilitates reading and writing of all GDAL compatible image file formats
and provides functions for geospatial processing.
* Free software: GNU General Public License v3
* Documentation: http://danschef.gitext.gfz-potsdam.de/geoarray/doc/
* Free software: GNU General Public License v3 or later (GPLv3+)
* Documentation: https://danschef.git-pages.gfz-potsdam.de/geoarray/doc/
Status
------
.. .. image:: https://img.shields.io/travis/danschef/geoarray.svg
:target: https://travis-ci.org/danschef/geoarray
.. .. image:: https://readthedocs.org/projects/geoarray/badge/?version=latest
:target: https://geoarray.readthedocs.io/en/latest/?badge=latest
:alt: Documentation Status
.. .. image:: https://pyup.io/repos/github/danschef/geoarray/shield.svg
:target: https://pyup.io/repos/github/danschef/geoarray/
:alt: Updates
.. image:: https://gitext.gfz-potsdam.de/danschef/geoarray/badges/master/build.svg
:target: https://gitext.gfz-potsdam.de/danschef/geoarray/commits/master
.. image:: https://gitext.gfz-potsdam.de/danschef/geoarray/badges/master/coverage.svg
:target: http://danschef.gitext.gfz-potsdam.de/geoarray/coverage/
.. image:: https://git.gfz-potsdam.de/danschef/geoarray/badges/master/pipeline.svg
:target: https://git.gfz-potsdam.de/danschef/geoarray/commits/master
.. image:: https://git.gfz-potsdam.de/danschef/geoarray/badges/master/coverage.svg
:target: https://danschef.git-pages.gfz-potsdam.de/geoarray/coverage/
.. image:: https://img.shields.io/pypi/v/geoarray.svg
:target: https://pypi.python.org/pypi/geoarray
.. image:: https://img.shields.io/conda/vn/conda-forge/geoarray.svg
:target: https://anaconda.org/conda-forge/geoarray
.. image:: https://img.shields.io/pypi/l/geoarray.svg
:target: https://gitext.gfz-potsdam.de/danschef/geoarray/blob/master/LICENSE
:target: https://git.gfz-potsdam.de/danschef/geoarray/blob/master/LICENSE
.. image:: https://img.shields.io/pypi/pyversions/geoarray.svg
:target: https://img.shields.io/pypi/pyversions/geoarray.svg
.. image:: https://img.shields.io/pypi/dm/geoarray.svg
:target: https://pypi.python.org/pypi/geoarray
See also the latest coverage_ report and the nosetests_ HTML report.
Features
--------
Features and usage
------------------
* TODO
* There is an example notebook that shows how to use geoarray: here_.
Installation
------------
geoarray depends on some open source packages which are usually installed without problems by the automatic install
routine. However, for some projects, we strongly recommend resolving the dependency before the automatic installer
is run. This approach avoids problems with conflicting versions of the same software.
Using conda_, the recommended approach is:
.. code-block:: console
Using Anaconda or Miniconda (recommended)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Using conda_ (latest version recommended), geoarray is installed as follows:
1. Create virtual environment for geoarray (optional but recommended):
.. code-block:: bash
$ conda create -c conda-forge --name geoarray python=3
$ conda activate geoarray
2. Then install geoarray itself:
.. code-block:: bash
$ conda install -c conda-forge geoarray
This is the preferred method to install geoarray, as it always installs the most recent stable release and
automatically resolves all the dependencies.
Using pip (not recommended)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
# create virtual environment for geoarray, this is optional
conda create -y -q --name geoarray python=3
source activate geoarray
conda install -y -q -c conda-forge numpy gdal scikit-image matplotlib pandas rasterio pyproj basemap shapely geopandas
conda install --yes -c ioam bokeh holoviews # optional
There is also a `pip`_ installer for geoarray. However, please note that geoarray depends on some
open source packages that may cause problems when installed with pip. Therefore, we strongly recommend
to resolve the following dependencies before the pip installer is run:
* cartopy
* gdal >=2.1.0
* geopandas
* holoviews # optional, in case you want to use interactive plotting
* matplotlib
* numpy
* pandas
* pyproj >2.2.0
* scikit-image
* shapely
To install geoarray, use the pip installer:
Then, the pip installer can be run by:
.. code-block:: console
.. code-block:: bash
pip install geoarray
$ pip install geoarray
If you don't have `pip`_ installed, this `Python installation guide`_ can guide you through the process.
* Or clone the repository via GIT and update the PATH environment variable:
.. code-block:: console
History / Changelog
-------------------
cd /your/installation/folder
git clone https://gitext.gfz-potsdam.de/danschef/geoarray.git
PATH=$PATH:~/path/to/your/installation/folder/geoarray
You can find the protocol of recent changes in the geoarray package
`here <https://git.gfz-potsdam.de/danschef/geoarray/-/blob/master/HISTORY.rst>`__.
Credits
......@@ -84,6 +110,9 @@ This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypack
.. _Cookiecutter: https://github.com/audreyr/cookiecutter
.. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage
.. _coverage: http://danschef.gitext.gfz-potsdam.de/geoarray/coverage/
.. _nosetests: http://danschef.gitext.gfz-potsdam.de/geoarray/nosetests_reports/nosetests.html
.. _coverage: https://danschef.git-pages.gfz-potsdam.de/geoarray/coverage/
.. _nosetests: https://danschef.git-pages.gfz-potsdam.de/geoarray/nosetests_reports/nosetests.html
.. _conda: https://conda.io/docs/
.. _here: https://git.gfz-potsdam.de/danschef/geoarray/-/blob/master/examples/notebooks/features_and_usage.ipynb
.. _pip: https://pip.pypa.io
.. _Python installation guide: http://docs.python-guide.org/en/latest/starting/installation/
......@@ -32,13 +32,13 @@ You can either clone the public repository:
.. code-block:: console
$ git clone https://gitext.gfz-potsdam.de/danschef/geoarray.git
$ git clone https://git.gfz-potsdam.de/danschef/geoarray.git
Or download the `tarball`_:
.. code-block:: console
$ curl -OL https://gitext.gfz-potsdam.de/danschef/geoarray/repository/archive.tar.gz?ref=master
$ curl -OL https://git.gfz-potsdam.de/danschef/geoarray/repository/archive.tar.gz?ref=master
Once you have a copy of the source, you can install it with:
......@@ -47,5 +47,5 @@ Once you have a copy of the source, you can install it with:
$ python setup.py install
.. _GitLab repo: https://gitext.gfz-potsdam.de/danschef/geoarray
.. _tarball: https://gitext.gfz-potsdam.de/danschef/geoarray/repository/archive.tar.gz?ref=master
.. _GitLab repo: https://git.gfz-potsdam.de/danschef/geoarray
.. _tarball: https://git.gfz-potsdam.de/danschef/geoarray/repository/archive.tar.gz?ref=master
This diff is collapsed.
# -*- coding: utf-8 -*-
# geoarray, A fast Python interface for image geodata - either on disk or in memory.
#
# Copyright (C) 2019 Daniel Scheffler (GFZ Potsdam, daniel.scheffler@gfz-potsdam.de)
#
# This software was developed within the context of the GeoMultiSens project funded
# by the German Federal Ministry of Education and Research
# (project grant code: 01 IS 14 010 A-C).
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
import os
if 'MPLBACKEND' not in os.environ:
os.environ['MPLBACKEND'] = 'Agg'
......
This diff is collapsed.
# -*- coding: utf-8 -*-
# geoarray, A fast Python interface for image geodata - either on disk or in memory.
#
# Copyright (C) 2019 Daniel Scheffler (GFZ Potsdam, daniel.scheffler@gfz-potsdam.de)
#
# This software was developed within the context of the GeoMultiSens project funded
# by the German Federal Ministry of Education and Research
# (project grant code: 01 IS 14 010 A-C).
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
import numpy as np
# internal imports
......@@ -17,7 +38,7 @@ class BadDataMask(GeoArray):
if self.is_inmem:
# validate input data - before converting to bool
self._validate_array_values(self.arr)
self.arr = self.arr.astype(np.bool)
self.arr = self.arr.astype(bool)
# del self._mask_baddata, self.mask_baddata # TODO delete property (requires deleter)
......@@ -29,7 +50,7 @@ class BadDataMask(GeoArray):
def arr(self, ndarray):
assert isinstance(ndarray, np.ndarray), "'arr' can only be set to a numpy array!"
self._validate_array_values(ndarray)
self._arr = ndarray.astype(np.bool)
self._arr = ndarray.astype(bool)
def _validate_array_values(self, maskarray):
pixelVals_in_mask = sorted(list(np.unique(maskarray)))
......@@ -50,7 +71,7 @@ class NoDataMask(GeoArray):
if self.is_inmem:
# validate input data - before converting to bool
self._validate_array_values(self.arr)
self.arr = self.arr.astype(np.bool)
self.arr = self.arr.astype(bool)
# del self._mask_nodata, self.mask_nodata # TODO delete property (requires deleter)
# TODO disk-mode: init must check the numbers of bands, and ideally also the pixel values in mask
......@@ -63,7 +84,7 @@ class NoDataMask(GeoArray):
def arr(self, ndarray):
assert isinstance(ndarray, np.ndarray), "'arr' can only be set to a numpy array!"
self._validate_array_values(ndarray)
self._arr = ndarray.astype(np.bool)
self._arr = ndarray.astype(bool)
def _validate_array_values(self, maskarray):
pixelVals_in_mask = sorted(list(np.unique(maskarray)))
......
# -*- coding: utf-8 -*-
# geoarray, A fast Python interface for image geodata - either on disk or in memory.
#
# Copyright (C) 2019 Daniel Scheffler (GFZ Potsdam, daniel.scheffler@gfz-potsdam.de)
#
# This software was developed within the context of the GeoMultiSens project funded
# by the German Federal Ministry of Education and Research
# (project grant code: 01 IS 14 010 A-C).
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Lesser General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
# details.
#
# You should have received a copy of the GNU Lesser General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
import os
from pprint import pformat
from copy import deepcopy
from typing import Union # noqa F401 # flake8 issue
from collections import OrderedDict
from geopandas import GeoDataFrame, GeoSeries
from pandas import DataFrame, Series
import numpy as np
try:
from osgeo import gdal
except ImportError:
import gdal
from osgeo import gdal
autohandled_meta = [
......@@ -27,32 +46,36 @@ autohandled_meta = [
class GDAL_Metadata(object):
def __init__(self, filePath='', nbands=1):
def __init__(self, filePath='', nbands=1, nodata_allbands=None):
# privates
self._global_meta = dict()
self._band_meta = dict()
self._global_meta = OrderedDict()
self._band_meta = OrderedDict()
self.bands = nbands
self.filePath = filePath
self.fileFormat = ''
self.nodata_allbands = nodata_allbands
if filePath:
self.read_from_file(filePath)
if nodata_allbands is not None:
self.global_meta.update({'data ignore value': str(nodata_allbands)})
@classmethod
def from_file(cls, filePath):
return GDAL_Metadata(filePath=filePath)
def to_DataFrame(self):
df = GeoDataFrame(columns=range(self.bands))
df = DataFrame(columns=range(self.bands))
# add global meta
for k, v in self.global_meta.items():
df.loc[k] = GeoSeries(dict(zip(df.columns, [v] * len(df.columns))))
df.loc[k] = Series(dict(zip(df.columns, [v] * len(df.columns))))
# add band meta
for k, v in self.band_meta.items():
df.loc[k] = GeoSeries(dict(zip(df.columns, v)))
df.loc[k] = Series(dict(zip(df.columns, v)))
return df
......@@ -73,8 +96,8 @@ class GDAL_Metadata(object):
@band_meta.setter
def band_meta(self, meta_dict):
if not isinstance(meta_dict, dict):
raise TypeError("Expected type 'dict', received '%s'." % type(meta_dict))
if not isinstance(meta_dict, (dict, OrderedDict)):
raise TypeError("Expected type 'dict'/'OrderedDict', received '%s'." % type(meta_dict))
for k, v in meta_dict.items():
if not isinstance(v, list):
......@@ -83,11 +106,11 @@ class GDAL_Metadata(object):
raise ValueError("The length of the given lists must be equal to the number of bands. "
"Received a list with %d items for '%s'." % (len(v), k))
self._band_meta = meta_dict # TODO convert strings to useful types
self._band_meta = OrderedDict(meta_dict) # TODO convert strings to useful types
@property
def all_meta(self):
all_meta = self.global_meta.copy()
all_meta = OrderedDict(self.global_meta.copy())
all_meta.update(self.band_meta)
return all_meta
......@@ -106,17 +129,17 @@ class GDAL_Metadata(object):
return param_value.strip()
def _convert_param_to_ENVI_str(self, param_value):
if isinstance(param_value, int):
if isinstance(param_value, (int, np.integer)):
return str(param_value)
elif isinstance(param_value, float):
elif isinstance(param_value, (float, np.floating)):
return '%f' % param_value
elif isinstance(param_value, list):
return '{ ' + ',\n'.join([self._convert_param_to_ENVI_str(i) for i in param_value]) + ' }'
else:
return param_value
return str(param_value)
def read_from_file(self, filePath):
assert ' ' not in filePath, "The given path contains whitespaces. This is not supported by GDAL."
......@@ -151,7 +174,9 @@ class GDAL_Metadata(object):
item_str.split('}')[0].strip() if item_str.strip().endswith('}') else
item_str.strip() for item_str in v.split(',')]
self.band_meta[k] = [self._convert_param_from_str(item_str) for item_str in item_list]
self.band_meta[k] = \
[self._convert_param_from_str(item_str) for item_str in item_list] \
if k != 'band_names' else item_list
else:
# global meta parameter
......@@ -190,8 +215,8 @@ class GDAL_Metadata(object):
return 'Metadata: \n\n' + pformat(self.all_meta)
def to_ENVI_metadict(self):
return dict(zip(self.all_meta.keys(),
[self._convert_param_to_ENVI_str(i) for i in self.all_meta.values()]))
return OrderedDict(zip(self.all_meta.keys(),
[self._convert_param_to_ENVI_str(i) for i in self.all_meta.values()]))