Commit 8829a837 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Merge branch 'master' into feature/implement_operators

Conflicts:
	geoarray/baseclasses.py
parents 323a51f1 a168f9b2
Pipeline #3099 failed with stages
in 1 minute and 24 seconds
......@@ -10,35 +10,90 @@ variables:
stages:
- test
- deploy_pages
- deploy_to_pypi
- deploy
- cleanup
test_geoarray:
stage: test
script:
- source /root/anaconda3/bin/activate
- export GDAL_DATA=/root/anaconda3/share/gdal
- source /root/miniconda3/bin/activate
- export GDAL_DATA=/root/miniconda3/share/gdal
- export PYTHONPATH=$PYTHONPATH:/root # /root <- directory needed later
- make coverage
- export LD_LIBRARY_PATH=/root/anaconda3/lib:$LD_LIBRARY_PATH # /root <- directory needed later
# update in-house dependencies
- pip install py_tools_ds>=0.9.0
# run tests
- make nosetests
- make docs
artifacts:
paths:
- htmlcov/
- docs/_build/html/
- nosetests.html
- nosetests.xml
when: always
pages:
stage: deploy_pages
test_styles:
stage: test
script:
- source /root/miniconda3/bin/activate
- export GDAL_DATA=/root/miniconda3/share/gdal
- export PYTHONPATH=$PYTHONPATH:/root # /root <- directory needed later
- pip install "pycodestyle>=2.0.0,!=2.4.0" # TODO remove as soon as docker runner is recreated
- make lint
artifacts:
paths:
- tests/linting/flake8.log
- tests/linting/pycodestyle.log
- tests/linting/pydocstyle.log
when: always
test_geoarray_install:
stage: test
script:
- source /root/miniconda3/bin/activate
- export GDAL_DATA=/root/miniconda3/share/gdal
- conda create -y -q --name geoarray python=3
- source activate geoarray
# resolve some requirements with conda
- conda install --yes -q -c conda-forge numpy scikit-image matplotlib geopandas gdal rasterio pyproj basemap shapely
# run installer
- python setup.py install
# test if its importable
- cd ..
- pwd
- ls
- python -c "import geoarray; print(geoarray)"
- python -c "from geoarray import GeoArray"
only:
- master
pages: # this job must be called 'pages' to advise GitLab to upload content to GitLab Pages
stage: deploy
dependencies:
- test_geoarray
script:
- mkdir -p public/coverage
- cp -r htmlcov/* public/coverage/
# Create the public directory
- rm -rf public
- mkdir public
- mkdir -p public/doc
- mkdir -p public/coverage
- mkdir -p public/nosetests_reports
# Copy over the docs
- cp -r docs/_build/html/* public/doc/
# Copy over the coverage reports
- cp -r htmlcov/* public/coverage/
# Copy over the nosetests reports
- cp nosetests.* public/nosetests_reports/
# Check if everything is working great
- ls -al public
- ls -al public/doc
- ls -al public/coverage
- ls -al public/nosetests_reports
artifacts:
paths:
- public
......@@ -48,11 +103,11 @@ pages:
deploy_pypi:
stage: deploy_to_pypi
stage: deploy
dependencies:
- test_geoarray
script: # Configure the PyPI credentials, then push the package, and cleanup the creds.
- source /root/anaconda3/bin/activate
- source /root/miniconda3/bin/activate
- printf "[distutils]\nindex-servers =\n pypi\n\n" >> ~/.pypirc
- printf "[pypi]\n""repository:"" https://upload.pypi.org/legacy/\n" >> ~/.pypirc
- printf "username= ${PYPI_USER}\n" >> ~/.pypirc
......
.PHONY: clean clean-test clean-pyc clean-build docs help
.PHONY: clean clean-test clean-pyc clean-build docs help nosetests
.DEFAULT_GOAL := help
define BROWSER_PYSCRIPT
import os, webbrowser, sys
......@@ -28,7 +28,6 @@ help:
clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts
clean-build: ## remove build artifacts
rm -fr build/
rm -fr dist/
......@@ -43,27 +42,36 @@ clean-pyc: ## remove Python file artifacts
find . -name '__pycache__' -exec rm -fr {} +
clean-test: ## remove test and coverage artifacts
coverage erase
rm -fr .tox/
rm -f .coverage
rm -fr htmlcov/
rm -fr nosetests.html
rm -fr nosetests.xml
lint: ## check style with flake8
flake8 geoarray tests
flake8 --max-line-length=120 geoarray tests > ./tests/linting/flake8.log
pycodestyle geoarray --exclude="*.ipynb,*.ipynb*" --max-line-length=120 > ./tests/linting/pycodestyle.log
-pydocstyle geoarray > ./tests/linting/pydocstyle.log
test: ## run tests quickly with the default Python
python setup.py test
python setup.py test
test-all: ## run tests on every Python version with tox
tox
coverage: ## check code coverage quickly with the default Python
coverage run --source geoarray setup.py test
coverage: clean-test ## check code coverage quickly with the default Python
coverage run --source geoarray setup.py test
coverage report -m
coverage html
# $(BROWSER) htmlcov/index.html
coverage report -m
coverage html
$(BROWSER) htmlcov/index.html
nosetests: clean-test ## Runs nosetests with coverage, xUnit and nose-html-output
## - puts the coverage results in the folder 'htmlcov'
## - generates 'nosetests.html' (--with-html)
## - generates 'nosetests.xml' (--with-xunit) which is currently not visualizable by GitLab
nosetests -vv --with-coverage --cover-package=geoarray --cover-erase --cover-html --cover-html-dir=htmlcov \
--with-html --with-xunit --rednose --force-color
docs: ## generate Sphinx HTML documentation, including API docs
rm -f docs/geoarray.rst
......@@ -73,7 +81,7 @@ docs: ## generate Sphinx HTML documentation, including API docs
$(MAKE) -C docs html
# $(MAKE) -C docs latex
# $(MAKE) -C docs latexpdf
$(BROWSER) docs/_build/html/index.html
# $(BROWSER) docs/_build/html/index.html
servedocs: docs ## compile the docs watching for changes
watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .
......
......@@ -30,9 +30,13 @@ Status
:target: http://danschef.gitext.gfz-potsdam.de/geoarray/coverage/
.. image:: https://img.shields.io/pypi/v/geoarray.svg
:target: https://pypi.python.org/pypi/geoarray
.. image:: https://img.shields.io/pypi/l/geoarray.svg
:target: https://gitext.gfz-potsdam.de/danschef/geoarray/blob/master/LICENSE
.. image:: https://img.shields.io/pypi/pyversions/geoarray.svg
:target: https://img.shields.io/pypi/pyversions/geoarray.svg
See also the latest coverage_ report.
See also the latest coverage_ report and the nosetests_ HTML report.
Features
......@@ -43,21 +47,36 @@ Features
Installation
------------
* Use the pip-Installer:
geoarray depends on some open source packages which are usually installed without problems by the automatic install
routine. However, for some projects, we strongly recommend resolving the dependency before the automatic installer
is run. This approach avoids problems with conflicting versions of the same software.
Using conda_, the recommended approach is:
.. code-block:: console
# create virtual environment for geoarray, this is optional
conda create -y -q --name geoarray python=3
source activate geoarray
conda install -y -q -c conda-forge numpy gdal scikit-image matplotlib pandas rasterio pyproj basemap shapely geopandas
conda install --yes -c ioam bokeh holoviews # optional
To install geoarray, use the pip installer:
.. code-block:: console
pip install geoarray
* Or clone the repository via GIT and update the PATH environment variable:
cd /your/installation/folder
.. code-block:: console
cd /your/installation/folder
git clone https://gitext.gfz-potsdam.de/danschef/geoarray.git
PATH=$PATH:~/path/to/your/installation/folder/geoarray
Credits
-------
......@@ -66,4 +85,5 @@ This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypack
.. _Cookiecutter: https://github.com/audreyr/cookiecutter
.. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage
.. _coverage: http://danschef.gitext.gfz-potsdam.de/geoarray/coverage/
.. _nosetests: http://danschef.gitext.gfz-potsdam.de/geoarray/nosetests_reports/nosetests.html
.. _conda: https://conda.io/docs/
# -*- coding: utf-8 -*-
__author__ = """Daniel Scheffler"""
__email__ = 'danschef@gfz-potsdam.de'
__version__ = '0.5.9'
__versionalias__ = 'v20170823.01'
import os
if 'MPLBACKEND' not in os.environ:
os.environ['MPLBACKEND'] = 'Agg'
from .baseclasses import GeoArray # noqa: E402
from .masks import BadDataMask # noqa: E402
from .masks import NoDataMask # noqa: E402
from .masks import CloudMask # noqa: E402
from .version import __version__, __versionalias__ # noqa (E402 + F401)
from .baseclasses import GeoArray
from .masks import BadDataMask
from .masks import NoDataMask
from .masks import CloudMask
__all__=['GeoArray',
'BadDataMask',
'NoDataMask',
'CloudMask'
]
__author__ = """Daniel Scheffler"""
__email__ = 'danschef@gfz-potsdam.de'
__all__ = ['__version__',
'__versionalias__',
'__author__',
'__email__',
'GeoArray',
'BadDataMask',
'NoDataMask',
'CloudMask'
]
This diff is collapsed.
# -*- coding: utf-8 -*-
__author__='Daniel Scheffler'
import numpy as np
# internal imports
from .baseclasses import GeoArray
__author__ = 'Daniel Scheffler'
class BadDataMask(GeoArray):
def __init__(self, path_or_array, geotransform=None, projection=None, bandnames=None, nodata=False, progress=True,
q=False):
super(BadDataMask, self).__init__(path_or_array, geotransform=geotransform, projection=projection,
bandnames=bandnames, nodata=nodata, progress=progress, q=q)
......@@ -19,8 +19,7 @@ class BadDataMask(GeoArray):
self._validate_array_values(self.arr)
self.arr = self.arr.astype(np.bool)
# del self._mask_baddata, self.mask_baddata # TODO delete property (requires deleter)
# del self._mask_baddata, self.mask_baddata # TODO delete property (requires deleter)
@property
def arr(self):
......@@ -37,17 +36,14 @@ class BadDataMask(GeoArray):
assert len(pixelVals_in_mask) <= 2, 'Bad data mask must have only two pixel values (boolean) - 0 and 1 or ' \
'False and True! The given mask for %s contains the values %s.' \
% (self.basename, pixelVals_in_mask)
assert pixelVals_in_mask in [[0, 1], [0],[1], [False, True], [False], [True]],\
assert pixelVals_in_mask in [[0, 1], [0], [1], [False, True], [False], [True]], \
'Found unsupported pixel values in the given bad data mask for %s: %s. Only the values True, False, 0 ' \
'and 1 are supported. ' % (self.basename, pixelVals_in_mask)
class NoDataMask(GeoArray):
def __init__(self, path_or_array, geotransform=None, projection=None, bandnames=None, nodata=False, progress=True,
q=False):
super(NoDataMask, self).__init__(path_or_array, geotransform=geotransform, projection=projection,
bandnames=bandnames, nodata=nodata, progress=progress, q=q)
......@@ -56,9 +52,8 @@ class NoDataMask(GeoArray):
self._validate_array_values(self.arr)
self.arr = self.arr.astype(np.bool)
# del self._mask_nodata, self.mask_nodata # TODO delete property (requires deleter)
# TODO disk-mode: init must check the numbers of bands, and ideally also the pixel values in mask
# del self._mask_nodata, self.mask_nodata # TODO delete property (requires deleter)
# TODO disk-mode: init must check the numbers of bands, and ideally also the pixel values in mask
@property
def arr(self):
......@@ -80,20 +75,16 @@ class NoDataMask(GeoArray):
'and 1 are supported. ' % (self.basename, pixelVals_in_mask)
class CloudMask(GeoArray):
def __init__(self, path_or_array, geotransform=None, projection=None, bandnames=None, nodata=None, progress=True,
q=False):
# TODO implement class definitions and specific metadata
super(CloudMask,self).__init__(path_or_array, geotransform=geotransform, projection=projection,
super(CloudMask, self).__init__(path_or_array, geotransform=geotransform, projection=projection,
bandnames=bandnames, nodata=nodata, progress=progress, q=q)
# del self._mask_nodata, self.mask_nodata # TODO delete property (requires deleter)
# TODO check that: "Automatically detected nodata value for CloudMask 'IN_MEM': 1.0"
def to_ENVI_classification(self):
raise NotImplementedError # TODO
def to_ENVI_classification(self): # pragma: no cover
raise NotImplementedError # TODO
# -*- coding: utf-8 -*-
import os
from pprint import pformat
from copy import deepcopy
from typing import Union # noqa F401 # flake8 issue
from geopandas import GeoDataFrame, GeoSeries
import numpy as np
try:
from osgeo import gdal
except ImportError:
import gdal
autohandled_meta = [
'bands',
'byte_order',
'coordinate_system_string',
'data_type',
'file_type',
'header_offset',
'interleave',
'lines',
'samples',
]
class GDAL_Metadata(object):
def __init__(self, filePath='', nbands=1):
# privates
self._global_meta = dict()
self._band_meta = dict()
self.bands = nbands
self.filePath = filePath
self.fileFormat = ''
if filePath:
self.read_from_file(filePath)
@classmethod
def from_file(cls, filePath):
return GDAL_Metadata(filePath=filePath)
def to_DataFrame(self):
df = GeoDataFrame(columns=range(self.bands))
# add global meta
for k, v in self.global_meta.items():
df.loc[k] = GeoSeries(dict(zip(df.columns, [v] * len(df.columns))))
# add band meta
for k, v in self.band_meta.items():
df.loc[k] = GeoSeries(dict(zip(df.columns, v)))
return df
@property
def global_meta(self):
return self._global_meta
@global_meta.setter
def global_meta(self, meta_dict):
if not isinstance(meta_dict, dict):
raise TypeError("Expected type 'dict', received '%s'." % type(meta_dict))
self._global_meta = meta_dict # TODO convert strings to useful types
@property
def band_meta(self):
return self._band_meta
@band_meta.setter
def band_meta(self, meta_dict):
if not isinstance(meta_dict, dict):
raise TypeError("Expected type 'dict', received '%s'." % type(meta_dict))
for k, v in meta_dict.items():
if not isinstance(v, list):
raise TypeError('The values of the given dictionary must be lists. Received %s for %s.' % (type(v), k))
if len(v) != self.bands:
raise ValueError("The length of the given lists must be equal to the number of bands. "
"Received a list with %d items for '%s'." % (len(v), k))
self._band_meta = meta_dict # TODO convert strings to useful types
@property
def all_meta(self):
all_meta = self.global_meta.copy()
all_meta.update(self.band_meta)
return all_meta
@staticmethod
def _convert_param_from_str(param_value):
try:
try:
return int(param_value) # NOTE: float('0.34') causes ValueError: invalid literal for int() with base 10
except ValueError:
return float(param_value)
except ValueError:
if param_value.startswith('{'):
param_value = param_value.split('{')[1]
if param_value.endswith('}'):
param_value = param_value.split('}')[0]
return param_value.strip()
def _convert_param_to_ENVI_str(self, param_value):
if isinstance(param_value, int):
return str(param_value)
elif isinstance(param_value, float):
return '%f' % param_value
elif isinstance(param_value, list):
return '{ ' + ',\n'.join([self._convert_param_to_ENVI_str(i) for i in param_value]) + ' }'
else:
return param_value
def read_from_file(self, filePath):
assert ' ' not in filePath, "The given path contains whitespaces. This is not supported by GDAL."
if not os.path.exists(filePath):
raise FileNotFoundError(filePath)
ds = gdal.Open(filePath)
try:
if not ds:
raise Exception('Error reading file: ' + gdal.GetLastErrorMsg())
self.bands = ds.RasterCount
self.fileFormat = ds.GetDriver().GetDescription()
###############
# ENVI format #
###############
if self.fileFormat == 'ENVI':
metadict = ds.GetMetadata('ENVI')
for k, v in metadict.items():
if k not in autohandled_meta:
if len(v.split(',')) == self.bands:
# band meta parameter
item_list = [
item_str.split('{')[1].strip() if item_str.strip().startswith('{') else
item_str.split('}')[0].strip() if item_str.strip().endswith('}') else
item_str.strip() for item_str in v.split(',')]
self.band_meta[k] = [self._convert_param_from_str(item_str) for item_str in item_list]
else:
# global meta parameter
self.global_meta[k] = self._convert_param_from_str(v)
#####################
# remaining formats #
#####################
else:
# read global domain metadata
self.global_meta = ds.GetMetadata()
# read band domain metadata
for b in range(self.bands):
band = ds.GetRasterBand(b + 1)
# meta_gs = GeoSeries(band.GetMetadata())
bandmeta_dict = band.GetMetadata()
for k, v in bandmeta_dict.items():
if k not in self.band_meta:
self.band_meta[k] = []
self.band_meta[k].append(self._convert_param_from_str(v))
# # fill metadata
# self.df[b] = meta_gs
del band
finally:
del ds
return self.all_meta
def __repr__(self):
return 'Metadata: \n\n' + pformat(self.all_meta)
def to_ENVI_metadict(self):
return dict(zip(self.all_meta.keys(),
[self._convert_param_to_ENVI_str(i) for i in self.all_meta.values()]))
def get_subset(self, bands2extract=None, keys2extract=None):
# type: (Union[slice, list, np.ndarray], Union[str, list]) -> 'GDAL_Metadata'
meta_sub = deepcopy(self)
# subset bands
if bands2extract is not None:
if isinstance(bands2extract, list):
bands2extract = np.array(bands2extract)
elif isinstance(bands2extract, (np.ndarray, slice)):
pass # all fine
else:
raise TypeError(bands2extract)
meta_sub.band_meta = self.band_meta.copy()
for k, v in meta_sub.band_meta.items():
meta_sub.band_meta[k] = list(np.array(v)[bands2extract])
meta_sub.bands = len(list(range(*bands2extract.indices(bands2extract.stop)))) \
if isinstance(bands2extract, slice) else bands2extract.size
# subset metadata keys
if keys2extract:
keys2extract = [keys2extract] if isinstance(keys2extract, str) else keys2extract
# global_meta = meta_sub.global_meta.copy()
for k in meta_sub.global_meta.copy().keys():
if k not in keys2extract:
del meta_sub.global_meta[k]
for k in meta_sub.band_meta.copy().keys():
if k not in keys2extract:
del meta_sub.band_meta[k]
if not meta_sub.all_meta:
raise ValueError(keys2extract, 'The given metadata keys do not exist.')
return meta_sub
def __getitem__(self, given):
if isinstance(given, int):
return self.get_subset(bands2extract=slice(given, given + 1))
elif isinstance(given, slice):
return self.get_subset(bands2extract=given)
elif isinstance(given, str):
return self.get_subset(keys2extract=given)
elif isinstance(given, list):
if isinstance(given[0], str):
return self.get_subset(keys2extract=given)
elif isinstance(given[0], int):
return self.get_subset(bands2extract=given)
else:
raise TypeError(given, 'Given list must contain string or integer items.')
elif isinstance(given, np.ndarray):
if given.ndim != 1:
raise TypeError(given, 'Given numpy array must be one-dimensional.')
return self.get_subset(bands2extract=given)
else:
raise TypeError(given)
# -*- coding: utf-8 -*-
__author__='Daniel Scheffler'