diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index db2efc4c86094ebbe6f80a339ca1dc8c4431b04d..aa43f14c77c792b3e1f937efd7e4699a50dd2dc6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,6 +29,7 @@ test_enpt: # run nosetests - make nosetests # test are called here # create the docs + - pip install sphinx_rtd_theme # Read-the-docs theme for SPHINX documentation - make docs artifacts: paths: @@ -91,6 +92,9 @@ test_enpt_install: # test importability - python -c "import enpt; print(enpt)" - python -c "from enpt.model.images import EnMAPL1Product_SensorGeo" + only: + - master + - dev pages: @@ -121,3 +125,4 @@ pages: expire_in: 30 days only: - master + - enhancement/improve_docs diff --git a/AUTHORS.rst b/AUTHORS.rst index eebf49209e8956e6e1db8cef2bcc7551fd0c8e85..c7ff055226edb28996730a7fb25636e566667ba5 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -10,7 +10,7 @@ Karl Segl Contributors ------------ -* Daniel Scheffler +* Daniel Scheffler (main developer of the EnPT source code) +* Niklas Bohn (main developer of the SICOR atmopsheric correction source code) * André Hollstein -* Stephane Guillaso -* Niklas Bohn +* Stéphane Guillaso diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index a54ff10c1cad175a58ebfc9483e019d22aa8048c..78e0da2341f5b3917f70db03720f5f71a11929d4 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -15,7 +15,7 @@ Types of Contributions Report Bugs ~~~~~~~~~~~ -Report bugs at https://github.com/audreyr/enpt/issues. +Report bugs at https://gitext.gfz-potsdam.de/EnMAP/GFZ_Tools_EnMAP_BOX/EnPT/issues If you are reporting a bug, please include: @@ -26,13 +26,13 @@ If you are reporting a bug, please include: Fix Bugs ~~~~~~~~ -Look through the GitHub issues for bugs. Anything tagged with "bug" +Look through the GitLab issues for bugs. Anything tagged with "bug" and "help wanted" is open to whoever wants to implement it. Implement Features ~~~~~~~~~~~~~~~~~~ -Look through the GitHub issues for features. Anything tagged with "enhancement" +Look through the GitLab issues for features. Anything tagged with "enhancement" and "help wanted" is open to whoever wants to implement it. Write Documentation @@ -45,26 +45,26 @@ articles, and such. Submit Feedback ~~~~~~~~~~~~~~~ -The best way to send feedback is to file an issue at https://github.com/audreyr/enpt/issues. +The best way to send feedback is to file an issue at https://gitext.gfz-potsdam.de/EnMAP/GFZ_Tools_EnMAP_BOX/EnPT/issues If you are proposing a feature: * Explain in detail how it would work. * Keep the scope as narrow as possible, to make it easier to implement. -* Remember that this is a volunteer-driven project, and that contributions - are welcome :) +* Remember that this is a volunteer-driven project, and that contributions are welcome :) Get Started! ------------ Ready to contribute? Here's how to set up `enpt` for local development. -1. Fork the `enpt` repo on GitHub. +1. Fork the `enpt` repo on GitLab. 2. Clone your fork locally:: - $ git clone git@github.com:your_name_here/enpt.git + $ git clone git@gitext.gfz-potsdam.de:your_name_here/enpt.git -3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: +3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up + your fork for local development:: $ mkvirtualenv enpt $ cd enpt/ @@ -76,33 +76,34 @@ Ready to contribute? Here's how to set up `enpt` for local development. Now you can make your changes locally. -5. When you're done making changes, check that your changes pass flake8 and the tests, including testing other Python versions with tox:: +5. When you're done making changes, check that your changes pass the codestyle and the software tests, including + testing other Python versions with tox:: - $ flake8 enpt tests - $ python setup.py test or py.test + $ make lint + $ python -m unittest $ tox To get flake8 and tox, just pip install them into your virtualenv. -6. Commit your changes and push your branch to GitHub:: +6. Commit your changes and push your branch to GitLab:: $ git add . $ git commit -m "Your detailed description of your changes." $ git push origin name-of-your-bugfix-or-feature -7. Submit a pull request through the GitHub website. +7. Submit a merge request through the GitLab website. Pull Request Guidelines ----------------------- Before you submit a pull request, check that it meets these guidelines: -1. The pull request should include tests. -2. If the pull request adds functionality, the docs should be updated. Put +1. The merge request should include tests. +2. If the merge request adds functionality, the docs should be updated. Put your new functionality into a function with a docstring, and add the feature to the list in README.rst. -3. The pull request should work for Python 2.6, 2.7, 3.3, 3.4 and 3.5, and for PyPy. Check - https://travis-ci.org/audreyr/enpt/pull_requests +3. The merge request should work for Python 2.6, 2.7, 3.4, 3.5, 3.6 and 3.7. Check + https://gitext.gfz-potsdam.de/EnMAP/GFZ_Tools_EnMAP_BOX/EnPT/merge_requests and make sure that the tests pass for all supported Python versions. Tips diff --git a/LICENSE b/LICENSE index 7ba689225b9619d465d851d7d50df2ae1461e76a..0053c3ad3f0939dcadc055fc577c426617bdee44 100644 --- a/LICENSE +++ b/LICENSE @@ -3,7 +3,7 @@ GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007 EnMAP PT - Copyright (C) 2017 Karl Segl + Copyright (C) 2019 Karl Segl This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/Makefile b/Makefile index 0ec2a7c4bd145e89caf341e3bab7d8205b93dd1e..8dcce2dd54ebca9eb5305e201345bc04f91ac095 100644 --- a/Makefile +++ b/Makefile @@ -80,9 +80,9 @@ nosetests: clean-test ## Runs nosetests with coverage, xUnit and nose-html-outpu docs: ## generate Sphinx HTML documentation, including API docs rm -f docs/enpt.rst rm -f docs/modules.rst - sphinx-apidoc -o docs/ enpt --private - $(MAKE) -C docs clean - $(MAKE) -C docs html + sphinx-apidoc enpt -o docs/ --private --doc-project 'API Reference' + $(MAKE) -C docs clean + $(MAKE) -C docs html #$(MAKE) -C docs latex #$(MAKE) -C docs latexpdf #$(BROWSER) docs/_build/html/index.html diff --git a/README.rst b/README.rst index af62e82bdde69810afc2c41550cc0f79f705a36b..300ae65e2cd09780a7bfcc4e676f8a25b63f7060 100644 --- a/README.rst +++ b/README.rst @@ -1,8 +1,15 @@ + +.. image:: docs/img/EnPT_Logo_clipped.png + :width: 150px + :alt: EnPT Logo + ============================= EnPT - EnMAP Processing Tools ============================= -* Project Description (TODO) +The EnPT Python package is an automated pre-processing pipeline for the new EnMAP hyperspectral satellite data. +It provides free and open-source features to transform EnMAP Level-1B data to Level-2A. The package has been developed +at the German Research Centre for Geosciences Potsdam (GFZ) as an alternative to the original DLR processing chain. Please check the documentation_ for usage and in depth information. @@ -10,16 +17,26 @@ License ------- Free software: GNU General Public License v3 -Features --------- +Feature overview +---------------- -* TODO +* read EnMAP Level-1B input data +* radiometric conversion to top-of-atmosphere radiance +* dead pixel correction +* atmospheric correction (based on SICOR_) +* conversion of top-of-atmosphere-radiance to top-of-atmosphere-reflectance +* detection and correction of geometric misregistrations compared to user provided spatial reference (based on AROSICS_) +* orthorectification +* write EnMAP Level-2 output data Status ------ -.. figure:: https://gitext.gfz-potsdam.de/EnMAP/GFZ_Tools_EnMAP_BOX/EnPT/badges/master/build.svg -.. figure:: https://gitext.gfz-potsdam.de/EnMAP/GFZ_Tools_EnMAP_BOX/EnPT/badges/master/coverage.svg +|badge1| |badge2| + +.. |badge1| image:: https://gitext.gfz-potsdam.de/EnMAP/GFZ_Tools_EnMAP_BOX/EnPT/badges/master/build.svg + +.. |badge2| image:: https://gitext.gfz-potsdam.de/EnMAP/GFZ_Tools_EnMAP_BOX/EnPT/badges/master/coverage.svg See also the latest coverage_ report and the nosetests_ HTML report. @@ -27,7 +44,8 @@ See also the latest coverage_ report and the nosetests_ HTML report. Credits ------- -This software was funded from BMBF under EnMAP ... +The development of the gms_preprocessing package was funded by the German Federal Ministry of Education and Research +(BMBF). This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template. @@ -35,4 +53,6 @@ This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypack .. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage .. _documentation: http://enmap.gitext.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/EnPT/doc/ .. _coverage: http://enmap.gitext.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/EnPT/coverage/ -.. _nosetests: http://enmap.gitext.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/EnPT/nosetests_reports/nosetests.html \ No newline at end of file +.. _nosetests: http://enmap.gitext.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/EnPT/nosetests_reports/nosetests.html +.. _SICOR: https://gitext.gfz-potsdam.de/EnMAP/sicor +.. _AROSICS: https://gitext.gfz-potsdam.de/danschef/arosics \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile index 73d556f29cf5add078b22c8b1d44598d1c8d3062..9dd1bbaac2e4cdc5813bb2e078613d5122ea19ab 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -21,6 +21,7 @@ I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext +# Put it first so that "make" without argument is like "make help". help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" diff --git a/docs/about.rst b/docs/about.rst new file mode 100644 index 0000000000000000000000000000000000000000..e3e8d292043f69b38f9da0c5ed59ca1cc9f80ef5 --- /dev/null +++ b/docs/about.rst @@ -0,0 +1,26 @@ +===== +About +===== + +.. image:: img/EnPT_Logo_clipped.png + :width: 150px + :alt: EnPT Logo + +The EnPT Python package is an automated pre-processing pipeline for the new EnMAP hyperspectral satellite data. +It provides free and open-source features to transform EnMAP Level-1B data to Level-2A. The package has been developed +at the German Research Centre for Geosciences Potsdam (GFZ) as an alternative to the original DLR processing chain. + +Feature overview +---------------- + +* read EnMAP Level-1B input data +* radiometric conversion to top-of-atmosphere radiance +* dead pixel correction +* atmospheric correction (based on SICOR_) +* conversion of top-of-atmosphere-radiance to top-of-atmosphere-reflectance +* detection and correction of geometric misregistrations compared to user provided spatial reference (based on AROSICS_) +* orthorectification +* write EnMAP Level-2 output data + +.. _SICOR: https://gitext.gfz-potsdam.de/EnMAP/sicor +.. _AROSICS: https://gitext.gfz-potsdam.de/danschef/arosics diff --git a/docs/algorithm_descriptions.rst b/docs/algorithm_descriptions.rst new file mode 100644 index 0000000000000000000000000000000000000000..a68111a8cb9e75f9143fcbc0fc965b0348833325 --- /dev/null +++ b/docs/algorithm_descriptions.rst @@ -0,0 +1,189 @@ +Algorithm descriptions +====================== + +EnMAP Level 1B data reader +************************** + +The EnPT reader module allows you to read in EnMAP Level-1B data in the official format as provided by the DLR. +After extraction EnPT expects a folder with the following files: + + +-----------------------------------------------+-----------------+ + | Filename | Description | + +===============================================+=================+ + |ENMAP*L1B*-HISTORY.XML | | + +-----------------------------------------------+-----------------+ + |ENMAP*L1B*-LOG.XML | | + +-----------------------------------------------+-----------------+ + |ENMAP*L1B*-METADATA.XML | | + +-----------------------------------------------+-----------------+ + |ENMAP*L1B*-QL_PIXELMASK_SWIR.GEOTIFF | | + +-----------------------------------------------+-----------------+ + |ENMAP*L1B*-QL_PIXELMASK_VNIR.GEOTIFF | | + +-----------------------------------------------+-----------------+ + |ENMAP*L1B*-QL_QUALITY_CIRRUS.GEOTIFF | | + +-----------------------------------------------+-----------------+ + |ENMAP*L1B*-QL_QUALITY_CLASSES.GEOTIFF | | + +-----------------------------------------------+-----------------+ + |ENMAP*L1B*-QL_QUALITY_CLOUD.GEOTIFF | | + +-----------------------------------------------+-----------------+ + |ENMAP*L1B*-QL_QUALITY_CLOUDSHADOW.GEOTIFF | | + +-----------------------------------------------+-----------------+ + |ENMAP*L1B*-QL_QUALITY_HAZE.GEOTIFF | | + +-----------------------------------------------+-----------------+ + |ENMAP*L1B*-QL_QUALITY_SNOW.GEOTIFF | | + +-----------------------------------------------+-----------------+ + |ENMAP*L1B*-QL_QUALITY_TESTFLAGS_SWIR.GEOTIFF | | + +-----------------------------------------------+-----------------+ + |ENMAP*L1B*-QL_QUALITY_TESTFLAGS_VNIR.GEOTIFF | | + +-----------------------------------------------+-----------------+ + |ENMAP*L1B*-QL_SWIR.GEOTIFF | | + +-----------------------------------------------+-----------------+ + |ENMAP*L1B*-QL_VNIR.GEOTIFF | | + +-----------------------------------------------+-----------------+ + |ENMAP*L1B*-SPECTRAL_IMAGE_SWIR.GEOTIFF | | + +-----------------------------------------------+-----------------+ + |ENMAP*L1B*-SPECTRAL_IMAGE_VNIR.GEOTIFF | | + +-----------------------------------------------+-----------------+ + +EnPT reads the raster files lazily / on-demand to save memory during execution. Metadata read from the +`ENMAP*-METADATA.XML` file are stored in memory and used at multiple stages of the pre-processing pipeline. + +**Conversion to top-of-atmosphere radiance** + +The image data is directly transformed from digital numbers (DNs, as provided by the DLR) to top-of-atmosphere radiance +in mW/m2/sr/nm. + +**Filling the gap between adjacent EnMAP images** + +There is a gap of 20 pixels between adjacent EnMAP Level-1B images provided by the DLR. To fill this gap, EnPT offers +the opportunity to provide the neighboring image which is then appended to the actual EnMAP image to be processed. + + + + +Dead pixel correction +********************* + +The EnPT dead pixel correction uses the pixel masks provided by DLR and interpolates the EnMAP image data at +the indicated dead pixel positions. It supports two interpolation algorithms: + + 1. spectral interpolation + * Interpolates the data in the spectral domain. + * Points outside the data range are extrapolated. + * possible interpolation methods: `linear`, `nearest`, `zero`, `slinear`, `quadratic`, `cubic`, etc. + 2. spatial interpolation + * Interpolates the data spatially. + * Remaining missing data positions (e.g., outermost columns) are spectrally interpolated. + * possible interpolation methods: `linear`, `bilinear`, `cubic`, `spline` + + + + +Atmospheric correction +********************** + +.. image:: https://gitext.gfz-potsdam.de/EnMAP/sicor/raw/master/docs/figs/sicor_logo_lr.png + :width: 150px + :alt: SICOR Logo + +EnPT uses `SICOR`_ (Sensor Independent Atmosperic Correction of optical Earth observation data from multi- and +hyperspectral instruments) for atmospheric correction, i.e., for the conversion of TOA-(top-of-atmosphere) radiance +to BOA- (bottom-of-atmosphere / surface) reflectance. SICOR is a Python based open-source package developed at the +German Research Centre for Geosciences (GFZ) Potsdam. For details on the underlying algorithm, please refer to the +`documentation pages of SICOR`_. + + + +Conversion of TOA-radiance to TOA-reflectance +********************************************* + +As an alternative to the atmospheric correction, your EnMAP Level-1B data can also be converted from +TOA-radiance to TOA-reflectance. This is useful to normalize unequal acquisition- and illumination geometries of +multiple EnMAP datasets but will not correct for atmospheric effects. + + + + +Spatial Co-Registration +*********************** + +.. image:: https://gitext.gfz-potsdam.de/danschef/arosics/raw/master/docs/images/arosics_logo.png + :width: 150px + :alt: AROSICS Logo + +For the detection of spatial misregistrations with regard to a user-provided spatial reference EnPT builds on the +open-source Python package `AROSICS`_ (An Automated and Robust Open-Source Image Co-Registration Software for +Multi-Sensor Satellite Data). It has been developed at the German Research Centre for Geosciences (GFZ) Potsdam. +For detailed algorithm description and use cases refer to the corresponding (open-access) paper that can be found here: +`Scheffler D, Hollstein A, Diedrich H, Segl K, Hostert P. AROSICS: An Automated and Robust Open-Source Image +Co-Registration Software for Multi-Sensor Satellite Data. Remote Sensing. 2017; 9(7):676`__. + +In EnPT, AROSICS is used to automacially compute thousands of tie points between a selected EnMAP band the +user-provided reference image. The computed shifts are later respected in the orthorectification step. + +__ http://www.mdpi.com/2072-4292/9/7/676 + + + + +Orthorectification +****************** + +EnMAP Level 1B data are provided in sensor geometry, i.e., the image data don't have map coordinates but only image +coordinates. For the geo-rectification of the data EnPT uses a set of Rational Polynomial Coefficients (RPCs) provided +for each band of the two EnMAP subsystems (VNIR and SWIR). Together with a user provided digital elevation model these +RPC coefficients enable a highly accurate assignment of map coordinates to each pixel of the EnMAP Level-1B images. +Resampling is done using a fast KDTree gaussian weighting neighbour approach implemented in the Python library +`pyresample`_ (find the documentation `here `__). +The spatial shifts computed during the co-registration step are respected here. + +In this processing step, the EnMAP VNIR is merged with the SWIR subsystem and from now on stored in a single 3D array. + + + + +EnMAP Level 2A data writer +************************** + +The EnPT writer module writes the computed EnMAP Level-2A data to disk after finishing the processing pipeline. The +data format produced by EnPT is based on the official DLR Level-2A format. However, due to differences in the +underlying algorithms, EnPT also produces a slightly different Level-2A data format. The differences are summarized +below: + + +-----------------------------------------------+-----+---------+-------------+ + | Filename | DLR | EnPT | Description | + +===============================================+=====+=========+=============+ + |ENMAP*L2A*-HISTORY.XML | yes | planned | | + +-----------------------------------------------+-----+---------+-------------+ + |ENMAP*L2A*-LOG.XML | yes | planned | | + +-----------------------------------------------+-----+---------+-------------+ + |ENMAP*L2A*-METADATA.XML | yes | yes | | + +-----------------------------------------------+-----+---------+-------------+ + |ENMAP*L2A*-QL_PIXELMASK_SWIR.GEOTIFF | yes | planned | | + +-----------------------------------------------+-----+---------+-------------+ + |ENMAP*L2A*-QL_PIXELMASK_VNIR.GEOTIFF | yes | planned | | + +-----------------------------------------------+-----+---------+-------------+ + |ENMAP*L2A*-QL_QUALITY_CIRRUS.GEOTIFF | yes | planned | | + +-----------------------------------------------+-----+---------+-------------+ + |ENMAP*L2A*-QL_QUALITY_CLASSES.GEOTIFF | yes | planned | | + +-----------------------------------------------+-----+---------+-------------+ + |ENMAP*L2A*-QL_QUALITY_CLOUD.GEOTIFF | yes | yes | | + +-----------------------------------------------+-----+---------+-------------+ + |ENMAP*L2A*-QL_QUALITY_CLOUDSHADOW.GEOTIFF | yes | planned | | + +-----------------------------------------------+-----+---------+-------------+ + |ENMAP*L2A*-QL_QUALITY_HAZE.GEOTIFF | yes | planned | | + +-----------------------------------------------+-----+---------+-------------+ + |ENMAP*L2A*-QL_QUALITY_SNOW.GEOTIFF | yes | planned | | + +-----------------------------------------------+-----+---------+-------------+ + |ENMAP*L2A*-QL_SWIR.GEOTIFF | yes | yes | | + +-----------------------------------------------+-----+---------+-------------+ + |ENMAP*L2A*-QL_VNIR.GEOTIFF | yes | yes | | + +-----------------------------------------------+-----+---------+-------------+ + |ENMAP*L2A*-SPECTRAL_IMAGE.GEOTIFF | yes | yes | | + +-----------------------------------------------+-----+---------+-------------+ + + +.. _SICOR: https://gitext.gfz-potsdam.de/EnMAP/sicor +.. _`documentation pages of SICOR`: http://enmap.gitext.gfz-potsdam.de/sicor/doc/ +.. _AROSICS: https://gitext.gfz-potsdam.de/danschef/arosics +.. _pyresample: https://github.com/pytroll/pyresample \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 07201b6e68ab2cd065224a22d3a78d13f5ac2950..4507eb0b3bccc5f45c2c494bb854e18d88a63378 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -40,12 +40,19 @@ import enpt # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'sphinx.ext.todo', 'sphinxarg.ext'] +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.githubpages', + 'sphinx.ext.viewcode', + 'sphinx.ext.todo', + 'sphinxarg.ext' +] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] # The suffix of source filenames. +# source_suffix = ['.rst', '.md'] source_suffix = '.rst' # The encoding of source files. @@ -56,7 +63,7 @@ master_doc = 'index' # General information about the project. project = u'EnPT' -copyright = u"2017, Karl Segl" +copyright = u"2019, Karl Segl, Daniel Scheffler, André Hollstein, Stéphane Guillaso, Niklas Bohn" # The version info for the project you're documenting, acts as replacement # for |version| and |release|, also used in various other places throughout @@ -79,7 +86,8 @@ release = enpt.__version__ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] # The reST default role (used for this markup: `text`) to use for all # documents. @@ -106,17 +114,36 @@ pygments_style = 'sphinx' # documents. # keep_warnings = False +# Define how to document class docstrings +# '__init__' documents only the __init__ methods, 'class' documents only the class methods and 'both' documents both +autoclass_content = 'both' + # -- Options for HTML output ------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +# html_theme = 'default' +html_theme = 'sphinx_rtd_theme' # The one installed via pip install sphinx_rtd_theme in the .gitlab.yml # Theme options are theme-specific and customize the look and feel of a # theme further. For a list of options available for each theme, see the # documentation. -# html_theme_options = {} +html_theme_options = { + 'canonical_url': '', + 'analytics_id': '', + 'logo_only': False, + 'display_version': True, + 'prev_next_buttons_location': 'bottom', + 'style_external_links': False, + 'vcs_pageview_mode': 'view', + # Toc options + 'collapse_navigation': True, + 'sticky_navigation': True, + 'navigation_depth': 4, + 'includehidden': True, + 'titles_only': False +} # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = [] @@ -131,7 +158,7 @@ html_theme = 'default' # The name of an image file (relative to this directory) to place at the # top of the sidebar. -# html_logo = None +html_logo = 'img/EnPT_Logo_transparent.png' # The name of an image file (within the static path) to use as favicon # of the docs. This file should be a Windows icon file (.ico) being diff --git a/docs/img/EnPT_Logo.png b/docs/img/EnPT_Logo.png new file mode 100644 index 0000000000000000000000000000000000000000..f45a41cadaaadb085c8d755260885edaf4f95140 Binary files /dev/null and b/docs/img/EnPT_Logo.png differ diff --git a/docs/img/EnPT_Logo_clipped.png b/docs/img/EnPT_Logo_clipped.png new file mode 100644 index 0000000000000000000000000000000000000000..494d0ae892712ac6b63e51eab6968dd7b95cc5d4 Binary files /dev/null and b/docs/img/EnPT_Logo_clipped.png differ diff --git a/docs/img/EnPT_Logo_transparent.png b/docs/img/EnPT_Logo_transparent.png new file mode 100644 index 0000000000000000000000000000000000000000..25ca62210ef19b010c666a4b6c121d08369c5e37 Binary files /dev/null and b/docs/img/EnPT_Logo_transparent.png differ diff --git a/docs/img/screenshot_enpt_enmapboxapp_874x1047.png b/docs/img/screenshot_enpt_enmapboxapp_874x1047.png new file mode 100644 index 0000000000000000000000000000000000000000..07885e6bebb00ac6c5920d5e9857817aceadda75 Binary files /dev/null and b/docs/img/screenshot_enpt_enmapboxapp_874x1047.png differ diff --git a/docs/index.rst b/docs/index.rst index 69597fa708c0f5925f756bda3e7d90cd1e22b4ce..0bc36ef1eaea9c6cd34d90c52d15862cc8d010be 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,17 +1,16 @@ -.. image:: img/logo3.jpeg - -Welcome to EnPT's documentation! -====================================== +EnPT documentation +================== -Contents: +**This documentation is structured as follows:** .. toctree:: :maxdepth: 2 - readme + about installation usage + algorithm_descriptions.rst modules contributing development diff --git a/docs/installation.rst b/docs/installation.rst index 1164669214c59ecd634048124e73c0b4bcbf21ee..e08b7db113826205c1e598ceec4d905c8614f718 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -1,51 +1,54 @@ -.. highlight:: shell - ============ Installation ============ +EnPT depends on some open source packages which are usually installed without problems by the automatic install +routine. However, for some projects, we strongly recommend resolving the dependency before the automatic installer +is run. This approach avoids problems with conflicting versions of the same software. +Using conda_, the recommended approach is: -Stable release --------------- - -To install EnPT, run this command in your terminal: - -.. code-block:: console - - $ pip install enpt - -This is the preferred method to install EnPT, as it will always install the most recent stable release. +.. _conda: https://conda.io/docs/ -If you don't have `pip`_ installed, this `Python installation guide`_ can guide -you through the process. +.. code-block:: bash -.. _pip: https://pip.pypa.io -.. _Python installation guide: http://docs.python-guide.org/en/latest/starting/installation/ + # create virtual environment for enpt, this is optional but recommended + conda create --name enpt python=3 + source activate enpt + # clone the source code of enpt + git clone https://gitext.gfz-potsdam.de/EnMAP/GFZ_Tools_EnMAP_BOX/EnPT.git + cd EnPT -From sources ------------- + # install some enpt dependencies that may cause trouble when installed via pip + conda install -c conda-forge scipy -The sources for EnPT can be downloaded from the `Github repo`_. + # install not pip-installable deps of geoarray + conda install -c conda-forge numpy scikit-image matplotlib pandas gdal rasterio pyproj basemap shapely + conda install -c conda-forge 'icu=58.*' -You can either clone the public repository: + # install py_tools_ds + conda install -c conda-forge pyresample + pip install 'py_tools_ds>=0.14.14' -.. code-block:: console + # install sicor + conda install -c conda-forge pygrib h5py pytables pyfftw numba llvmlite scikit-learn + rm -rf context/sicor + git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git ./context/sicor + cd ./context/sicor + make install + cd ../../ - $ git clone git://github.com/audreyr/enpt + # install enpt + make install + cd .. + pwd + ls -Or download the `tarball`_: -.. code-block:: console - - $ curl -OL https://github.com/audreyr/enpt/tarball/master - -Once you have a copy of the source, you can install it with: - -.. code-block:: console - - $ python setup.py install +This is the preferred method to install EnPT, as it will always install the most recent stable release. +If you don't have `pip`_ installed, this `Python installation guide`_ can guide +you through the process. -.. _Github repo: https://github.com/audreyr/enpt -.. _tarball: https://github.com/audreyr/enpt/tarball/master +.. _pip: https://pip.pypa.io +.. _Python installation guide: http://docs.python-guide.org/en/latest/starting/installation/ diff --git a/docs/modules.rst b/docs/modules.rst deleted file mode 100644 index 2e7e368a877bbd0a1a3abbc680857cbacc1c541b..0000000000000000000000000000000000000000 --- a/docs/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -enpt -==== - -.. toctree:: - :maxdepth: 4 - - enpt diff --git a/docs/usage.rst b/docs/usage.rst index 0fe9b4d87f44b2d7ba513725fd08fa9611f12858..1176105d192686a1fd9790b801179bf43d6cf1b3 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1,9 +1,118 @@ Usage ===== -To use EnPT in a project:: +Usage of the Python API +*********************** + +**Run the whole pipeline at once** + +To run the whole EnPT processing pipeline via the Python API: + +.. code-block:: python + + from enpt.execution.controller import EnPT_Controller + + config_minimal = dict( + path_l1b_enmap_image='/path/ENMAP*L1B*.zip', + path_dem='/path/to/overlapping/DEM.bsq' + ) + CTR = EnPT_Controller(**config_minimal) + CTR.run_all_processors() + +Further *configuration parameters* are documented +`here `__. +Note that the class `EnPTConfig` takes the same keyword arguments like the `EnPT_Controller` class. + +You can also pass a *JSON-File with your EnPT configuration* to the `EnPT_Controller` class. This allows you to easily +copy and reuse configuration files. A template with all possible options and defaults can be found in +`enpt/options/options_default.json`_. + +The corresponding Python call looks like this: + +.. code-block:: python + + from enpt.execution.controller import EnPT_Controller + from enpt.options.config import EnPTConfig + + CFG = EnPTConfig(json_config='/path/to/your/config_file.json') + CTR = EnPT_Controller(config=CFG) + CTR.run_all_processors() + + +**Run each processor module separately** + +To run the different processor modules separately, follow the instructions below. +Note, that the processors are built on each other, so the order must be unchanged. + +1. To get an instance of *EnPT_Controller*: + + .. code-block:: python + + CTR = EnPT_Controller(**your_config_parameters) # config parameters are described above + +2. *Read the EnMAP Level-1B data* into memory, convert DNs to radiance and compute coordiante layers: + + .. code-block:: python + + CTR.read_L1B_data() + + Get the in-memory Python object representing the Level-1 dataset (NOTE: the attributes and methods of this + Python object are documented + `here `__): + + .. code-block:: python + + L1_obj = CTR.L1_obj + +3. To correct *dead pixels*: + + .. code-block:: python + + CTR.L1_obj.correct_dead_pixels() + +4. *reflectance* conversion + + To run the *atmospheric correction* using SICOR_: + + .. code-block:: python + + CTR.run_atmospheric_correction() + + To run convert top-of-atmosphere radiance to *top-of-atmosphere reflectance* (as an alternative to atmospheric correction): + + .. code-block:: python + + CTR.run_toaRad2toaRef() + +5. To compute *spatial misregistration* with respect to a user-provided reference dataset + (NOTE: This feature is yet implemented): + + .. code-block:: python + + CTR.run_geometry_processor() + +6. To run the *orthorectification*: + + .. code-block:: python + + self.run_orthorectification() + + From now on, you may get the in-memory EnMAP Level-2 Python object (attributes and methods are documented + `here `__): + + .. code-block:: python + + L2_obj = CTR.L2_obj + +7. To *write the EnMAP Level-2 data* to disk: + + .. code-block:: python + + L2_obj = CTR.write_output() + + +---- - import enpt Command line utilities ********************** @@ -17,3 +126,21 @@ At the command line, EnPT provides the **enpt_cli.py** command: :filename: ./../bin/enpt_cli.py :func: get_enpt_argparser :prog: enpt_cli.py + + +---- + + +QGIS GUI +******** + +There is a separate graphical user interface (GUI) for EnPT than can be installed as an EnMAP-Box application in QGIS. +To install it in QGIS, please refer to the separate repository enpt_enmapboxapp_. + +Here is screenshot of the current version: + +.. image:: img/screenshot_enpt_enmapboxapp_874x1047.png + +.. _enpt_enmapboxapp: https://gitext.gfz-potsdam.de/EnMAP/GFZ_Tools_EnMAP_BOX/enpt_enmapboxapp +.. _enpt/options/options_default.json: https://gitext.gfz-potsdam.de/EnMAP/GFZ_Tools_EnMAP_BOX/EnPT/blob/master/enpt/options/options_default.json +.. _SICOR: https://gitext.gfz-potsdam.de/EnMAP/sicor \ No newline at end of file diff --git a/enpt/options/config.py b/enpt/options/config.py index 14f0f8f8e6f26dfaca24c8fdb959bc5a889213dc..923825f510698844279d3a2222437b38385a1dc9 100644 --- a/enpt/options/config.py +++ b/enpt/options/config.py @@ -80,8 +80,92 @@ class EnPTConfig(object): def __init__(self, json_config='', **user_opts): """Create a job configuration. - :param json_config path to JSON file containing configuration parameters or a string in JSON format - :param user_opts keyword arguments + :arg json_config: + path to JSON file containing configuration parameters or a string in JSON format + + :key CPUs: + number of CPU cores to be used for processing (default: "None" -> use all available + + :key path_l1b_enmap_image: + input path of the EnMAP L1B image to be processed + (zip-archive or root directory; must be given if not contained in --json-config.) + + :key path_l1b_enmap_image_gapfill: + input path of an adjacent EnMAP L1B image to be used for gap-filling (zip-archive or root directory) + + :key path_dem: + input path of digital elevation model in map or sensor geometry; GDAL compatible file format (must cover + the EnMAP L1B data completely if given in map geometry or must have the same pixel dimensions like the + EnMAP L1B data if given in sensor geometry) + + :key average_elevation: + average elevation in meters above sea level; may be provided if no DEM is available; ignored if DEM is given + + :key output_dir: + output directory where processed data and log files are saved + + :key working_dir: + directory to be used for temporary files + + :key n_lines_to_append: + number of lines to be added to the main image [if None, use the whole imgap]. + Requires 'path_l1b_enmap_image_gapfill' to be set. + + :key disable_progress_bars: + whether to disable all progress bars during processing + + :key path_earthSunDist: + input path of the earth sun distance model + + :key path_solar_irr: + input path of the solar irradiance model + + :key scale_factor_toa_ref: + scale factor to be applied to TOA reflectance result + + :key enable_keystone_correction: + Enable keystone correction + + :key enable_vnir_swir_coreg: + Enable VNIR/SWIR co-registration + + :key path_reference_image: + Reference image for co-registration. + + :key enable_ac: + Enable atmospheric correction using SICOR algorithm (default: True). + If False, the L2A output contains top-of-atmosphere reflectance. + + :key sicor_cache_dir: + SICOR cache directory + + :key auto_download_ecmwf: + Automatically download ECMWF data for atmospheric correction + + :key enable_cloud_screening: + Enable cloud screening during atmospheric correction + + :key scale_factor_boa_ref: + Scale factor to be applied to BOA reflectance result + + :key run_smile_P: + Enable extra smile detection and correction (provider smile coefficients are ignored) + + :key run_deadpix_P: + Enable dead pixel correction + + :key deadpix_P_algorithm: + Algorithm for dead pixel correction ('spectral' or 'spatial') + + :key deadpix_P_interp_spectral: + Spectral interpolation algorithm to be used during dead pixel correction + ('linear', 'nearest', 'zero', 'slinear', 'quadratic', 'cubic') + + :key deadpix_P_interp_spatial: + Spatial interpolation algorithm to be used during dead pixel correction + ('linear', 'nearest', 'zero', 'slinear', 'quadratic', 'cubic') + :key ortho_resampAlg: + Ortho-rectification resampling algorithm ('nearest', 'bilinear', 'gauss') """ # fixed attributes diff --git a/enpt/processors/dead_pixel_correction/dead_pixel_correction.py b/enpt/processors/dead_pixel_correction/dead_pixel_correction.py index 3dae81667e3ca5568554f239e43d823e8b98d025..b5848619817ecc21f7152d29a9da4cd75b6a64e2 100644 --- a/enpt/processors/dead_pixel_correction/dead_pixel_correction.py +++ b/enpt/processors/dead_pixel_correction/dead_pixel_correction.py @@ -16,6 +16,18 @@ from geoarray import GeoArray class Dead_Pixel_Corrector(object): + """ + The EnPT dead pixel correction uses the pixel masks provided by DLR and interpolates the EnMAP image + data at the indicated dead pixel positions. It supports two interpolation algorithms: + + 1. spectral interpolation + * Interpolates the data in the spectral domain. + * Points outside the data range are extrapolated. + 2. spatial interpolation + * Interpolates the data spatially. + * Remaining missing data positions (e.g., outermost columns) are spectrally interpolated. + """ + def __init__(self, algorithm: str = 'spectral', interp_spectral: str = 'linear', interp_spatial: str = 'linear', CPUs: int = None, logger: logging.Logger = None): """Get an instance of Dead_Pixel_Corrector.