From 29c305e656ba1ab018d24c828ebe1dcbe1160bf5 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Wed, 11 Apr 2018 13:48:09 +0200 Subject: [PATCH 01/31] Implemented SICOR (not yet working). --- enpt/execution/controller.py | 2 +- enpt/model/images.py | 5 +++ enpt/processors/__init__.py | 8 +++++ .../atmospheric_correction/__init__.py | 4 +++ .../atmospheric_correction.py | 34 +++++++++++++++++++ .../radiometric_transform/__init__.py | 4 --- enpt/utils/path_generator.py | 8 +++++ tests/test_radiometric_transform.py | 2 +- 8 files changed, 61 insertions(+), 6 deletions(-) diff --git a/enpt/execution/controller.py b/enpt/execution/controller.py index d8d051b..9466f54 100644 --- a/enpt/execution/controller.py +++ b/enpt/execution/controller.py @@ -81,7 +81,7 @@ class EnPT_Controller(object): def run_atmospheric_correction(self): """Run atmospheric correction only.""" - pass + self.L1_obj.run_AC() def write_output(self): if self.cfg.output_dir: diff --git a/enpt/model/images.py b/enpt/model/images.py index a6551ed..1fc3b28 100644 --- a/enpt/model/images.py +++ b/enpt/model/images.py @@ -472,6 +472,11 @@ class EnMAPL1Product_SensorGeo(object): if self.swir.detector_meta.unitcode != 'TOARad': self.swir.DN2TOARadiance() + def run_AC(self): + from ..processors import AtmosphericCorrector + AC = AtmosphericCorrector(config=self.cfg) + AC.run_ac(self) + def save(self, outdir: str, suffix="") -> str: """Save this product to disk using almost the same format as for reading. diff --git a/enpt/processors/__init__.py b/enpt/processors/__init__.py index ffcaa00..5ce1f34 100644 --- a/enpt/processors/__init__.py +++ b/enpt/processors/__init__.py @@ -1,2 +1,10 @@ # -*- coding: utf-8 -*- """EnPT 'processors' module containing all EnPT processor sub-modules.""" + +from .radiometric_transform.radiometric_transform import Radiometric_Transformer +from .atmospheric_correction.atmospheric_correction import AtmosphericCorrector + +__all__ = [ + "Radiometric_Transformer", + "AtmosphericCorrector" +] diff --git a/enpt/processors/atmospheric_correction/__init__.py b/enpt/processors/atmospheric_correction/__init__.py index b613402..a60f20a 100644 --- a/enpt/processors/atmospheric_correction/__init__.py +++ b/enpt/processors/atmospheric_correction/__init__.py @@ -1,2 +1,6 @@ # -*- coding: utf-8 -*- """EnPT 'atmospheric correction module.""" + +from .atmospheric_correction import AtmosphericCorrector + +__all__ = ['AtmosphericCorrector'] \ No newline at end of file diff --git a/enpt/processors/atmospheric_correction/atmospheric_correction.py b/enpt/processors/atmospheric_correction/atmospheric_correction.py index a55ee17..803cc4d 100644 --- a/enpt/processors/atmospheric_correction/atmospheric_correction.py +++ b/enpt/processors/atmospheric_correction/atmospheric_correction.py @@ -3,3 +3,37 @@ Performs the atmospheric correction of EnMAP L1B data. """ + +from sicor.sicor_enmap import sicor_ac_enmap +from sicor.options import get_options as get_ac_options + +from ...model.images import EnMAPL1Product_SensorGeo +from ...options.config import EnPTConfig +from ...utils.path_generator import get_path_ac_options + + +class AtmosphericCorrector(object): + """Class for performing atmospheric correction of EnMAP L1 images using SICOR.""" + + def __init__(self, config: EnPTConfig=None): + """Create an instance of AtmosphericCorrector.""" + self.cfg = config + + @property + def options(self): + path_opts = get_path_ac_options() + + try: + return get_ac_options(path_opts) + except FileNotFoundError: + raise FileNotFoundError('Could not locate options file for atmospheric correction at %s.' % path_opts) + + def run_ac(self, enmap_ImageL1: EnMAPL1Product_SensorGeo): + enmap_ImageL1.logger.info("Starting atmospheric correction for VNIR and SWIR detector. " + "Source radiometric unit code is '%s'." % enmap_ImageL1.meta.vnir.unitcode) + + # run AC + enmap_l2a_sens_geo, state, fits = sicor_ac_enmap(enmap_l1b=enmap_ImageL1, options=self.options, + logger=enmap_ImageL1.logger, debug=True) + + return enmap_l2a_sens_geo diff --git a/enpt/processors/radiometric_transform/__init__.py b/enpt/processors/radiometric_transform/__init__.py index c0335e5..53c4849 100644 --- a/enpt/processors/radiometric_transform/__init__.py +++ b/enpt/processors/radiometric_transform/__init__.py @@ -1,6 +1,2 @@ # -*- coding: utf-8 -*- """EnPT 'radiometric transform' module containing eveything related to radiometric transformations.""" - -from .radiometric_transform import Radiometric_Transformer - -__all__ = ['Radiometric_Transformer'] diff --git a/enpt/utils/path_generator.py b/enpt/utils/path_generator.py index 0b8c6f0..ec3ed85 100644 --- a/enpt/utils/path_generator.py +++ b/enpt/utils/path_generator.py @@ -48,3 +48,11 @@ class PathGenL1BProduct(object): def _find_in_metaxml(self, expression): return self.xml.findall(expression)[0].text.replace("\n", "").strip() + + +def get_path_ac_options() -> str: + """Returns the path of the options json file needed for atmospheric correction.""" + from sicor import options + path_ac = os.path.join(os.path.dirname(options.__file__), 'enmap_options.json') + + return path_ac diff --git a/tests/test_radiometric_transform.py b/tests/test_radiometric_transform.py index 61c8a0a..ccf9270 100644 --- a/tests/test_radiometric_transform.py +++ b/tests/test_radiometric_transform.py @@ -7,7 +7,7 @@ import tempfile import zipfile from datetime import datetime -from enpt.processors.radiometric_transform import Radiometric_Transformer +from enpt.processors import Radiometric_Transformer from enpt.options.config import EnPTConfig from . import config_for_testing -- GitLab From ae8ded30df88da51a3f27f08cc06b17376c98689 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Wed, 11 Apr 2018 14:49:47 +0200 Subject: [PATCH 02/31] Revised implementation of SICOR. Added sicor to package dependencies. Updated CI docker image recipe. --- .gitlab-ci.yml | 10 +++++ enpt/execution/controller.py | 2 +- .../atmospheric_correction.py | 44 +++++++++++++++---- enpt/utils/path_generator.py | 12 +++++ requirements.txt | 1 + setup.py | 1 + .../build_enpt_testsuite_image.sh | 4 ++ tests/gitlab_CI_docker/context/enpt_ci.docker | 11 +++++ .../context/environment_enpt.yml | 14 +++++- 9 files changed, 88 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8a0f230..342b217 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,16 @@ test_enpt: - source /root/miniconda3/bin/activate enpt - export GDAL_DATA=/root/miniconda3/envs/enpt/share/gdal - export PYTHONPATH=$PYTHONPATH:/root # /root <- here are the sicor tables + + # update sicor + # - conda install -y -q -c conda-forge basemap + # - rm -rf context/sicor + # - git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git ./context/sicor + # - cd ./context/sicor + # - make download-tables + # - python setup.py install + # - cd ../../ + # run nosetests - make nosetests # test are called here # create the docs diff --git a/enpt/execution/controller.py b/enpt/execution/controller.py index 9466f54..3030d8f 100644 --- a/enpt/execution/controller.py +++ b/enpt/execution/controller.py @@ -11,7 +11,7 @@ import warnings from ..options.config import EnPTConfig from ..io.reader import L1B_Reader -from ..processors.radiometric_transform import Radiometric_Transformer +from ..processors import Radiometric_Transformer from ..model.images import EnMAPL1Product_SensorGeo diff --git a/enpt/processors/atmospheric_correction/atmospheric_correction.py b/enpt/processors/atmospheric_correction/atmospheric_correction.py index 803cc4d..fea79f8 100644 --- a/enpt/processors/atmospheric_correction/atmospheric_correction.py +++ b/enpt/processors/atmospheric_correction/atmospheric_correction.py @@ -3,13 +3,15 @@ Performs the atmospheric correction of EnMAP L1B data. """ +import tempfile +import pprint from sicor.sicor_enmap import sicor_ac_enmap from sicor.options import get_options as get_ac_options from ...model.images import EnMAPL1Product_SensorGeo from ...options.config import EnPTConfig -from ...utils.path_generator import get_path_ac_options +from ...utils.path_generator import get_path_ac_options, get_path_ac_aerosol_table, get_path_ac_ch4_table class AtmosphericCorrector(object): @@ -19,21 +21,45 @@ class AtmosphericCorrector(object): """Create an instance of AtmosphericCorrector.""" self.cfg = config - @property - def options(self): + @staticmethod + def get_ac_options(buffer_dir): path_opts = get_path_ac_options() try: - return get_ac_options(path_opts) + options = get_ac_options(path_opts) + + # adjust options + options['EnMAP']['buffer_dir'] = buffer_dir + for vv in options["RTFO"].values(): + vv["hash_formats"] = dict(spr='%.0f', + coz='%.0f,', + cwv='%.0f,', + tmp='%0f,', + tau_a='%.2f,', + vza='%.0f,') + options["ECMWF"]["path_db"] = "./ecmwf" + path_tbl_aerosol = get_path_ac_aerosol_table() + path_tbl_ch4 = get_path_ac_ch4_table() + for name, val in options["RTFO"].items(): + if "aerosol" in name: + val['atm_tables_fn'] = path_tbl_aerosol + if "ch4" in name: + val['atm_tables_fn'] = path_tbl_ch4 + + return options + except FileNotFoundError: raise FileNotFoundError('Could not locate options file for atmospheric correction at %s.' % path_opts) def run_ac(self, enmap_ImageL1: EnMAPL1Product_SensorGeo): - enmap_ImageL1.logger.info("Starting atmospheric correction for VNIR and SWIR detector. " - "Source radiometric unit code is '%s'." % enmap_ImageL1.meta.vnir.unitcode) + with tempfile.TemporaryDirectory(self.cfg.working_dir) as buffer_dir: + options = self.get_ac_options(buffer_dir) + enmap_ImageL1.logger.debug('AC options: \n' + pprint.pformat(options)) - # run AC - enmap_l2a_sens_geo, state, fits = sicor_ac_enmap(enmap_l1b=enmap_ImageL1, options=self.options, - logger=enmap_ImageL1.logger, debug=True) + # run AC + enmap_ImageL1.logger.info("Starting atmospheric correction for VNIR and SWIR detector. " + "Source radiometric unit code is '%s'." % enmap_ImageL1.meta.vnir.unitcode) + enmap_l2a_sens_geo, state, fits = sicor_ac_enmap(enmap_l1b=enmap_ImageL1, options=options, + logger=enmap_ImageL1.logger, debug=True) return enmap_l2a_sens_geo diff --git a/enpt/utils/path_generator.py b/enpt/utils/path_generator.py index ec3ed85..c5762a6 100644 --- a/enpt/utils/path_generator.py +++ b/enpt/utils/path_generator.py @@ -56,3 +56,15 @@ def get_path_ac_options() -> str: path_ac = os.path.join(os.path.dirname(options.__file__), 'enmap_options.json') return path_ac + + +def get_path_ac_aerosol_table(): + import sicor + return os.path.join(sicor.__path__[0], 'tables', + 'linear_atm_functions_ncwv_5_npre_4_ncoz_2_ntmp_2_wvl_350.0_2550.0_1.00_pca.h5') + + +def get_path_ac_ch4_table(): + import sicor + return os.path.join(sicor.__path__[0], 'tables', + 'linear_atm_functions_ncwv_4_npre_2_ncoz_2_ntmp_1_nch4_4_wvl_350.0_2550.0_1.00_pca.h5') diff --git a/requirements.txt b/requirements.txt index 74df5e2..0688135 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ spectral>=0.16 cerberus jsmin matplotlib +git+https://gitext.gfz-potsdam.de/EnMAP/sicor.git diff --git a/setup.py b/setup.py index 685e018..2176478 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,7 @@ with open("enpt/version.py") as version_file: requirements = [ # put package requirements here 'numpy', 'scipy', 'geoarray>=0.7.15', 'spectral>=0.16', 'cerberus', 'jsmin', 'matplotlib' + # 'sicor', # pip install git+https://gitext.gfz-potsdam.de/EnMAP/sicor.git ] test_requirements = ['coverage', 'nose', 'nose-htmloutput', 'rednose'] diff --git a/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh b/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh index 1119454..c5fc8f2 100755 --- a/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh +++ b/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh @@ -5,6 +5,10 @@ dockerfile="enpt_ci.docker" tag="enpt_ci:latest" gitlab_runner="enpt_gitlab_CI_runner" +# get sicor project +rm -rf context/sicor +git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git ./context/sicor + echo "#### Build runner docker image" sudo docker rmi ${tag} sudo docker build -f ${context_dir}/${dockerfile} -m 20G -t ${tag} ${context_dir} diff --git a/tests/gitlab_CI_docker/context/enpt_ci.docker b/tests/gitlab_CI_docker/context/enpt_ci.docker index e9d1020..ddb3b32 100644 --- a/tests/gitlab_CI_docker/context/enpt_ci.docker +++ b/tests/gitlab_CI_docker/context/enpt_ci.docker @@ -30,3 +30,14 @@ RUN /bin/bash -i -c "wget https://github.com/git-lfs/git-lfs/releases/download/v tar -zxvf git-lfs-linux-amd64-${git_lfs_v}.tar.gz; \ cd git-lfs-${git_lfs_v}; \ bash ./install.sh" + +# copy sicor code to /tmp +COPY sicor /tmp/sicor + +# install sicor +RUN bash -i -c "source /root/miniconda3/bin/activate; \ + cd /tmp/sicor/ ; \ + make clean ; \ + make requirements ; \ + make download-tables ; \ + pip install . --no-cache-dir" diff --git a/tests/gitlab_CI_docker/context/environment_enpt.yml b/tests/gitlab_CI_docker/context/environment_enpt.yml index f1b7bbf..dc66d5b 100644 --- a/tests/gitlab_CI_docker/context/environment_enpt.yml +++ b/tests/gitlab_CI_docker/context/environment_enpt.yml @@ -22,6 +22,15 @@ dependencies: - shapely - holoviews - bokeh + + # sicor + - glymur + - pygrib + - cachetools + - pyhdf + - h5py + - pytables + - pip: - scipy - geoarray>=0.7.15 @@ -37,4 +46,7 @@ dependencies: - nose2 - nose-htmloutput - coverage - - rednose \ No newline at end of file + - rednose + + # sicor + - https://software.ecmwf.int/wiki/download/attachments/56664858/ecmwf-api-client-python.tgz \ No newline at end of file -- GitLab From 219a0596966780cbc3a914d553bf37ce2c0a2365 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Wed, 11 Apr 2018 14:54:00 +0200 Subject: [PATCH 03/31] Changed CI docker image tag. --- tests/gitlab_CI_docker/build_enpt_testsuite_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh b/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh index c5fc8f2..39a0381 100755 --- a/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh +++ b/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh @@ -2,7 +2,7 @@ context_dir="./context" dockerfile="enpt_ci.docker" -tag="enpt_ci:latest" +tag="enpt_ci:0.3.2" gitlab_runner="enpt_gitlab_CI_runner" # get sicor project -- GitLab From 8009c38df3f326274a2e2807a9381117d3370de8 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Wed, 11 Apr 2018 17:25:40 +0200 Subject: [PATCH 04/31] Updated CI docker image recipe. --- tests/gitlab_CI_docker/context/enpt_ci.docker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/gitlab_CI_docker/context/enpt_ci.docker b/tests/gitlab_CI_docker/context/enpt_ci.docker index ddb3b32..78c5a74 100644 --- a/tests/gitlab_CI_docker/context/enpt_ci.docker +++ b/tests/gitlab_CI_docker/context/enpt_ci.docker @@ -35,7 +35,7 @@ RUN /bin/bash -i -c "wget https://github.com/git-lfs/git-lfs/releases/download/v COPY sicor /tmp/sicor # install sicor -RUN bash -i -c "source /root/miniconda3/bin/activate; \ +RUN bash -i -c "source /root/miniconda3/bin/activate enpt; \ cd /tmp/sicor/ ; \ make clean ; \ make requirements ; \ -- GitLab From 26db1ffc25151eadfd04dbf8035a5d8bbe8f0a12 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Wed, 11 Apr 2018 18:05:10 +0200 Subject: [PATCH 05/31] Added dependencies of arosics to environment_enpt.yml. --- tests/gitlab_CI_docker/context/environment_enpt.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/gitlab_CI_docker/context/environment_enpt.yml b/tests/gitlab_CI_docker/context/environment_enpt.yml index dc66d5b..7863817 100644 --- a/tests/gitlab_CI_docker/context/environment_enpt.yml +++ b/tests/gitlab_CI_docker/context/environment_enpt.yml @@ -23,7 +23,11 @@ dependencies: - holoviews - bokeh - # sicor + # arosics + - pyfftw + - pykrige + + # sicor - glymur - pygrib - cachetools -- GitLab From 76e03c2319aa779e5fcb90160bb7ae724e4ee65e Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Wed, 11 Apr 2018 19:08:08 +0200 Subject: [PATCH 06/31] Fixed usage of wrong sicor options file. Added options parameter 'sicor_cache_dir'. --- enpt/execution/controller.py | 6 +- enpt/options/config.py | 3 + enpt/options/options_default.json | 3 + enpt/options/options_schema.py | 2 + .../atmospheric_correction/__init__.py | 2 +- .../atmospheric_correction.py | 27 +++----- enpt/utils/path_generator.py | 14 +--- tests/linting/pydocstyle.log | 64 +++++++++++++------ 8 files changed, 67 insertions(+), 54 deletions(-) diff --git a/enpt/execution/controller.py b/enpt/execution/controller.py index 3030d8f..64a3659 100644 --- a/enpt/execution/controller.py +++ b/enpt/execution/controller.py @@ -79,6 +79,9 @@ class EnPT_Controller(object): # run transformation to TOARef self.L1_obj = RT.transform_TOARad2TOARef(self.L1_obj) + def run_geometry_processor(self): + pass + def run_atmospheric_correction(self): """Run atmospheric correction only.""" self.L1_obj.run_AC() @@ -90,7 +93,8 @@ class EnPT_Controller(object): def run_all_processors(self): """Run all processors at once.""" try: - self.run_toaRad2toaRef() + # self.run_toaRad2toaRef() + self.run_geometry_processor() self.run_atmospheric_correction() self.write_output() finally: diff --git a/enpt/options/config.py b/enpt/options/config.py index c20b9ba..c23ed60 100644 --- a/enpt/options/config.py +++ b/enpt/options/config.py @@ -17,6 +17,8 @@ from collections import OrderedDict, Mapping import numpy as np from multiprocessing import cpu_count +import sicor + from .options_schema import \ enpt_schema_input, \ enpt_schema_config_output, \ @@ -88,6 +90,7 @@ class EnPTConfig(object): self.path_reference_image = gp('path_reference_image') # atmospheric_correction + self.sicor_cache_dir = gp('sicor_cache_dir', fallback=sicor.__path__[0]) self.auto_download_ecmwf = gp('auto_download_ecmwf') self.enable_cloud_screening = gp('enable_cloud_screening') diff --git a/enpt/options/options_default.json b/enpt/options/options_default.json index fc9476f..0cac801 100644 --- a/enpt/options/options_default.json +++ b/enpt/options/options_default.json @@ -26,6 +26,9 @@ }, "atmospheric_correction": { + "sicor_cache_dir": "", /*directory to be used to stored sicor cache files + NOTE: SICOR stores intermediate results there that need to computed only once + for atmospheric correction of multiple EnMAP images. (default: 'auto'*/ "auto_download_ecmwf": false, "enable_cloud_screening": false }, diff --git a/enpt/options/options_schema.py b/enpt/options/options_schema.py index b10a715..3096a83 100644 --- a/enpt/options/options_schema.py +++ b/enpt/options/options_schema.py @@ -41,6 +41,7 @@ enpt_schema_input = dict( atmospheric_correction=dict( type='dict', required=False, schema=dict( + sicor_cache_dir=dict(type='string', required=False), auto_download_ecmwf=dict(type='boolean', required=False), enable_cloud_screening=dict(type='boolean', required=False), )), @@ -89,6 +90,7 @@ parameter_mapping = dict( path_reference_image=('processors', 'geometry', 'path_reference_image'), # processors > atmospheric_correction + sicor_cache_dir=('processors', 'atmospheric_correction', 'sicor_cache_dir'), auto_download_ecmwf=('processors', 'atmospheric_correction', 'auto_download_ecmwf'), enable_cloud_screening=('processors', 'atmospheric_correction', 'enable_cloud_screening'), diff --git a/enpt/processors/atmospheric_correction/__init__.py b/enpt/processors/atmospheric_correction/__init__.py index a60f20a..7e1cd01 100644 --- a/enpt/processors/atmospheric_correction/__init__.py +++ b/enpt/processors/atmospheric_correction/__init__.py @@ -3,4 +3,4 @@ from .atmospheric_correction import AtmosphericCorrector -__all__ = ['AtmosphericCorrector'] \ No newline at end of file +__all__ = ['AtmosphericCorrector'] diff --git a/enpt/processors/atmospheric_correction/atmospheric_correction.py b/enpt/processors/atmospheric_correction/atmospheric_correction.py index fea79f8..0e4634d 100644 --- a/enpt/processors/atmospheric_correction/atmospheric_correction.py +++ b/enpt/processors/atmospheric_correction/atmospheric_correction.py @@ -3,7 +3,6 @@ Performs the atmospheric correction of EnMAP L1B data. """ -import tempfile import pprint from sicor.sicor_enmap import sicor_ac_enmap @@ -11,7 +10,7 @@ from sicor.options import get_options as get_ac_options from ...model.images import EnMAPL1Product_SensorGeo from ...options.config import EnPTConfig -from ...utils.path_generator import get_path_ac_options, get_path_ac_aerosol_table, get_path_ac_ch4_table +from ...utils.path_generator import get_path_ac_options class AtmosphericCorrector(object): @@ -38,13 +37,6 @@ class AtmosphericCorrector(object): tau_a='%.2f,', vza='%.0f,') options["ECMWF"]["path_db"] = "./ecmwf" - path_tbl_aerosol = get_path_ac_aerosol_table() - path_tbl_ch4 = get_path_ac_ch4_table() - for name, val in options["RTFO"].items(): - if "aerosol" in name: - val['atm_tables_fn'] = path_tbl_aerosol - if "ch4" in name: - val['atm_tables_fn'] = path_tbl_ch4 return options @@ -52,14 +44,13 @@ class AtmosphericCorrector(object): raise FileNotFoundError('Could not locate options file for atmospheric correction at %s.' % path_opts) def run_ac(self, enmap_ImageL1: EnMAPL1Product_SensorGeo): - with tempfile.TemporaryDirectory(self.cfg.working_dir) as buffer_dir: - options = self.get_ac_options(buffer_dir) - enmap_ImageL1.logger.debug('AC options: \n' + pprint.pformat(options)) - - # run AC - enmap_ImageL1.logger.info("Starting atmospheric correction for VNIR and SWIR detector. " - "Source radiometric unit code is '%s'." % enmap_ImageL1.meta.vnir.unitcode) - enmap_l2a_sens_geo, state, fits = sicor_ac_enmap(enmap_l1b=enmap_ImageL1, options=options, - logger=enmap_ImageL1.logger, debug=True) + options = self.get_ac_options(buffer_dir=self.cfg.sicor_cache_dir) + enmap_ImageL1.logger.debug('AC options: \n' + pprint.pformat(options)) + + # run AC + enmap_ImageL1.logger.info("Starting atmospheric correction for VNIR and SWIR detector. " + "Source radiometric unit code is '%s'." % enmap_ImageL1.meta.vnir.unitcode) + enmap_l2a_sens_geo, state, fits = sicor_ac_enmap(enmap_l1b=enmap_ImageL1, options=options, + logger=enmap_ImageL1.logger, debug=True) return enmap_l2a_sens_geo diff --git a/enpt/utils/path_generator.py b/enpt/utils/path_generator.py index c5762a6..2f74904 100644 --- a/enpt/utils/path_generator.py +++ b/enpt/utils/path_generator.py @@ -53,18 +53,6 @@ class PathGenL1BProduct(object): def get_path_ac_options() -> str: """Returns the path of the options json file needed for atmospheric correction.""" from sicor import options - path_ac = os.path.join(os.path.dirname(options.__file__), 'enmap_options.json') + path_ac = os.path.join(os.path.dirname(options.__file__), 'sicor_enmap_user_options.json') return path_ac - - -def get_path_ac_aerosol_table(): - import sicor - return os.path.join(sicor.__path__[0], 'tables', - 'linear_atm_functions_ncwv_5_npre_4_ncoz_2_ntmp_2_wvl_350.0_2550.0_1.00_pca.h5') - - -def get_path_ac_ch4_table(): - import sicor - return os.path.join(sicor.__path__[0], 'tables', - 'linear_atm_functions_ncwv_4_npre_2_ncoz_2_ntmp_1_nch4_4_wvl_350.0_2550.0_1.00_pca.h5') diff --git a/tests/linting/pydocstyle.log b/tests/linting/pydocstyle.log index 9db0162..78dbcd4 100644 --- a/tests/linting/pydocstyle.log +++ b/tests/linting/pydocstyle.log @@ -1,10 +1,12 @@ enpt/version.py:1 at module level: D100: Missing docstring in public module -enpt/model/metadata.py:185 in public method `calc_solar_irradiance_CWL_FWHM_per_band`: +enpt/model/images.py:475 in public method `run_AC`: D102: Missing docstring in public method -enpt/model/metadata.py:248 in public method `get_earth_sun_distance`: +enpt/model/metadata.py:208 in public method `calc_solar_irradiance_CWL_FWHM_per_band`: + D102: Missing docstring in public method +enpt/model/metadata.py:266 in public method `get_earth_sun_distance`: D202: No blank lines allowed after function docstring (found 1) -enpt/model/metadata.py:248 in public method `get_earth_sun_distance`: +enpt/model/metadata.py:266 in public method `get_earth_sun_distance`: D400: First line should end with a period (not ')') enpt/model/srf.py:11 in public class `SRF`: D101: Missing docstring in public class @@ -16,45 +18,65 @@ enpt/model/srf.py:114 in public method `__getitem__`: D105: Missing docstring in magic method enpt/model/srf.py:117 in public method `__iter__`: D105: Missing docstring in magic method +enpt/utils/path_generator.py:53 in public function `get_path_ac_options`: + D401: First line should be in imperative mood ('Return', not 'Returns') +enpt/utils/logging.py:83 in public method `__getstate__`: + D105: Missing docstring in magic method +enpt/utils/logging.py:87 in public method `__setstate__`: + D401: First line should be in imperative mood ('Define', not 'Defines') +enpt/utils/logging.py:138 in public method `__del__`: + D105: Missing docstring in magic method +enpt/utils/logging.py:141 in public method `__enter__`: + D105: Missing docstring in magic method +enpt/utils/logging.py:144 in public method `__exit__`: + D105: Missing docstring in magic method +enpt/processors/atmospheric_correction/atmospheric_correction.py:24 in public method `get_ac_options`: + D102: Missing docstring in public method +enpt/processors/atmospheric_correction/atmospheric_correction.py:46 in public method `run_ac`: + D102: Missing docstring in public method enpt/execution/controller.py:50 in public method `read_L1B_data`: D205: 1 blank line required between summary line and description (found 0) enpt/execution/controller.py:50 in public method `read_L1B_data`: D400: First line should end with a period (not ':') -enpt/options/config.py:34 in public class `EnPTConfig`: +enpt/execution/controller.py:82 in public method `run_geometry_processor`: + D102: Missing docstring in public method +enpt/execution/controller.py:89 in public method `write_output`: + D102: Missing docstring in public method +enpt/options/config.py:36 in public class `EnPTConfig`: D101: Missing docstring in public class -enpt/options/config.py:35 in public method `__init__`: +enpt/options/config.py:37 in public method `__init__`: D202: No blank lines allowed after function docstring (found 1) -enpt/options/config.py:102 in public method `absPath`: +enpt/options/config.py:113 in public method `absPath`: D102: Missing docstring in public method -enpt/options/config.py:105 in public method `get_parameter`: +enpt/options/config.py:116 in public method `get_parameter`: D102: Missing docstring in public method -enpt/options/config.py:168 in public method `to_dict`: +enpt/options/config.py:179 in public method `to_dict`: D202: No blank lines allowed after function docstring (found 1) -enpt/options/config.py:182 in public method `to_jsonable_dict`: +enpt/options/config.py:193 in public method `to_jsonable_dict`: D102: Missing docstring in public method -enpt/options/config.py:193 in public method `__repr__`: +enpt/options/config.py:204 in public method `__repr__`: D105: Missing docstring in magic method -enpt/options/config.py:197 in public function `json_to_python`: +enpt/options/config.py:208 in public function `json_to_python`: D103: Missing docstring in public function -enpt/options/config.py:230 in public function `python_to_json`: +enpt/options/config.py:241 in public function `python_to_json`: D103: Missing docstring in public function -enpt/options/config.py:252 in public class `EnPTValidator`: +enpt/options/config.py:263 in public class `EnPTValidator`: D101: Missing docstring in public class -enpt/options/config.py:253 in public method `__init__`: +enpt/options/config.py:264 in public method `__init__`: D205: 1 blank line required between summary line and description (found 0) -enpt/options/config.py:253 in public method `__init__`: +enpt/options/config.py:264 in public method `__init__`: D400: First line should end with a period (not 'r') -enpt/options/config.py:261 in public method `validate`: +enpt/options/config.py:272 in public method `validate`: D102: Missing docstring in public method -enpt/options/config.py:266 in public function `get_options`: +enpt/options/config.py:277 in public function `get_options`: D202: No blank lines allowed after function docstring (found 1) enpt/options/__init__.py:1 at module level: D104: Missing docstring in public package -enpt/options/options_schema.py:93 in public function `get_updated_schema`: +enpt/options/options_schema.py:108 in public function `get_updated_schema`: D103: Missing docstring in public function -enpt/options/options_schema.py:94 in private nested function `deep_update`: +enpt/options/options_schema.py:109 in private nested function `deep_update`: D202: No blank lines allowed after function docstring (found 1) -enpt/options/options_schema.py:94 in private nested function `deep_update`: +enpt/options/options_schema.py:109 in private nested function `deep_update`: D400: First line should end with a period (not 'e') -enpt/options/options_schema.py:113 in public function `get_param_from_json_config`: +enpt/options/options_schema.py:128 in public function `get_param_from_json_config`: D103: Missing docstring in public function -- GitLab From 1b1ec8291e43428336b04ce193f5c59f78339d80 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Wed, 11 Apr 2018 19:19:00 +0200 Subject: [PATCH 07/31] Fixed missing dependency. --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 342b217..3cab1bf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,7 @@ test_enpt: - source /root/miniconda3/bin/activate enpt - export GDAL_DATA=/root/miniconda3/envs/enpt/share/gdal - export PYTHONPATH=$PYTHONPATH:/root # /root <- here are the sicor tables + - pip install xlrd # FIXME due to https://gitext.gfz-potsdam.de/EnMAP/sicor/issues/23 # update sicor # - conda install -y -q -c conda-forge basemap -- GitLab From c2d0b1773a305dc007f54aa8f977739e09d8bd8e Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Mon, 16 Apr 2018 12:18:07 +0200 Subject: [PATCH 08/31] Fixed missing mu_sun in metadata (needed by SICOR). --- enpt/model/metadata.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/enpt/model/metadata.py b/enpt/model/metadata.py index cb32a0a..2861d4b 100644 --- a/enpt/model/metadata.py +++ b/enpt/model/metadata.py @@ -65,6 +65,7 @@ class EnMAP_Metadata_L1B_Detector_SensorGeo(object): self.geom_view_azimuth = None # type: float # viewing azimuth angle self.geom_sun_zenith = None # type: float # sun zenith angle self.geom_sun_azimuth = None # type: float # sun azimuth angle + self.mu_sun = None # type: float # needed by SICOR for TOARad > TOARef conversion self.lat_UL_UR_LL_LR = None # type: list # latitude coordinates for UL, UR, LL, LR self.lon_UL_UR_LL_LR = None # type: list # longitude coordinates for UL, UR, LL, LR self.lats = None # type: np.ndarray # 2D array of latitude coordinates according to given lon/lat sampling @@ -106,6 +107,7 @@ class EnMAP_Metadata_L1B_Detector_SensorGeo(object): xml.findall("%s/illumination_geometry/zenith_angle" % lbl)[0].text.split()[0]) self.geom_sun_azimuth = np.float( xml.findall("%s/illumination_geometry/azimuth_angle" % lbl)[0].text.split()[0]) + self.mu_sun = np.cos(np.deg2rad(self.geom_sun_zenith)) self.lat_UL_UR_LL_LR = \ [float(xml.findall("%s/geometry/bounding_box/%s_northing" % (lbl, corner))[0].text.split()[0]) for corner in ("UL", "UR", "LL", "LR")] -- GitLab From 9699a405aad28be39e006658dae03bda044cd421 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Tue, 17 Apr 2018 12:07:52 +0200 Subject: [PATCH 09/31] AC results are now correctly joined. Added config parameters 'scale_factor_toa_ref' and 'scale_factor_boa_ref'. Refactored _EnMAP_Image._arr to _EnMAP_Image._data. --- enpt/model/images.py | 29 +++++++++---------- enpt/options/config.py | 2 ++ enpt/options/options_default.json | 6 ++-- enpt/options/options_schema.py | 2 ++ .../atmospheric_correction.py | 19 ++++++++++-- .../radiometric_transform.py | 8 ++--- 6 files changed, 42 insertions(+), 24 deletions(-) diff --git a/enpt/model/images.py b/enpt/model/images.py index 1fc3b28..c1020b5 100644 --- a/enpt/model/images.py +++ b/enpt/model/images.py @@ -47,7 +47,7 @@ class _EnMAP_Image(object): self.paths = SimpleNamespace() # protected attributes - self._arr = None + self._data = None self._mask_nodata = None self._mask_clouds = None self._mask_clouds_confidence = None @@ -61,7 +61,7 @@ class _EnMAP_Image(object): self.basename = '' @property - def data(self): + def data(self) -> GeoArray: """Return the actual EnMAP image data. Bundled with all the corresponding metadata. @@ -105,26 +105,25 @@ class _EnMAP_Image(object): :return: instance of geoarray.GeoArray """ - # TODO this must return a subset if self.subset is not None - return self._arr + return self._data @data.setter def data(self, *geoArr_initArgs): if geoArr_initArgs[0] is not None: # TODO this must be able to handle subset inputs in tiled processing - if self._arr and len(geoArr_initArgs[0]) and isinstance(geoArr_initArgs[0], np.ndarray): - self._arr = GeoArray(geoArr_initArgs[0], geotransform=self._arr.gt, projection=self._arr.prj) + if self._data and len(geoArr_initArgs[0]) and isinstance(geoArr_initArgs[0], np.ndarray): + self._data = GeoArray(geoArr_initArgs[0], geotransform=self._data.gt, projection=self._data.prj) else: - self._arr = GeoArray(*geoArr_initArgs) + self._data = GeoArray(*geoArr_initArgs) else: del self.data @data.deleter def data(self): - self._arr = None + self._data = None @property - def mask_clouds(self): + def mask_clouds(self) -> GeoArray: """Return the cloud mask. Bundled with all the corresponding metadata. @@ -155,7 +154,7 @@ class _EnMAP_Image(object): self._mask_clouds = None @property - def mask_clouds_confidence(self): + def mask_clouds_confidence(self) -> GeoArray: """Return pixelwise information on the cloud mask confidence. Bundled with all the corresponding metadata. @@ -190,7 +189,7 @@ class _EnMAP_Image(object): self._mask_clouds_confidence = None @property - def dem(self): + def dem(self) -> GeoArray: """Return an SRTM DEM in the exact dimension an pixel grid of self.arr. :return: geoarray.GeoArray @@ -220,7 +219,7 @@ class _EnMAP_Image(object): self._dem = None @property - def ac_errors(self): + def ac_errors(self) -> GeoArray: """Return error information calculated by the atmospheric correction. :return: geoarray.GeoArray @@ -251,7 +250,7 @@ class _EnMAP_Image(object): self._ac_errors = None @property - def deadpixelmap(self): + def deadpixelmap(self) -> GeoArray: """Return the dead pixel map. Bundled with all the corresponding metadata. Dimensions: (bands x columns). @@ -558,7 +557,7 @@ class EnMAP_Detector_MapGeo(_EnMAP_Image): super(EnMAP_Detector_MapGeo, self).__init__() @property - def mask_nodata(self): + def mask_nodata(self) -> GeoArray: """Return the no data mask. Bundled with all the corresponding metadata. @@ -592,7 +591,7 @@ class EnMAP_Detector_MapGeo(_EnMAP_Image): def mask_nodata(self): self._mask_nodata = None - def calc_mask_nodata(self, fromBand=None, overwrite=False): + def calc_mask_nodata(self, fromBand=None, overwrite=False) -> GeoArray: """Calculate a no data mask with (values: 0=nodata; 1=data). :param fromBand: index of the band to be used (if None, all bands are used) diff --git a/enpt/options/config.py b/enpt/options/config.py index c23ed60..3786edc 100644 --- a/enpt/options/config.py +++ b/enpt/options/config.py @@ -83,6 +83,7 @@ class EnPTConfig(object): # toa_ref self.path_earthSunDist = self.absPath(gp('path_earthSunDist')) self.path_solar_irr = self.absPath(gp('path_solar_irr')) + self.scale_factor_toa_ref = gp('scale_factor_toa_ref'), # geometry self.enable_keystone_correction = gp('enable_keystone_correction') @@ -93,6 +94,7 @@ class EnPTConfig(object): self.sicor_cache_dir = gp('sicor_cache_dir', fallback=sicor.__path__[0]) self.auto_download_ecmwf = gp('auto_download_ecmwf') self.enable_cloud_screening = gp('enable_cloud_screening') + self.scale_factor_boa_ref = gp('scale_factor_boa_ref'), # smile self.run_smile_P = gp('run_smile_P') diff --git a/enpt/options/options_default.json b/enpt/options/options_default.json index 0cac801..47d5819 100644 --- a/enpt/options/options_default.json +++ b/enpt/options/options_default.json @@ -16,7 +16,8 @@ "processors": { "toa_ref": { "path_earthSunDist": "./resources/earth_sun_distance/Earth_Sun_distances_per_day_edited__1980_2030.csv", /*input path of the earth sun distance model*/ - "path_solar_irr": "./resources/solar_irradiance/SUNp1fontenla__350-2500nm_@0.1nm_converted.txt" /*input path of the solar irradiance model*/ + "path_solar_irr": "./resources/solar_irradiance/SUNp1fontenla__350-2500nm_@0.1nm_converted.txt", /*input path of the solar irradiance model*/ + "scale_factor_toa_ref": 10000 /*scale factor to be applied to TOA reflectance result*/ }, "geometry": { @@ -30,7 +31,8 @@ NOTE: SICOR stores intermediate results there that need to computed only once for atmospheric correction of multiple EnMAP images. (default: 'auto'*/ "auto_download_ecmwf": false, - "enable_cloud_screening": false + "enable_cloud_screening": false, + "scale_factor_boa_ref": 10000 /*scale factor to be applied to BOA reflectance result*/ }, "smile": { diff --git a/enpt/options/options_schema.py b/enpt/options/options_schema.py index 3096a83..7816d61 100644 --- a/enpt/options/options_schema.py +++ b/enpt/options/options_schema.py @@ -83,6 +83,7 @@ parameter_mapping = dict( # processors > toa_ref path_earthSunDist=('processors', 'toa_ref', 'path_earthSunDist'), path_solar_irr=('processors', 'toa_ref', 'path_solar_irr'), + scale_factor_toa_ref=('processors', 'toa_ref', 'scale_factor_toa_ref'), # processors > geometry enable_keystone_correction=('processors', 'geometry', 'enable_keystone_correction'), @@ -93,6 +94,7 @@ parameter_mapping = dict( sicor_cache_dir=('processors', 'atmospheric_correction', 'sicor_cache_dir'), auto_download_ecmwf=('processors', 'atmospheric_correction', 'auto_download_ecmwf'), enable_cloud_screening=('processors', 'atmospheric_correction', 'enable_cloud_screening'), + scale_factor_boa_ref=('processors', 'atmospheric_correction', 'scale_factor_boa_ref'), # processors > smile run_smile_P=('processors', 'smile', 'run_processor'), diff --git a/enpt/processors/atmospheric_correction/atmospheric_correction.py b/enpt/processors/atmospheric_correction/atmospheric_correction.py index 0e4634d..85b1a99 100644 --- a/enpt/processors/atmospheric_correction/atmospheric_correction.py +++ b/enpt/processors/atmospheric_correction/atmospheric_correction.py @@ -4,6 +4,7 @@ Performs the atmospheric correction of EnMAP L1B data. """ import pprint +import numpy as np from sicor.sicor_enmap import sicor_ac_enmap from sicor.options import get_options as get_ac_options @@ -43,7 +44,7 @@ class AtmosphericCorrector(object): except FileNotFoundError: raise FileNotFoundError('Could not locate options file for atmospheric correction at %s.' % path_opts) - def run_ac(self, enmap_ImageL1: EnMAPL1Product_SensorGeo): + def run_ac(self, enmap_ImageL1: EnMAPL1Product_SensorGeo) -> EnMAPL1Product_SensorGeo: options = self.get_ac_options(buffer_dir=self.cfg.sicor_cache_dir) enmap_ImageL1.logger.debug('AC options: \n' + pprint.pformat(options)) @@ -53,4 +54,18 @@ class AtmosphericCorrector(object): enmap_l2a_sens_geo, state, fits = sicor_ac_enmap(enmap_l1b=enmap_ImageL1, options=options, logger=enmap_ImageL1.logger, debug=True) - return enmap_l2a_sens_geo + # join results + enmap_ImageL1.logger.info('Joining results of atmospheric correction.') + + for in_detector, out_detector in zip([enmap_ImageL1.vnir, enmap_ImageL1.swir], + [enmap_l2a_sens_geo.vnir, enmap_l2a_sens_geo.swir]): + in_detector.data = (out_detector.data[:] * self.cfg.scale_factor_boa_ref).astype(np.int16) + # NOTE: geotransform and projection are missing due to sensor geometry + + del in_detector.data_l2a # FIXME sicor sets data_l2a to float array -> not needed + del in_detector.unit # FIXME sicor sets unit to '1' -> not needed + + in_detector.detector_meta.unit = '0-%d' % self.cfg.scale_factor_boa_ref + in_detector.detector_meta.unitcode = 'BOARef' + + return enmap_ImageL1 diff --git a/enpt/processors/radiometric_transform/radiometric_transform.py b/enpt/processors/radiometric_transform/radiometric_transform.py index e23bf06..a09ce72 100644 --- a/enpt/processors/radiometric_transform/radiometric_transform.py +++ b/enpt/processors/radiometric_transform/radiometric_transform.py @@ -21,8 +21,7 @@ class Radiometric_Transformer(object): self.solarIrr = config.path_solar_irr # path of model for solar irradiance self.earthSunDist = config.path_earthSunDist # path of model for earth sun distance - @staticmethod - def transform_TOARad2TOARef(enmap_ImageL1: EnMAPL1Product_SensorGeo, scale_factor: int=10000): + def transform_TOARad2TOARef(self, enmap_ImageL1: EnMAPL1Product_SensorGeo): """Transform top-of-atmosphere radiance to top-of-atmosphere reflectance. NOTE: The following formula is used: @@ -30,7 +29,6 @@ class Radiometric_Transformer(object): (solIrr * math.cos(zenithAngleDeg)) :param enmap_ImageL1: instance of the class 'EnMAPL1Product_ImGeo' - :param scale_factor: scale factor to be applied to TOA reflectance result :return: """ for detectorName in enmap_ImageL1.detector_attrNames: @@ -41,7 +39,7 @@ class Radiometric_Transformer(object): # compute TOA reflectance constant = \ - scale_factor * math.pi * enmap_ImageL1.meta.earthSunDist ** 2 / \ + self.cfg.scale_factor_toa_ref * math.pi * enmap_ImageL1.meta.earthSunDist ** 2 / \ (math.cos(math.radians(detector.detector_meta.geom_sun_zenith))) solIrr = np.array([detector.detector_meta.solar_irrad[band] for band in detector.detector_meta.srf.bands])\ .reshape(1, 1, detector.data.bands) @@ -49,7 +47,7 @@ class Radiometric_Transformer(object): # update EnMAP image detector.data = toaRef - detector.detector_meta.unit = '0-%d' % scale_factor + detector.detector_meta.unit = '0-%d' % self.cfg.scale_factor_toa_ref detector.detector_meta.unitcode = 'TOARef' return enmap_ImageL1 -- GitLab From 30ac11f51580872f71ed7dcb5ae94c27fae1a087 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Tue, 17 Apr 2018 16:22:53 +0200 Subject: [PATCH 10/31] Fixed 'directory not empty' exception during test teardown. --- tests/linting/pydocstyle.log | 44 ++++++++++++++++++------------------ tests/test_controller.py | 3 ++- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/tests/linting/pydocstyle.log b/tests/linting/pydocstyle.log index 78dbcd4..94811de 100644 --- a/tests/linting/pydocstyle.log +++ b/tests/linting/pydocstyle.log @@ -1,12 +1,12 @@ enpt/version.py:1 at module level: D100: Missing docstring in public module -enpt/model/images.py:475 in public method `run_AC`: +enpt/model/images.py:474 in public method `run_AC`: D102: Missing docstring in public method -enpt/model/metadata.py:208 in public method `calc_solar_irradiance_CWL_FWHM_per_band`: +enpt/model/metadata.py:210 in public method `calc_solar_irradiance_CWL_FWHM_per_band`: D102: Missing docstring in public method -enpt/model/metadata.py:266 in public method `get_earth_sun_distance`: +enpt/model/metadata.py:268 in public method `get_earth_sun_distance`: D202: No blank lines allowed after function docstring (found 1) -enpt/model/metadata.py:266 in public method `get_earth_sun_distance`: +enpt/model/metadata.py:268 in public method `get_earth_sun_distance`: D400: First line should end with a period (not ')') enpt/model/srf.py:11 in public class `SRF`: D101: Missing docstring in public class @@ -30,9 +30,9 @@ enpt/utils/logging.py:141 in public method `__enter__`: D105: Missing docstring in magic method enpt/utils/logging.py:144 in public method `__exit__`: D105: Missing docstring in magic method -enpt/processors/atmospheric_correction/atmospheric_correction.py:24 in public method `get_ac_options`: +enpt/processors/atmospheric_correction/atmospheric_correction.py:25 in public method `get_ac_options`: D102: Missing docstring in public method -enpt/processors/atmospheric_correction/atmospheric_correction.py:46 in public method `run_ac`: +enpt/processors/atmospheric_correction/atmospheric_correction.py:47 in public method `run_ac`: D102: Missing docstring in public method enpt/execution/controller.py:50 in public method `read_L1B_data`: D205: 1 blank line required between summary line and description (found 0) @@ -46,37 +46,37 @@ enpt/options/config.py:36 in public class `EnPTConfig`: D101: Missing docstring in public class enpt/options/config.py:37 in public method `__init__`: D202: No blank lines allowed after function docstring (found 1) -enpt/options/config.py:113 in public method `absPath`: +enpt/options/config.py:115 in public method `absPath`: D102: Missing docstring in public method -enpt/options/config.py:116 in public method `get_parameter`: +enpt/options/config.py:118 in public method `get_parameter`: D102: Missing docstring in public method -enpt/options/config.py:179 in public method `to_dict`: +enpt/options/config.py:181 in public method `to_dict`: D202: No blank lines allowed after function docstring (found 1) -enpt/options/config.py:193 in public method `to_jsonable_dict`: +enpt/options/config.py:195 in public method `to_jsonable_dict`: D102: Missing docstring in public method -enpt/options/config.py:204 in public method `__repr__`: +enpt/options/config.py:206 in public method `__repr__`: D105: Missing docstring in magic method -enpt/options/config.py:208 in public function `json_to_python`: +enpt/options/config.py:210 in public function `json_to_python`: D103: Missing docstring in public function -enpt/options/config.py:241 in public function `python_to_json`: +enpt/options/config.py:243 in public function `python_to_json`: D103: Missing docstring in public function -enpt/options/config.py:263 in public class `EnPTValidator`: +enpt/options/config.py:265 in public class `EnPTValidator`: D101: Missing docstring in public class -enpt/options/config.py:264 in public method `__init__`: +enpt/options/config.py:266 in public method `__init__`: D205: 1 blank line required between summary line and description (found 0) -enpt/options/config.py:264 in public method `__init__`: +enpt/options/config.py:266 in public method `__init__`: D400: First line should end with a period (not 'r') -enpt/options/config.py:272 in public method `validate`: +enpt/options/config.py:274 in public method `validate`: D102: Missing docstring in public method -enpt/options/config.py:277 in public function `get_options`: +enpt/options/config.py:279 in public function `get_options`: D202: No blank lines allowed after function docstring (found 1) enpt/options/__init__.py:1 at module level: D104: Missing docstring in public package -enpt/options/options_schema.py:108 in public function `get_updated_schema`: +enpt/options/options_schema.py:110 in public function `get_updated_schema`: D103: Missing docstring in public function -enpt/options/options_schema.py:109 in private nested function `deep_update`: +enpt/options/options_schema.py:111 in private nested function `deep_update`: D202: No blank lines allowed after function docstring (found 1) -enpt/options/options_schema.py:109 in private nested function `deep_update`: +enpt/options/options_schema.py:111 in private nested function `deep_update`: D400: First line should end with a period (not 'e') -enpt/options/options_schema.py:128 in public function `get_param_from_json_config`: +enpt/options/options_schema.py:130 in public function `get_param_from_json_config`: D103: Missing docstring in public function diff --git a/tests/test_controller.py b/tests/test_controller.py index 195de5b..16def72 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -20,7 +20,8 @@ class Test_EnPT_Controller(TestCase): self.CTR = EnPT_Controller(**config_for_testing) def tearDown(self): - shutil.rmtree(self.CTR.cfg.output_dir) + # NOTE: ignore_errors deletes the folder, regardless of whether it contains read-only files + shutil.rmtree(self.CTR.cfg.output_dir, ignore_errors=True) def test_run_all_processors(self): self.CTR.run_all_processors() -- GitLab From 8a630d4daa0aa51eb5ceb6f45c912e8f04f223eb Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Wed, 18 Apr 2018 22:12:05 +0200 Subject: [PATCH 11/31] Fixed import issues within tests. Fixed test _radiometric_transform. --- enpt/options/config.py | 8 +++++++- tests/__init__.py | 9 --------- tests/test_cli_parser.py | 4 ++-- tests/test_config.py | 7 ++++++- tests/test_controller.py | 10 +++++++--- tests/test_l1b_reader.py | 3 +-- tests/test_radiometric_transform.py | 16 +++++++++------- 7 files changed, 32 insertions(+), 25 deletions(-) diff --git a/enpt/options/config.py b/enpt/options/config.py index 3786edc..cd37950 100644 --- a/enpt/options/config.py +++ b/enpt/options/config.py @@ -33,6 +33,12 @@ path_enptlib = os.path.dirname(pkgutil.get_loader("enpt").path) path_options_default = os.path.join(path_enptlib, 'options', 'options_default.json') +config_for_testing = dict( + path_l1b_enmap_image=os.path.join(path_enptlib, 'tests', 'data', 'EnMAP_Level_1B', 'AlpineTest1_CWV2_SM0.zip'), + output_dir=os.path.join(path_enptlib, 'tests', 'data', 'test_outputs') +) + + class EnPTConfig(object): def __init__(self, json_config='', **user_opts): """Create a job configuration. @@ -83,7 +89,7 @@ class EnPTConfig(object): # toa_ref self.path_earthSunDist = self.absPath(gp('path_earthSunDist')) self.path_solar_irr = self.absPath(gp('path_solar_irr')) - self.scale_factor_toa_ref = gp('scale_factor_toa_ref'), + self.scale_factor_toa_ref = gp('scale_factor_toa_ref') # geometry self.enable_keystone_correction = gp('enable_keystone_correction') diff --git a/tests/__init__.py b/tests/__init__.py index 6093974..40a96af 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,10 +1 @@ # -*- coding: utf-8 -*- - -import os - -enptRepo_rootpath = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) - -config_for_testing = dict( - path_l1b_enmap_image=os.path.join(enptRepo_rootpath, 'tests', 'data', 'EnMAP_Level_1B', 'AlpineTest1_CWV2_SM0.zip'), - output_dir=os.path.join(enptRepo_rootpath, 'tests', 'data', 'test_outputs') -) diff --git a/tests/test_cli_parser.py b/tests/test_cli_parser.py index b9e43c3..c866e2c 100644 --- a/tests/test_cli_parser.py +++ b/tests/test_cli_parser.py @@ -13,10 +13,10 @@ import os from runpy import run_path from multiprocessing import cpu_count -from enpt import __path__ +import enpt -path_run_enpt = os.path.abspath(os.path.join(__path__[0], '..', 'bin', 'enpt_cli.py')) +path_run_enpt = os.path.abspath(os.path.join(enpt.__path__[0], '..', 'bin', 'enpt_cli.py')) class Test_CLIParser(TestCase): diff --git a/tests/test_config.py b/tests/test_config.py index 66a7820..6f13d53 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python # -*- coding: utf-8 -*- """ @@ -11,7 +12,7 @@ from json import \ dumps, \ JSONDecodeError -from unittest import TestCase +from unittest import TestCase, main from enpt.options.config import \ get_options, \ @@ -87,3 +88,7 @@ class Test_EnPTConfig(TestCase): # check validity EnPTValidator(allow_unknown=True, schema=enpt_schema_config_output).validate(params) + + +if __name__ == '__main__': + main() diff --git a/tests/test_controller.py b/tests/test_controller.py index 16def72..2c53f83 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python # -*- coding: utf-8 -*- """ @@ -7,12 +8,11 @@ test_controller Tests for `execution.controller` module. """ -from unittest import TestCase +from unittest import TestCase, main import shutil from enpt.execution.controller import EnPT_Controller - -from . import config_for_testing +from enpt.options.config import config_for_testing class Test_EnPT_Controller(TestCase): @@ -25,3 +25,7 @@ class Test_EnPT_Controller(TestCase): def test_run_all_processors(self): self.CTR.run_all_processors() + + +if __name__ == '__main__': + main() diff --git a/tests/test_l1b_reader.py b/tests/test_l1b_reader.py index 62a6026..780a77d 100644 --- a/tests/test_l1b_reader.py +++ b/tests/test_l1b_reader.py @@ -17,8 +17,7 @@ import zipfile from datetime import datetime import shutil -from enpt.options.config import EnPTConfig -from . import config_for_testing +from enpt.options.config import EnPTConfig, config_for_testing class Test_L1B_Reader(unittest.TestCase): diff --git a/tests/test_radiometric_transform.py b/tests/test_radiometric_transform.py index ccf9270..2433d77 100644 --- a/tests/test_radiometric_transform.py +++ b/tests/test_radiometric_transform.py @@ -1,23 +1,21 @@ -# -*- coding: utf-8 -*- +#!/usr/bin/env python +# -*- coding: utf-8 -*- import os -import unittest +from unittest import TestCase, main from glob import glob import tempfile import zipfile from datetime import datetime from enpt.processors import Radiometric_Transformer -from enpt.options.config import EnPTConfig +from enpt.options.config import EnPTConfig, config_for_testing -from . import config_for_testing - -class Test_Radiometric_Transformer(unittest.TestCase): +class Test_Radiometric_Transformer(TestCase): def setUp(self): """Set up the needed test data""" - self.cfg = EnPTConfig(**config_for_testing) self.pathList_testimages = glob(os.path.join(os.path.dirname(__file__), "data", "EnMAP_Level_1B", "*.zip")) self.RT = Radiometric_Transformer(config=self.cfg) @@ -48,3 +46,7 @@ class Test_Radiometric_Transformer(unittest.TestCase): self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) self.assertTrue(L1_obj.vnir.detector_meta.unitcode == 'TOARef') self.assertTrue(L1_obj.swir.detector_meta.unitcode == 'TOARef') + + +if __name__ == "__main__": + main() -- GitLab From ed988a59918fa940db363bc87c930fd6e4b036a2 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Wed, 18 Apr 2018 22:23:02 +0200 Subject: [PATCH 12/31] Fixed linting. --- .gitlab-ci.yml | 1 + .../context/environment_enpt.yml | 2 +- tests/linting/pydocstyle.log | 28 +++++++++---------- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3cab1bf..1638e95 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,6 +44,7 @@ test_styles: - source /root/miniconda3/bin/activate enpt - export GDAL_DATA=/root/miniconda3/envs/enpt/share/gdal - export PYTHONPATH=$PYTHONPATH:/root # /root <- directory needed later + - pip install flake8 # fixes version incompatibility between flake8 and pycodestyle - make lint artifacts: paths: diff --git a/tests/gitlab_CI_docker/context/environment_enpt.yml b/tests/gitlab_CI_docker/context/environment_enpt.yml index 7863817..f3b271e 100644 --- a/tests/gitlab_CI_docker/context/environment_enpt.yml +++ b/tests/gitlab_CI_docker/context/environment_enpt.yml @@ -42,8 +42,8 @@ dependencies: - cerberus - jsmin - sphinx-argparse - - flake8 - pycodestyle + - flake8 - pylint - pydocstyle - nose diff --git a/tests/linting/pydocstyle.log b/tests/linting/pydocstyle.log index 94811de..358d86e 100644 --- a/tests/linting/pydocstyle.log +++ b/tests/linting/pydocstyle.log @@ -42,33 +42,33 @@ enpt/execution/controller.py:82 in public method `run_geometry_processor`: D102: Missing docstring in public method enpt/execution/controller.py:89 in public method `write_output`: D102: Missing docstring in public method -enpt/options/config.py:36 in public class `EnPTConfig`: +enpt/options/config.py:42 in public class `EnPTConfig`: D101: Missing docstring in public class -enpt/options/config.py:37 in public method `__init__`: +enpt/options/config.py:43 in public method `__init__`: D202: No blank lines allowed after function docstring (found 1) -enpt/options/config.py:115 in public method `absPath`: +enpt/options/config.py:121 in public method `absPath`: D102: Missing docstring in public method -enpt/options/config.py:118 in public method `get_parameter`: +enpt/options/config.py:124 in public method `get_parameter`: D102: Missing docstring in public method -enpt/options/config.py:181 in public method `to_dict`: +enpt/options/config.py:187 in public method `to_dict`: D202: No blank lines allowed after function docstring (found 1) -enpt/options/config.py:195 in public method `to_jsonable_dict`: +enpt/options/config.py:201 in public method `to_jsonable_dict`: D102: Missing docstring in public method -enpt/options/config.py:206 in public method `__repr__`: +enpt/options/config.py:212 in public method `__repr__`: D105: Missing docstring in magic method -enpt/options/config.py:210 in public function `json_to_python`: +enpt/options/config.py:216 in public function `json_to_python`: D103: Missing docstring in public function -enpt/options/config.py:243 in public function `python_to_json`: +enpt/options/config.py:249 in public function `python_to_json`: D103: Missing docstring in public function -enpt/options/config.py:265 in public class `EnPTValidator`: +enpt/options/config.py:271 in public class `EnPTValidator`: D101: Missing docstring in public class -enpt/options/config.py:266 in public method `__init__`: +enpt/options/config.py:272 in public method `__init__`: D205: 1 blank line required between summary line and description (found 0) -enpt/options/config.py:266 in public method `__init__`: +enpt/options/config.py:272 in public method `__init__`: D400: First line should end with a period (not 'r') -enpt/options/config.py:274 in public method `validate`: +enpt/options/config.py:280 in public method `validate`: D102: Missing docstring in public method -enpt/options/config.py:279 in public function `get_options`: +enpt/options/config.py:285 in public function `get_options`: D202: No blank lines allowed after function docstring (found 1) enpt/options/__init__.py:1 at module level: D104: Missing docstring in public package -- GitLab From 52258a15b5f0650a598e97f35e052e11183228bc Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Wed, 18 Apr 2018 23:01:17 +0200 Subject: [PATCH 13/31] Fixed path within config_for_testing. --- .gitlab-ci.yml | 2 +- enpt/execution/controller.py | 2 +- enpt/options/config.py | 3 ++- requirements_dev.txt | 2 -- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1638e95..9661ed3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,7 +44,7 @@ test_styles: - source /root/miniconda3/bin/activate enpt - export GDAL_DATA=/root/miniconda3/envs/enpt/share/gdal - export PYTHONPATH=$PYTHONPATH:/root # /root <- directory needed later - - pip install flake8 # fixes version incompatibility between flake8 and pycodestyle + - pip install flake8 # FIXME version incompatibility between flake8 and pycodestyle - make lint artifacts: paths: diff --git a/enpt/execution/controller.py b/enpt/execution/controller.py index 64a3659..a6c0488 100644 --- a/enpt/execution/controller.py +++ b/enpt/execution/controller.py @@ -57,7 +57,7 @@ class EnPT_Controller(object): if not os.path.isdir(path_enmap_image) and \ not (os.path.exists(path_enmap_image) and path_enmap_image.endswith('.zip')): raise ValueError("The parameter 'path_enmap_image' must be a directory or the path to an existing zip " - "archive.") + "archive. Received %s." % path_enmap_image) # extract L1B image archive if needed if path_enmap_image.endswith('.zip'): diff --git a/enpt/options/config.py b/enpt/options/config.py index cd37950..e478866 100644 --- a/enpt/options/config.py +++ b/enpt/options/config.py @@ -34,7 +34,8 @@ path_options_default = os.path.join(path_enptlib, 'options', 'options_default.js config_for_testing = dict( - path_l1b_enmap_image=os.path.join(path_enptlib, 'tests', 'data', 'EnMAP_Level_1B', 'AlpineTest1_CWV2_SM0.zip'), + path_l1b_enmap_image=os.path.abspath( + os.path.join(path_enptlib, '..', 'tests', 'data', 'EnMAP_Level_1B', 'AlpineTest1_CWV2_SM0.zip')), output_dir=os.path.join(path_enptlib, 'tests', 'data', 'test_outputs') ) diff --git a/requirements_dev.txt b/requirements_dev.txt index 9147e4e..985c960 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -6,5 +6,3 @@ flake8==2.6.0 tox==2.3.1 coverage==4.1 Sphinx==1.4.8 - - -- GitLab From 58aade42e3222ed28a6a97be01fe1c71a8b48133 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Fri, 11 May 2018 19:59:29 +0200 Subject: [PATCH 14/31] Temporarily disabled Test_EnPT_Controller.test_run_all_processors() because of too slow SICOR implementation. --- .gitlab-ci.yml | 1 + tests/test_controller.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9661ed3..abdc93c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,6 +57,7 @@ test_styles: test_enpt_install: stage: test script: + # TODO: add sicor here - source /root/miniconda3/bin/activate - conda create -y -q --name enpt_test python=3 - source activate enpt_test diff --git a/tests/test_controller.py b/tests/test_controller.py index 2c53f83..0df787e 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -8,7 +8,7 @@ test_controller Tests for `execution.controller` module. """ -from unittest import TestCase, main +from unittest import TestCase, main, SkipTest import shutil from enpt.execution.controller import EnPT_Controller @@ -23,6 +23,7 @@ class Test_EnPT_Controller(TestCase): # NOTE: ignore_errors deletes the folder, regardless of whether it contains read-only files shutil.rmtree(self.CTR.cfg.output_dir, ignore_errors=True) + @SkipTest def test_run_all_processors(self): self.CTR.run_all_processors() -- GitLab From 2fe99dabaafbf23f0ab5b6b31b7cfcc08ed95979 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Mon, 28 May 2018 15:08:03 +0200 Subject: [PATCH 15/31] Re-enabled Test_EnPT_Controller.test_run_all_processors() and modified AtmosphericCorrector.run_ac() to match requirements of current sicor 'feature/improve_enmap' branch. --- .../atmospheric_correction/atmospheric_correction.py | 4 ++-- tests/test_controller.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/enpt/processors/atmospheric_correction/atmospheric_correction.py b/enpt/processors/atmospheric_correction/atmospheric_correction.py index 85b1a99..d263a5c 100644 --- a/enpt/processors/atmospheric_correction/atmospheric_correction.py +++ b/enpt/processors/atmospheric_correction/atmospheric_correction.py @@ -51,8 +51,8 @@ class AtmosphericCorrector(object): # run AC enmap_ImageL1.logger.info("Starting atmospheric correction for VNIR and SWIR detector. " "Source radiometric unit code is '%s'." % enmap_ImageL1.meta.vnir.unitcode) - enmap_l2a_sens_geo, state, fits = sicor_ac_enmap(enmap_l1b=enmap_ImageL1, options=options, - logger=enmap_ImageL1.logger, debug=True) + enmap_l2a_sens_geo, state, cwv_map, ch4_map = sicor_ac_enmap(enmap_l1b=enmap_ImageL1, options=options, + logger=enmap_ImageL1.logger, debug=True) # join results enmap_ImageL1.logger.info('Joining results of atmospheric correction.') diff --git a/tests/test_controller.py b/tests/test_controller.py index 0df787e..2c53f83 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -8,7 +8,7 @@ test_controller Tests for `execution.controller` module. """ -from unittest import TestCase, main, SkipTest +from unittest import TestCase, main import shutil from enpt.execution.controller import EnPT_Controller @@ -23,7 +23,6 @@ class Test_EnPT_Controller(TestCase): # NOTE: ignore_errors deletes the folder, regardless of whether it contains read-only files shutil.rmtree(self.CTR.cfg.output_dir, ignore_errors=True) - @SkipTest def test_run_all_processors(self): self.CTR.run_all_processors() -- GitLab From 5d9bac26aa22de7a33f080e712514e19020e691b Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Mon, 28 May 2018 15:46:20 +0200 Subject: [PATCH 16/31] Added dependencies 'utm', 'lxml' and 'imageio' to environment.yml. --- tests/gitlab_CI_docker/context/environment_enpt.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/gitlab_CI_docker/context/environment_enpt.yml b/tests/gitlab_CI_docker/context/environment_enpt.yml index f3b271e..00d5136 100644 --- a/tests/gitlab_CI_docker/context/environment_enpt.yml +++ b/tests/gitlab_CI_docker/context/environment_enpt.yml @@ -41,6 +41,9 @@ dependencies: - spectral>=0.16 - cerberus - jsmin + - utm + - lxml + - imageio - sphinx-argparse - pycodestyle - flake8 -- GitLab From 3ebf44ba5084a3117c2f65873893d6561dce080a Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Mon, 28 May 2018 15:52:12 +0200 Subject: [PATCH 17/31] Updated docker CI installer to use sicor branch 'feature/improve_enmap'. Updated Miniconda and git lfs versions. --- tests/gitlab_CI_docker/build_enpt_testsuite_image.sh | 3 ++- tests/gitlab_CI_docker/context/enpt_ci.docker | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh b/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh index 39a0381..7569eae 100755 --- a/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh +++ b/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh @@ -7,7 +7,8 @@ gitlab_runner="enpt_gitlab_CI_runner" # get sicor project rm -rf context/sicor -git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git ./context/sicor +# git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git ./context/sicor +git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git --branch feature/improve_enmap --single-branch ./context/sicor echo "#### Build runner docker image" sudo docker rmi ${tag} diff --git a/tests/gitlab_CI_docker/context/enpt_ci.docker b/tests/gitlab_CI_docker/context/enpt_ci.docker index 78c5a74..bccc243 100644 --- a/tests/gitlab_CI_docker/context/enpt_ci.docker +++ b/tests/gitlab_CI_docker/context/enpt_ci.docker @@ -5,11 +5,11 @@ RUN yum update -y && \ yum install -y wget vim bzip2 gcc gcc-c++ make libgl1-mesa-glx mesa-libGL qt5-qtbase-gui git nano tree gdb texlive # change version numbers for future upgrades -# ENV miniconda_dl 'Miniconda3-latest-Linux-x86_64.sh' +ENV miniconda_dl 'Miniconda3-latest-Linux-x86_64.sh' # currently 4.5.1 # use miniconda 4.3.31 as workaround for "IsADirectoryError(21, 'Is a directory')" with version 4.4.10 (currently latest) -ENV miniconda_dl 'Miniconda3-4.3.31-Linux-x86_64.sh' +# ENV miniconda_dl 'Miniconda3-4.3.31-Linux-x86_64.sh' ENV envconfig 'environment_enpt.yml' -ENV git_lfs_v='2.1.1' +ENV git_lfs_v='2.4.1' RUN /bin/bash -i -c "cd /root; wget https://repo.continuum.io/miniconda/$miniconda_dl ; \ bash -i /root/$miniconda_dl -b ; \ -- GitLab From d5f9a8be398f1f9148e79760edccefc76421a178 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Mon, 28 May 2018 16:07:46 +0200 Subject: [PATCH 18/31] Cleaned up. --- .gitlab-ci.yml | 1 - tests/gitlab_CI_docker/context/environment_enpt.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index abdc93c..6fdb134 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,7 +13,6 @@ test_enpt: - source /root/miniconda3/bin/activate enpt - export GDAL_DATA=/root/miniconda3/envs/enpt/share/gdal - export PYTHONPATH=$PYTHONPATH:/root # /root <- here are the sicor tables - - pip install xlrd # FIXME due to https://gitext.gfz-potsdam.de/EnMAP/sicor/issues/23 # update sicor # - conda install -y -q -c conda-forge basemap diff --git a/tests/gitlab_CI_docker/context/environment_enpt.yml b/tests/gitlab_CI_docker/context/environment_enpt.yml index 00d5136..eea54bc 100644 --- a/tests/gitlab_CI_docker/context/environment_enpt.yml +++ b/tests/gitlab_CI_docker/context/environment_enpt.yml @@ -14,7 +14,7 @@ dependencies: - scikit-image - rasterio - pyproj - - lxml + # - lxml - geopandas - ipython - matplotlib -- GitLab From e150e4d2f55b156aaa46d5f771133f977c2aa9ea Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Mon, 28 May 2018 16:09:23 +0200 Subject: [PATCH 19/31] Updated docker image tag. --- tests/gitlab_CI_docker/build_enpt_testsuite_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh b/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh index 7569eae..833a034 100755 --- a/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh +++ b/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh @@ -2,7 +2,7 @@ context_dir="./context" dockerfile="enpt_ci.docker" -tag="enpt_ci:0.3.2" +tag="enpt_ci:0.4.0b2" gitlab_runner="enpt_gitlab_CI_runner" # get sicor project -- GitLab From eb10fa849f413ec3b3df284ce191a48cdc617151 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Mon, 28 May 2018 16:17:15 +0200 Subject: [PATCH 20/31] Moved a comment. --- tests/gitlab_CI_docker/context/enpt_ci.docker | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/gitlab_CI_docker/context/enpt_ci.docker b/tests/gitlab_CI_docker/context/enpt_ci.docker index bccc243..9b13071 100644 --- a/tests/gitlab_CI_docker/context/enpt_ci.docker +++ b/tests/gitlab_CI_docker/context/enpt_ci.docker @@ -4,8 +4,8 @@ FROM centos:7 RUN yum update -y && \ yum install -y wget vim bzip2 gcc gcc-c++ make libgl1-mesa-glx mesa-libGL qt5-qtbase-gui git nano tree gdb texlive -# change version numbers for future upgrades -ENV miniconda_dl 'Miniconda3-latest-Linux-x86_64.sh' # currently 4.5.1 +# change version numbers for future upgrades # currently Anaconda 4.5.1 +ENV miniconda_dl 'Miniconda3-latest-Linux-x86_64.sh' # use miniconda 4.3.31 as workaround for "IsADirectoryError(21, 'Is a directory')" with version 4.4.10 (currently latest) # ENV miniconda_dl 'Miniconda3-4.3.31-Linux-x86_64.sh' ENV envconfig 'environment_enpt.yml' -- GitLab From 2619cd899fd15158c4bc3045174c277439c0a69f Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Mon, 28 May 2018 17:53:49 +0200 Subject: [PATCH 21/31] Added a comment. --- tests/gitlab_CI_docker/context/enpt_ci.docker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/gitlab_CI_docker/context/enpt_ci.docker b/tests/gitlab_CI_docker/context/enpt_ci.docker index 9b13071..193f812 100644 --- a/tests/gitlab_CI_docker/context/enpt_ci.docker +++ b/tests/gitlab_CI_docker/context/enpt_ci.docker @@ -4,7 +4,7 @@ FROM centos:7 RUN yum update -y && \ yum install -y wget vim bzip2 gcc gcc-c++ make libgl1-mesa-glx mesa-libGL qt5-qtbase-gui git nano tree gdb texlive -# change version numbers for future upgrades # currently Anaconda 4.5.1 +# change version numbers for future upgrades # currently Anaconda 4.5.1 (worked fine) ENV miniconda_dl 'Miniconda3-latest-Linux-x86_64.sh' # use miniconda 4.3.31 as workaround for "IsADirectoryError(21, 'Is a directory')" with version 4.4.10 (currently latest) # ENV miniconda_dl 'Miniconda3-4.3.31-Linux-x86_64.sh' -- GitLab From 6a51d67052a173211636387a586ef27620ad55a4 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Wed, 30 May 2018 11:54:18 +0200 Subject: [PATCH 22/31] Added sicor setup to 'test_enpt_install' CI job. --- .gitlab-ci.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6fdb134..c56e266 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -56,20 +56,31 @@ test_styles: test_enpt_install: stage: test script: - # TODO: add sicor here - source /root/miniconda3/bin/activate - conda create -y -q --name enpt_test python=3 - source activate enpt_test + # install some dependencies that cause trouble when installed via pip - conda install -y -c conda-forge scipy # scikit-image, matplotlib + # install not pip-installable deps of geoarray - conda install -y -c conda-forge numpy scikit-image matplotlib pandas gdal rasterio pyproj basemap shapely - conda install -y -c conda-forge 'icu=58.*' lxml # fixes bug for conda-forge gdal build + + # install sicor + - conda install -y -q -c conda-forge pygrib h5py pytables pyfftw numba llvmlite + - rm -rf context/sicor + - git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git ./context/sicor + - cd ./context/sicor + - make install + - cd ../../ + # install enpt - make install - cd .. - pwd - ls + # test importability - python -c "import enpt; print(enpt)" - python -c "from enpt.model.images import EnMAPL1Product_SensorGeo" -- GitLab From 79b1df0aadffbac1389c7de257844427af82ff89 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Thu, 31 May 2018 15:11:30 +0200 Subject: [PATCH 23/31] Updated docker installer to include sicor cache in docker image to speed up CI tests. --- enpt/options/options_default.json | 2 +- tests/gitlab_CI_docker/build_enpt_testsuite_image.sh | 7 ++++++- tests/gitlab_CI_docker/context/enpt_ci.docker | 4 ++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/enpt/options/options_default.json b/enpt/options/options_default.json index 47d5819..598bded 100644 --- a/enpt/options/options_default.json +++ b/enpt/options/options_default.json @@ -29,7 +29,7 @@ "atmospheric_correction": { "sicor_cache_dir": "", /*directory to be used to stored sicor cache files NOTE: SICOR stores intermediate results there that need to computed only once - for atmospheric correction of multiple EnMAP images. (default: 'auto'*/ + for atmospheric correction of multiple EnMAP images. (default: 'auto')*/ "auto_download_ecmwf": false, "enable_cloud_screening": false, "scale_factor_boa_ref": 10000 /*scale factor to be applied to BOA reflectance result*/ diff --git a/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh b/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh index 833a034..ec3dcdd 100755 --- a/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh +++ b/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh @@ -2,7 +2,7 @@ context_dir="./context" dockerfile="enpt_ci.docker" -tag="enpt_ci:0.4.0b2" +tag="enpt_ci:0.4.0b3" gitlab_runner="enpt_gitlab_CI_runner" # get sicor project @@ -10,6 +10,11 @@ rm -rf context/sicor # git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git ./context/sicor git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git --branch feature/improve_enmap --single-branch ./context/sicor +# download sicor cache (fastens SICOR CI tests a lot, but cache needs to be updated manually using a local sicor repo: +# 1. clone a fresh copy of sicor or delete sicor/sicor/aerosol_0_ch4_34d3778719cc87188787de09bb8f870d16050078.pkl.zip +# 2. run a sicor test including sicor_ac or enmap_ac (recreates cache file) -> upload newly created cache file +wget http://ouo.io/uCQxof -P ./context/ + echo "#### Build runner docker image" sudo docker rmi ${tag} sudo docker build -f ${context_dir}/${dockerfile} -m 20G -t ${tag} ${context_dir} diff --git a/tests/gitlab_CI_docker/context/enpt_ci.docker b/tests/gitlab_CI_docker/context/enpt_ci.docker index 193f812..29edd1c 100644 --- a/tests/gitlab_CI_docker/context/enpt_ci.docker +++ b/tests/gitlab_CI_docker/context/enpt_ci.docker @@ -18,6 +18,10 @@ RUN /bin/bash -i -c "cd /root; wget https://repo.continuum.io/miniconda/$minicon # copy some needed stuff to /root COPY *.yml /root/ +# copy sicor cache files to sicor root directory (speeds up SICOR CI tests because table subsets dont have to be created each time) +# -> sicor root directory is the default directory of these cache files if sicor_cache_dir is not set in EnPT options +COPY *.zip /tmp/sicor/sicor + # create an environment with all packages specified in $envconfig (name is 'enpt' -> also specified in $envconfig) RUN /bin/bash -i -c "source /root/miniconda3/bin/activate ; \ conda env update -f /root/$envconfig" -- GitLab From 3d19b90cc12d724f8ad9b8eea3f8b46f61d4d891 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Thu, 31 May 2018 15:38:59 +0200 Subject: [PATCH 24/31] Disabled download. --- tests/gitlab_CI_docker/build_enpt_testsuite_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh b/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh index ec3dcdd..5806c66 100755 --- a/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh +++ b/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh @@ -13,7 +13,7 @@ git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git --branch feature/improve # download sicor cache (fastens SICOR CI tests a lot, but cache needs to be updated manually using a local sicor repo: # 1. clone a fresh copy of sicor or delete sicor/sicor/aerosol_0_ch4_34d3778719cc87188787de09bb8f870d16050078.pkl.zip # 2. run a sicor test including sicor_ac or enmap_ac (recreates cache file) -> upload newly created cache file -wget http://ouo.io/uCQxof -P ./context/ +# wget http://ouo.io/uCQxof -P ./context/ echo "#### Build runner docker image" sudo docker rmi ${tag} -- GitLab From 567bdfacfcec45484be798be8bf1a8eb2cebf157 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Thu, 31 May 2018 16:33:03 +0200 Subject: [PATCH 25/31] Capped version of pycodestyle to <2.4.0 to avoid ImportError. --- tests/gitlab_CI_docker/context/environment_enpt.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/gitlab_CI_docker/context/environment_enpt.yml b/tests/gitlab_CI_docker/context/environment_enpt.yml index eea54bc..4f67e4e 100644 --- a/tests/gitlab_CI_docker/context/environment_enpt.yml +++ b/tests/gitlab_CI_docker/context/environment_enpt.yml @@ -45,7 +45,7 @@ dependencies: - lxml - imageio - sphinx-argparse - - pycodestyle + - pycodestyle<2.4.0 # version 2.4.0 causes ImportError: module 'pycodestyle' has no attribute 'break_around_binary_operator' - flake8 - pylint - pydocstyle -- GitLab From 538344447355748a0a1635d045007d6b50b6aa64 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Fri, 1 Jun 2018 12:04:22 +0200 Subject: [PATCH 26/31] Fixed typo. --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 70f97e1..b5da1c2 100644 --- a/README.rst +++ b/README.rst @@ -6,7 +6,7 @@ EnPT - EnMAP Processing Tools Please check the documentation_ for usage and in depth information. -Licence +License ------- Free software: GNU General Public License v3 @@ -25,7 +25,7 @@ See also the latest coverage_ report and the nosetests_ HTML report. Credits ---------- +------- This software was funded from BMBF under EnMAP ... -- GitLab From 1ec60c7ce2914d0fa55c28b86f359e8793b3d483 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Fri, 1 Jun 2018 12:22:24 +0200 Subject: [PATCH 27/31] Moved copy command for sicor aerosol cache to the end of the docker image installer. --- tests/gitlab_CI_docker/build_enpt_testsuite_image.sh | 2 +- tests/gitlab_CI_docker/context/enpt_ci.docker | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh b/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh index 5806c66..4acaa48 100755 --- a/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh +++ b/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh @@ -2,7 +2,7 @@ context_dir="./context" dockerfile="enpt_ci.docker" -tag="enpt_ci:0.4.0b3" +tag="enpt_ci:0.4.0b4" gitlab_runner="enpt_gitlab_CI_runner" # get sicor project diff --git a/tests/gitlab_CI_docker/context/enpt_ci.docker b/tests/gitlab_CI_docker/context/enpt_ci.docker index 29edd1c..04fb766 100644 --- a/tests/gitlab_CI_docker/context/enpt_ci.docker +++ b/tests/gitlab_CI_docker/context/enpt_ci.docker @@ -18,10 +18,6 @@ RUN /bin/bash -i -c "cd /root; wget https://repo.continuum.io/miniconda/$minicon # copy some needed stuff to /root COPY *.yml /root/ -# copy sicor cache files to sicor root directory (speeds up SICOR CI tests because table subsets dont have to be created each time) -# -> sicor root directory is the default directory of these cache files if sicor_cache_dir is not set in EnPT options -COPY *.zip /tmp/sicor/sicor - # create an environment with all packages specified in $envconfig (name is 'enpt' -> also specified in $envconfig) RUN /bin/bash -i -c "source /root/miniconda3/bin/activate ; \ conda env update -f /root/$envconfig" @@ -45,3 +41,7 @@ RUN bash -i -c "source /root/miniconda3/bin/activate enpt; \ make requirements ; \ make download-tables ; \ pip install . --no-cache-dir" + +# copy sicor cache files to sicor root directory (speeds up SICOR CI tests because table subsets dont have to be created each time) +# -> sicor root directory is the default directory of these cache files if sicor_cache_dir is not set in EnPT options +COPY *.zip /tmp/sicor/sicor -- GitLab From f050dad4b05a2a12a3ded164ed1eca0ab0132a7b Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Fri, 1 Jun 2018 13:45:30 +0200 Subject: [PATCH 28/31] SICOR is now installed in pip development mode so that its root directory matched the subsequent COPY command. --- tests/gitlab_CI_docker/context/enpt_ci.docker | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/gitlab_CI_docker/context/enpt_ci.docker b/tests/gitlab_CI_docker/context/enpt_ci.docker index 04fb766..ac07286 100644 --- a/tests/gitlab_CI_docker/context/enpt_ci.docker +++ b/tests/gitlab_CI_docker/context/enpt_ci.docker @@ -40,8 +40,9 @@ RUN bash -i -c "source /root/miniconda3/bin/activate enpt; \ make clean ; \ make requirements ; \ make download-tables ; \ - pip install . --no-cache-dir" + pip install -e . --no-cache-dir" # copy sicor cache files to sicor root directory (speeds up SICOR CI tests because table subsets dont have to be created each time) # -> sicor root directory is the default directory of these cache files if sicor_cache_dir is not set in EnPT options COPY *.zip /tmp/sicor/sicor + -- GitLab From a8a6896ab82c409349357f07a103e7627b5b6e4f Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Fri, 1 Jun 2018 14:11:10 +0200 Subject: [PATCH 29/31] Added a comment. --- tests/gitlab_CI_docker/context/enpt_ci.docker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/gitlab_CI_docker/context/enpt_ci.docker b/tests/gitlab_CI_docker/context/enpt_ci.docker index ac07286..a64bded 100644 --- a/tests/gitlab_CI_docker/context/enpt_ci.docker +++ b/tests/gitlab_CI_docker/context/enpt_ci.docker @@ -34,7 +34,7 @@ RUN /bin/bash -i -c "wget https://github.com/git-lfs/git-lfs/releases/download/v # copy sicor code to /tmp COPY sicor /tmp/sicor -# install sicor +# install sicor (in pip development mode so that its root directory in /tmp/sicor matching with the subsequent COPY command) RUN bash -i -c "source /root/miniconda3/bin/activate enpt; \ cd /tmp/sicor/ ; \ make clean ; \ -- GitLab From c04a78cc6bc52666ffd48fb64966e4bd5e447db3 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Fri, 1 Jun 2018 14:24:39 +0200 Subject: [PATCH 30/31] Fix for wrong output directory of test logfile. --- enpt/options/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/enpt/options/config.py b/enpt/options/config.py index e478866..fd552f2 100644 --- a/enpt/options/config.py +++ b/enpt/options/config.py @@ -36,7 +36,7 @@ path_options_default = os.path.join(path_enptlib, 'options', 'options_default.js config_for_testing = dict( path_l1b_enmap_image=os.path.abspath( os.path.join(path_enptlib, '..', 'tests', 'data', 'EnMAP_Level_1B', 'AlpineTest1_CWV2_SM0.zip')), - output_dir=os.path.join(path_enptlib, 'tests', 'data', 'test_outputs') + output_dir=os.path.join(path_enptlib, '..', 'tests', 'data', 'test_outputs') ) @@ -81,7 +81,7 @@ class EnPTConfig(object): # output options # ################## - self.output_dir = self.absPath(gp('output_dir')) + self.output_dir = self.absPath(gp('output_dir', fallback=os.path.abspath(os.path.curdir))) ########################### # processor configuration # -- GitLab From 6423b746c925adb196c9b728e2a761f33f4000e7 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Fri, 1 Jun 2018 14:33:51 +0200 Subject: [PATCH 31/31] Set log level in test mode to 'DEBUG'. Logs within enpt/tests/data/test_outputs/ are now left over as artifacts during 'test_enpt' CI job. --- .gitlab-ci.yml | 2 +- enpt/options/config.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c56e266..b290d9d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -43,10 +43,10 @@ test_styles: - source /root/miniconda3/bin/activate enpt - export GDAL_DATA=/root/miniconda3/envs/enpt/share/gdal - export PYTHONPATH=$PYTHONPATH:/root # /root <- directory needed later - - pip install flake8 # FIXME version incompatibility between flake8 and pycodestyle - make lint artifacts: paths: + - tests/data/test_outputs/*.log - tests/linting/flake8.log - tests/linting/pycodestyle.log - tests/linting/pydocstyle.log diff --git a/enpt/options/config.py b/enpt/options/config.py index fd552f2..9d016ab 100644 --- a/enpt/options/config.py +++ b/enpt/options/config.py @@ -36,6 +36,7 @@ path_options_default = os.path.join(path_enptlib, 'options', 'options_default.js config_for_testing = dict( path_l1b_enmap_image=os.path.abspath( os.path.join(path_enptlib, '..', 'tests', 'data', 'EnMAP_Level_1B', 'AlpineTest1_CWV2_SM0.zip')), + log_level='DEBUG', output_dir=os.path.join(path_enptlib, '..', 'tests', 'data', 'test_outputs') ) -- GitLab