From 0bbc4b62fd477c0a11d09ae981fd02b98f0cb07e Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Mon, 18 Feb 2019 13:26:19 +0100 Subject: [PATCH 01/24] Updated SICOR implementation. --- bin/enpt_cli.py | 4 +- enpt/model/metadata.py | 2 +- enpt/options/config.py | 4 +- enpt/options/options_default.json | 8 ++-- enpt/options/options_schema.py | 4 +- .../atmospheric_correction.py | 44 ++++++++++--------- .../dead_pixel_correction.py | 2 +- .../spatial_transform/spatial_transform.py | 2 + enpt/utils/path_generator.py | 4 +- 9 files changed, 40 insertions(+), 34 deletions(-) diff --git a/bin/enpt_cli.py b/bin/enpt_cli.py index eb890e5..e7f8496 100644 --- a/bin/enpt_cli.py +++ b/bin/enpt_cli.py @@ -68,10 +68,10 @@ def get_enpt_argparser(): add('--enable_ac', type=bool, default=True, help="Enable atmospheric correction using SICOR algorithm (default: True). If False, the L2A output contains " "top-of-atmosphere reflectance") - add('--sicor_cache_dir', type=str, default=None, - help='SICOR cache directory') add('--auto_download_ecmwf', type=bool, default=False, help='Automatically download ECMWF data for atmospheric correction') + add('--enable_ice_retrieval', type=bool, default=True, + help='Enable ice retrieval (default); increases accuracy of water vapour retrieval') add('--enable_cloud_screening', type=bool, default=False, help='Enable cloud screening during atmospheric correction') add('--scale_factor_boa_ref', type=int, default=10000, diff --git a/enpt/model/metadata.py b/enpt/model/metadata.py index 299c403..4afb359 100644 --- a/enpt/model/metadata.py +++ b/enpt/model/metadata.py @@ -801,7 +801,7 @@ class EnMAP_Metadata_L2A_MapGeo(object): # FIXME this is the size of the VNIR/SWIR stack size = [F['size'] for F in self.fileinfos if os.path.splitext(F['name'])[0].endswith('-SPECTRAL_IMAGE')][0] xml.find("product/image/%s/size" % lbl).text = str(size) - # FIXME DLR data dimensions equal either L2A data nor L1B data + # FIXME DLR data dimensions equal neither L2A data nor L1B data xml.find("product/image/%s/channels" % lbl).text = str(detMetaL1B.nwvl) xml.find("product/image/%s/dimension/rows" % lbl).text = str(self.nrows) xml.find("product/image/%s/dimension/columns" % lbl).text = str(self.ncols) diff --git a/enpt/options/config.py b/enpt/options/config.py index 14f0f8f..4b29bf1 100644 --- a/enpt/options/config.py +++ b/enpt/options/config.py @@ -45,7 +45,7 @@ config_for_testing = dict( n_lines_to_append=50, disable_progress_bars=True, is_dlr_dataformat=False, - enable_ac=False + enable_ac=True ) @@ -143,8 +143,8 @@ class EnPTConfig(object): # atmospheric_correction self.enable_ac = gp('enable_ac') - self.sicor_cache_dir = gp('sicor_cache_dir', fallback=sicor.__path__[0]) self.auto_download_ecmwf = gp('auto_download_ecmwf') + self.enable_ice_retrieval = gp('enable_ice_retrieval') self.enable_cloud_screening = gp('enable_cloud_screening') self.scale_factor_boa_ref = gp('scale_factor_boa_ref'), diff --git a/enpt/options/options_default.json b/enpt/options/options_default.json index 474a279..a132771 100644 --- a/enpt/options/options_default.json +++ b/enpt/options/options_default.json @@ -38,11 +38,9 @@ "atmospheric_correction": { "enable_ac": true, /*enable atmospheric correction using SICOR algorithm (default: True). If False, the L2A output contains top-of-atmosphere reflectance.*/ - "sicor_cache_dir": "", /*directory to be used to stored sicor cache files - NOTE: SICOR stores intermediate results there that need to computed only once - for atmospheric correction of multiple EnMAP images. (default: 'auto')*/ - "auto_download_ecmwf": false, - "enable_cloud_screening": false, + "auto_download_ecmwf": false, /*FIXME this might be not needed anymore in future*/ + "enable_ice_retrieval": true, /*enable ice retrieval (default); increases accuracy of water vapour retrieval*/ + "enable_cloud_screening": false, /*FIXME this is currently not implemented*/ "scale_factor_boa_ref": 10000 /*scale factor to be applied to BOA reflectance result*/ }, diff --git a/enpt/options/options_schema.py b/enpt/options/options_schema.py index f10af6d..1da9b4b 100644 --- a/enpt/options/options_schema.py +++ b/enpt/options/options_schema.py @@ -46,8 +46,8 @@ enpt_schema_input = dict( type='dict', required=False, schema=dict( enable_ac=dict(type='boolean', required=False), - sicor_cache_dir=dict(type='string', required=False), auto_download_ecmwf=dict(type='boolean', required=False), + enable_ice_retrieval=dict(type='boolean', required=False), enable_cloud_screening=dict(type='boolean', required=False), )), @@ -106,8 +106,8 @@ parameter_mapping = dict( # processors > atmospheric_correction enable_ac=('processors', 'atmospheric_correction', 'enable_ac'), - sicor_cache_dir=('processors', 'atmospheric_correction', 'sicor_cache_dir'), auto_download_ecmwf=('processors', 'atmospheric_correction', 'auto_download_ecmwf'), + enable_ice_retrieval=('processors', 'atmospheric_correction', 'enable_ice_retrieval'), enable_cloud_screening=('processors', 'atmospheric_correction', 'enable_cloud_screening'), scale_factor_boa_ref=('processors', 'atmospheric_correction', 'scale_factor_boa_ref'), diff --git a/enpt/processors/atmospheric_correction/atmospheric_correction.py b/enpt/processors/atmospheric_correction/atmospheric_correction.py index 541ef6e..aea973e 100644 --- a/enpt/processors/atmospheric_correction/atmospheric_correction.py +++ b/enpt/processors/atmospheric_correction/atmospheric_correction.py @@ -21,23 +21,19 @@ class AtmosphericCorrector(object): """Create an instance of AtmosphericCorrector.""" self.cfg = config - def get_ac_options(self, buffer_dir): + def get_ac_options(self): path_opts = get_path_ac_options() try: - options = get_ac_options(path_opts) + options = get_ac_options(path_opts, validation=False) # FIXME validation is currently not implemented # adjust options - options['EnMAP']['buffer_dir'] = buffer_dir - for vv in options["RTFO"].values(): - vv["hash_formats"] = dict(spr='%.0f', - coz='%.0f,', - cwv='%.0f,', - tmp='%0f,', - tau_a='%.2f,', - vza='%.0f,') - vv["disable_progress_bars"] = self.cfg.disable_progress_bars - options["ECMWF"]["path_db"] = "./ecmwf" + options["EnMAP"]["Retrieval"]["fn_LUT"] = \ + "/home/gfz-fe/scheffler/python_deployed/sicor_aux_data/EnMAP_LUT_MOD5_formatted_1nm" # FIXME hardcoded + # options["ECMWF"]["path_db"] = "./ecmwf" # disbled as it is not needed at the moment + # TODO disable_progress_bars? + + options["EnMAP"]["Retrieval"]["ice"] = self.cfg.enable_ice_retrieval return options @@ -45,29 +41,37 @@ class AtmosphericCorrector(object): raise FileNotFoundError('Could not locate options file for atmospheric correction at %s.' % path_opts) def run_ac(self, enmap_ImageL1: EnMAPL1Product_SensorGeo) -> EnMAPL1Product_SensorGeo: - options = self.get_ac_options(buffer_dir=self.cfg.sicor_cache_dir) + options = self.get_ac_options() enmap_ImageL1.logger.debug('AC options: \n' + pprint.pformat(options)) # run AC enmap_ImageL1.logger.info("Starting atmospheric correction for VNIR and SWIR detector. " "Source radiometric unit code is '%s'." % enmap_ImageL1.meta.vnir.unitcode) - enmap_l2a_sens_geo, state = sicor_ac_enmap(enmap_l1b=enmap_ImageL1, options=options, - logger=enmap_ImageL1.logger) + + # run SICOR + # NOTE: - enmap_l2a_vnir, enmap_l2a_swir: reflectance between 0 and 1 + # - cwv_model, cwc_model, toa_model have the SWIR geometry + # - currently, the slower method is implemented, + # -> otherwise options["EnMAP"]["Retrieval"]["fast"] must be true # TODO + if self.cfg.enable_ice_retrieval: + enmap_l2a_vnir, enmap_l2a_swir, cwv_model, cwc_model, ice_model, toa_model = \ + sicor_ac_enmap(enmap_l1b=enmap_ImageL1, options=options, logger=enmap_ImageL1.logger) + else: + enmap_l2a_vnir, enmap_l2a_swir, cwv_model, cwc_model, toa_model = \ + sicor_ac_enmap(enmap_l1b=enmap_ImageL1, options=options, logger=enmap_ImageL1.logger) # join results enmap_ImageL1.logger.info('Joining results of atmospheric correction.') for in_detector, out_detector in zip([enmap_ImageL1.vnir, enmap_ImageL1.swir], - [enmap_l2a_sens_geo.vnir, enmap_l2a_sens_geo.swir]): - in_detector.data = (out_detector.data[:] * self.cfg.scale_factor_boa_ref).astype(np.int16) + [enmap_l2a_vnir, enmap_l2a_swir]): + in_detector.data = (out_detector * self.cfg.scale_factor_boa_ref).astype(np.int16) # NOTE: geotransform and projection are missing due to sensor geometry - del in_detector.data_l2a # FIXME sicor sets data_l2a to float array -> not needed - del in_detector.unit # FIXME sicor sets unit to '1' -> not needed - in_detector.detector_meta.unit = '0-%d' % self.cfg.scale_factor_boa_ref in_detector.detector_meta.unitcode = 'BOARef' # FIXME what about mask_clouds, mask_clouds_confidence, ac_errors? + # FIXME use cwv_model, cwc_model, toa_model also for EnPT? return enmap_ImageL1 diff --git a/enpt/processors/dead_pixel_correction/dead_pixel_correction.py b/enpt/processors/dead_pixel_correction/dead_pixel_correction.py index 3dae816..624bd40 100644 --- a/enpt/processors/dead_pixel_correction/dead_pixel_correction.py +++ b/enpt/processors/dead_pixel_correction/dead_pixel_correction.py @@ -309,7 +309,7 @@ def interp_nodata_spatially_3d(data_3d: np.ndarray, axis: int = 0, nodata: Union assert axis < 2 badmask_full = _get_baddata_mask(data_3d, nodata) - + from multiprocessing.dummy import Pool if CPUs > 1: with Pool(CPUs or cpu_count()) as pool: args = [[data_3d[:, :, band], axis, badmask_full[:, :, band], method, fill_value, implementation] diff --git a/enpt/processors/spatial_transform/spatial_transform.py b/enpt/processors/spatial_transform/spatial_transform.py index d2f5193..4172646 100644 --- a/enpt/processors/spatial_transform/spatial_transform.py +++ b/enpt/processors/spatial_transform/spatial_transform.py @@ -455,6 +455,8 @@ class RPC_3D_Geolayer_Generator(object): args = [(coeffs, self.enmapIm_cornerCoords, self.enmapIm_dims_sensorgeo, idx) for coeffs, idx in zip(rpc_coeffs_list, band_inds)] + from multiprocessing.dummy import Pool + with Pool(self.CPUs, initializer=mp_initializer_for_RPC_3D_Geolayer_Generator, initargs=(self.dem,)) as pool: diff --git a/enpt/utils/path_generator.py b/enpt/utils/path_generator.py index d85c8b4..1fdd75e 100644 --- a/enpt/utils/path_generator.py +++ b/enpt/utils/path_generator.py @@ -58,6 +58,8 @@ class PathGenL1BProduct(object): def get_path_ac_options() -> str: """Returns the path of the options json file needed for atmospheric correction.""" from sicor import options - path_ac = os.path.join(os.path.dirname(options.__file__), 'sicor_enmap_user_options.json') + path_ac = os.path.join(os.path.dirname(options.__file__), 'enmap_options.json') + # FIXME temporarily disabled because not implemented at the moment: + # path_ac = os.path.join(os.path.dirname(options.__file__), 'sicor_enmap_user_options.json') return path_ac -- GitLab From 81bd2cb59b10a977b3bc36a20b566a8b71708bdb Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Tue, 19 Feb 2019 22:29:48 +0100 Subject: [PATCH 02/24] Updated SICOR implementation. --- enpt/execution/controller.py | 4 +-- enpt/options/config.py | 12 ++++---- .../atmospheric_correction.py | 17 ++++++----- .../dead_pixel_correction.py | 2 +- .../spatial_transform/spatial_transform.py | 2 -- tests/linting/pydocstyle.log | 30 +++++++++---------- tests/test_l1b_reader.py | 2 +- 7 files changed, 35 insertions(+), 34 deletions(-) diff --git a/enpt/execution/controller.py b/enpt/execution/controller.py index a1001f2..f2fd38a 100644 --- a/enpt/execution/controller.py +++ b/enpt/execution/controller.py @@ -121,10 +121,10 @@ class EnPT_Controller(object): # self.run_toaRad2toaRef() # this is only needed for geometry processor but AC expects radiance self.run_dem_processor() if self.cfg.enable_ac: - self.L1_obj.logger.info('Skipping atmospheric correction as configured and ' - 'computing top-of-atmosphere reflectance instead.') self.run_atmospheric_correction() else: + self.L1_obj.logger.info('Skipping atmospheric correction as configured and ' + 'computing top-of-atmosphere reflectance instead.') self.run_toaRad2toaRef() self.run_geometry_processor() self.run_orthorectification() diff --git a/enpt/options/config.py b/enpt/options/config.py index 4b29bf1..491d6e3 100644 --- a/enpt/options/config.py +++ b/enpt/options/config.py @@ -17,8 +17,6 @@ from collections import OrderedDict, Mapping import numpy as np from multiprocessing import cpu_count -import sicor - from .options_schema import \ enpt_schema_input, \ enpt_schema_config_output, \ @@ -45,7 +43,9 @@ config_for_testing = dict( n_lines_to_append=50, disable_progress_bars=True, is_dlr_dataformat=False, - enable_ac=True + enable_ac=True, + enable_ice_retrieval=False, + CPUs=16 ) @@ -65,8 +65,10 @@ config_for_testing_dlr = dict( n_lines_to_append=50, disable_progress_bars=True, is_dlr_dataformat=True, - enable_ac=False, - ortho_resampAlg='gauss' + enable_ac=True, + enable_ice_retrieval=False, + CPUs=32, + ortho_resampAlg='gauss', ) diff --git a/enpt/processors/atmospheric_correction/atmospheric_correction.py b/enpt/processors/atmospheric_correction/atmospheric_correction.py index aea973e..104c7d2 100644 --- a/enpt/processors/atmospheric_correction/atmospheric_correction.py +++ b/enpt/processors/atmospheric_correction/atmospheric_correction.py @@ -5,6 +5,7 @@ Performs the atmospheric correction of EnMAP L1B data. """ import pprint import numpy as np +from multiprocessing import cpu_count from sicor.sicor_enmap import sicor_ac_enmap from sicor.options import get_options as get_ac_options @@ -33,7 +34,10 @@ class AtmosphericCorrector(object): # options["ECMWF"]["path_db"] = "./ecmwf" # disbled as it is not needed at the moment # TODO disable_progress_bars? + # always use the fast implementation (the slow implementation was only a temporary solution) + options["EnMAP"]["Retrieval"]["fast"] = True options["EnMAP"]["Retrieval"]["ice"] = self.cfg.enable_ice_retrieval + options["EnMAP"]["Retrieval"]["cpu"] = self.cfg.CPUs or cpu_count() return options @@ -51,14 +55,11 @@ class AtmosphericCorrector(object): # run SICOR # NOTE: - enmap_l2a_vnir, enmap_l2a_swir: reflectance between 0 and 1 # - cwv_model, cwc_model, toa_model have the SWIR geometry - # - currently, the slower method is implemented, - # -> otherwise options["EnMAP"]["Retrieval"]["fast"] must be true # TODO - if self.cfg.enable_ice_retrieval: - enmap_l2a_vnir, enmap_l2a_swir, cwv_model, cwc_model, ice_model, toa_model = \ - sicor_ac_enmap(enmap_l1b=enmap_ImageL1, options=options, logger=enmap_ImageL1.logger) - else: - enmap_l2a_vnir, enmap_l2a_swir, cwv_model, cwc_model, toa_model = \ - sicor_ac_enmap(enmap_l1b=enmap_ImageL1, options=options, logger=enmap_ImageL1.logger) + # - currently, the fast method is implemented, + # -> otherwise options["EnMAP"]["Retrieval"]["fast"] must be false + # - ice_model is None if self.cfg.enable_ice_retrieval is False + enmap_l2a_vnir, enmap_l2a_swir, cwv_model, cwc_model, ice_model, toa_model, se, scem, srem = \ + sicor_ac_enmap(enmap_l1b=enmap_ImageL1, options=options, logger=enmap_ImageL1.logger) # join results enmap_ImageL1.logger.info('Joining results of atmospheric correction.') diff --git a/enpt/processors/dead_pixel_correction/dead_pixel_correction.py b/enpt/processors/dead_pixel_correction/dead_pixel_correction.py index 624bd40..3dae816 100644 --- a/enpt/processors/dead_pixel_correction/dead_pixel_correction.py +++ b/enpt/processors/dead_pixel_correction/dead_pixel_correction.py @@ -309,7 +309,7 @@ def interp_nodata_spatially_3d(data_3d: np.ndarray, axis: int = 0, nodata: Union assert axis < 2 badmask_full = _get_baddata_mask(data_3d, nodata) - from multiprocessing.dummy import Pool + if CPUs > 1: with Pool(CPUs or cpu_count()) as pool: args = [[data_3d[:, :, band], axis, badmask_full[:, :, band], method, fill_value, implementation] diff --git a/enpt/processors/spatial_transform/spatial_transform.py b/enpt/processors/spatial_transform/spatial_transform.py index 4172646..d2f5193 100644 --- a/enpt/processors/spatial_transform/spatial_transform.py +++ b/enpt/processors/spatial_transform/spatial_transform.py @@ -455,8 +455,6 @@ class RPC_3D_Geolayer_Generator(object): args = [(coeffs, self.enmapIm_cornerCoords, self.enmapIm_dims_sensorgeo, idx) for coeffs, idx in zip(rpc_coeffs_list, band_inds)] - from multiprocessing.dummy import Pool - with Pool(self.CPUs, initializer=mp_initializer_for_RPC_3D_Geolayer_Generator, initargs=(self.dem,)) as pool: diff --git a/tests/linting/pydocstyle.log b/tests/linting/pydocstyle.log index 6c091c5..402b172 100644 --- a/tests/linting/pydocstyle.log +++ b/tests/linting/pydocstyle.log @@ -148,7 +148,7 @@ enpt/processors/dem_preprocessing/dem_preprocessing.py:65 in public method `to_s D102: Missing docstring in public method enpt/processors/dead_pixel_correction/dead_pixel_correction.py:18 in public class `Dead_Pixel_Corrector`: D101: Missing docstring in public class -enpt/processors/atmospheric_correction/atmospheric_correction.py:24 in public method `get_ac_options`: +enpt/processors/atmospheric_correction/atmospheric_correction.py:25 in public method `get_ac_options`: D102: Missing docstring in public method enpt/processors/atmospheric_correction/atmospheric_correction.py:47 in public method `run_ac`: D102: Missing docstring in public method @@ -172,33 +172,33 @@ enpt/execution/controller.py:94 in public method `run_geometry_processor`: D102: Missing docstring in public method enpt/execution/controller.py:106 in public method `write_output`: D102: Missing docstring in public method -enpt/options/config.py:79 in public class `EnPTConfig`: +enpt/options/config.py:81 in public class `EnPTConfig`: D101: Missing docstring in public class -enpt/options/config.py:80 in public method `__init__`: +enpt/options/config.py:82 in public method `__init__`: D202: No blank lines allowed after function docstring (found 1) -enpt/options/config.py:170 in public method `absPath`: +enpt/options/config.py:172 in public method `absPath`: D102: Missing docstring in public method -enpt/options/config.py:173 in public method `get_parameter`: +enpt/options/config.py:175 in public method `get_parameter`: D102: Missing docstring in public method -enpt/options/config.py:236 in public method `to_dict`: +enpt/options/config.py:238 in public method `to_dict`: D202: No blank lines allowed after function docstring (found 1) -enpt/options/config.py:250 in public method `to_jsonable_dict`: +enpt/options/config.py:252 in public method `to_jsonable_dict`: D102: Missing docstring in public method -enpt/options/config.py:261 in public method `__repr__`: +enpt/options/config.py:263 in public method `__repr__`: D105: Missing docstring in magic method -enpt/options/config.py:265 in public function `json_to_python`: +enpt/options/config.py:267 in public function `json_to_python`: D103: Missing docstring in public function -enpt/options/config.py:298 in public function `python_to_json`: +enpt/options/config.py:300 in public function `python_to_json`: D103: Missing docstring in public function -enpt/options/config.py:320 in public class `EnPTValidator`: +enpt/options/config.py:322 in public class `EnPTValidator`: D101: Missing docstring in public class -enpt/options/config.py:321 in public method `__init__`: +enpt/options/config.py:323 in public method `__init__`: D205: 1 blank line required between summary line and description (found 0) -enpt/options/config.py:321 in public method `__init__`: +enpt/options/config.py:323 in public method `__init__`: D400: First line should end with a period (not 'r') -enpt/options/config.py:329 in public method `validate`: +enpt/options/config.py:331 in public method `validate`: D102: Missing docstring in public method -enpt/options/config.py:334 in public function `get_options`: +enpt/options/config.py:336 in public function `get_options`: D202: No blank lines allowed after function docstring (found 1) enpt/options/__init__.py:1 at module level: D104: Missing docstring in public package diff --git a/tests/test_l1b_reader.py b/tests/test_l1b_reader.py index e3603ab..328e9e0 100644 --- a/tests/test_l1b_reader.py +++ b/tests/test_l1b_reader.py @@ -122,7 +122,7 @@ class Test_L1B_Reader(unittest.TestCase): n_lines)) print("-----------------------------------------------------------------------------------------------") - if n_lines is "all": + if n_lines == "all": n_lines = None print("") -- GitLab From 09bdaf91895780e9ead11110efdac3c4c12b7ed3 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Tue, 19 Feb 2019 22:32:21 +0100 Subject: [PATCH 03/24] Updated minimal version of py_tools_ds (fixes issue #29 (Exception: dlopen: cannot load any more object with static TLS)). --- .gitlab-ci.yml | 4 ++-- requirements.txt | 2 +- setup.py | 2 +- tests/gitlab_CI_docker/context/environment_enpt.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 92164cb..36bfb37 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,7 +15,7 @@ test_enpt: - export PYTHONPATH=$PYTHONPATH:/root # /root <- here are the sicor tables # update py_tools_ds - - pip install 'py_tools_ds>=0.14.8' + - pip install 'py_tools_ds>=0.14.11' # update sicor # - conda install -y -q -c conda-forge basemap @@ -71,7 +71,7 @@ test_enpt_install: - conda install -y -c conda-forge 'icu=58.*' # fixes bug for conda-forge gdal build # install py_tools_ds - - pip install 'py_tools_ds>=0.14.8' + - pip install 'py_tools_ds>=0.14.11' # install sicor - conda install -y -q -c conda-forge pygrib h5py pytables pyfftw numba llvmlite scikit-learn diff --git a/requirements.txt b/requirements.txt index 2626286..7f96643 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy pandas scipy geoarray>=0.8.9 -py_tools_ds>=0.14.8 +py_tools_ds>=0.14.11 cerberus jsmin matplotlib diff --git a/setup.py b/setup.py index b5aa6c8..cbac0b0 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ with open("enpt/version.py") as version_file: exec(version_file.read(), version) requirements = [ # put package requirements here - 'numpy', 'pandas', 'scipy', 'geoarray>=0.8.9', 'py_tools_ds>=0.14.8', 'cerberus', 'jsmin', 'matplotlib', 'tqdm', + 'numpy', 'pandas', 'scipy', 'geoarray>=0.8.9', 'py_tools_ds>=0.14.11', 'cerberus', 'jsmin', 'matplotlib', 'tqdm', 'utm', 'lxml', 'numpy-indexed' # 'sicor', # pip install git+https://gitext.gfz-potsdam.de/EnMAP/sicor.git ] diff --git a/tests/gitlab_CI_docker/context/environment_enpt.yml b/tests/gitlab_CI_docker/context/environment_enpt.yml index 00aa1be..663248b 100644 --- a/tests/gitlab_CI_docker/context/environment_enpt.yml +++ b/tests/gitlab_CI_docker/context/environment_enpt.yml @@ -44,7 +44,7 @@ dependencies: - pip: - scipy - geoarray>=0.8.9 - - py_tools_ds>=0.14.8 + - py_tools_ds>=0.14.11 - cerberus - jsmin - tqdm -- GitLab From 83a0c1776c0312cd428ec418374a4336f60f7f31 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Wed, 20 Feb 2019 12:30:12 +0100 Subject: [PATCH 04/24] Updated LUT path. --- .../atmospheric_correction/atmospheric_correction.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/enpt/processors/atmospheric_correction/atmospheric_correction.py b/enpt/processors/atmospheric_correction/atmospheric_correction.py index 104c7d2..b9a5534 100644 --- a/enpt/processors/atmospheric_correction/atmospheric_correction.py +++ b/enpt/processors/atmospheric_correction/atmospheric_correction.py @@ -6,7 +6,9 @@ Performs the atmospheric correction of EnMAP L1B data. import pprint import numpy as np from multiprocessing import cpu_count +from os import path +import sicor from sicor.sicor_enmap import sicor_ac_enmap from sicor.options import get_options as get_ac_options @@ -29,8 +31,9 @@ class AtmosphericCorrector(object): options = get_ac_options(path_opts, validation=False) # FIXME validation is currently not implemented # adjust options + # FIXME this path should be already known to sicor options["EnMAP"]["Retrieval"]["fn_LUT"] = \ - "/home/gfz-fe/scheffler/python_deployed/sicor_aux_data/EnMAP_LUT_MOD5_formatted_1nm" # FIXME hardcoded + path.join(path.abspath(sicor.__path__[0]), 'tables', 'EnMAP_LUT_MOD5_formatted_1nm') # options["ECMWF"]["path_db"] = "./ecmwf" # disbled as it is not needed at the moment # TODO disable_progress_bars? -- GitLab From fb6a03ec2bce4251e8bfabfa65e5fbd29340c021 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Fri, 22 Feb 2019 14:44:10 +0100 Subject: [PATCH 05/24] Updated HISTORY.rst --- HISTORY.rst | 7 ++++--- enpt/model/images.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index c9d8dab..ee3e476 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,12 +2,13 @@ History ======= -0.x.x (coming soon) +0.8.x (coming soon) ------------------- -New features: +New features / improvements: -* TBD +* enhanced AC results due to updated SICOR implementation + (currently dependent from SICOR branch "feature/three_phases_of_water_retieval") 0.7.0 (2019-01-21) diff --git a/enpt/model/images.py b/enpt/model/images.py index 0984818..048ac7a 100644 --- a/enpt/model/images.py +++ b/enpt/model/images.py @@ -388,7 +388,7 @@ class EnMAP_Detector_SensorGeo(_EnMAP_Image): algo = self.cfg.deadpix_P_algorithm method_spectral, method_spatial = self.cfg.deadpix_P_interp_spectral, self.cfg.deadpix_P_interp_spatial self.logger.info("Correcting dead pixels of %s detector...\n" - "Used algorithm / interpolation: %s / %s" + "Used algorithm: %s interpolation in the %s domain" % (self.detector_name, algo, method_spectral if algo == 'spectral' else method_spatial)) self.data = \ -- GitLab From 0716a1c1e32bca5300624cf9c09ed80210431492 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Thu, 7 Mar 2019 15:23:23 +0100 Subject: [PATCH 06/24] Added log messages to orthorectifier. --- enpt/processors/orthorectification/orthorectification.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/enpt/processors/orthorectification/orthorectification.py b/enpt/processors/orthorectification/orthorectification.py index f285e4a..6401094 100644 --- a/enpt/processors/orthorectification/orthorectification.py +++ b/enpt/processors/orthorectification/orthorectification.py @@ -66,13 +66,16 @@ class Orthorectifier(object): # transform VNIR and SWIR to map geometry GeoTransformer = Geometry_Transformer if lons_vnir.ndim == 2 else Geometry_Transformer_3D + enmap_ImageL1.logger.info('Orthorectifying VNIR data...') GT_vnir = GeoTransformer(lons=lons_vnir, lats=lats_vnir, **kw_init) vnir_mapgeo, vnir_gt, vnir_prj = GT_vnir.to_map_geometry(enmap_ImageL1.vnir.data[:], **kw_trafo) + enmap_ImageL1.logger.info('Orthorectifying SWIR data...') GT_swir = GeoTransformer(lons=lons_swir, lats=lats_swir, **kw_init) swir_mapgeo, swir_gt, swir_prj = GT_swir.to_map_geometry(enmap_ImageL1.swir.data[:], **kw_trafo) # combine VNIR and SWIR + enmap_ImageL1.logger.info('Merging VNIR and SWIR data...') L2_obj.data = self._get_VNIR_SWIR_stack(vnir_mapgeo, swir_mapgeo, vnir_gt, swir_gt, vnir_prj, swir_prj, enmap_ImageL1.meta.vnir.wvl_center, enmap_ImageL1.meta.swir.wvl_center) @@ -82,6 +85,7 @@ class Orthorectifier(object): ** kw_init) # FIXME cloud mask applies to BOTH detectors + enmap_ImageL1.logger.info('Orthorectifying cloud mask...') L2_obj.mask_clouds = GeoArray(*GT_2D.to_map_geometry(enmap_ImageL1.vnir.mask_clouds, **kw_trafo)) # TODO transform mask_clouds_confidence, ac_errors, pixel masks @@ -89,6 +93,7 @@ class Orthorectifier(object): # metadata adjustments # ######################## + enmap_ImageL1.logger.info('Generating L2A metadata...') L2_obj.meta = EnMAP_Metadata_L2A_MapGeo(config=self.cfg, meta_l1b=enmap_ImageL1.meta, dims_mapgeo=L2_obj.data.shape, -- GitLab From 6bf8d78e6e5cf4ac9aa1096cd921d32979ad3140 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Thu, 7 Mar 2019 16:38:22 +0100 Subject: [PATCH 07/24] Removed installation of icu=58 within test_install CI job. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 36bfb37..b7a9e09 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -68,7 +68,7 @@ test_enpt_install: # install not pip-installable deps of geoarray - conda install -y -c conda-forge numpy scikit-image matplotlib pandas gdal rasterio pyproj basemap shapely - - conda install -y -c conda-forge 'icu=58.*' # fixes bug for conda-forge gdal build + # - conda install -y -c conda-forge 'icu=58.*' # fixes bug for conda-forge gdal build # install py_tools_ds - pip install 'py_tools_ds>=0.14.11' -- GitLab From f7ba92c986559a4cfa88612a2175e76a539702c1 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Thu, 7 Mar 2019 16:53:39 +0100 Subject: [PATCH 08/24] Updated docker container setup. --- tests/gitlab_CI_docker/build_enpt_testsuite_image.sh | 2 +- tests/gitlab_CI_docker/context/enpt_ci.docker | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh b/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh index ec2072f..7e5eebe 100755 --- a/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh +++ b/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh @@ -2,7 +2,7 @@ context_dir="./context" dockerfile="enpt_ci.docker" -tag="enpt_ci:0.7.0" +tag="enpt_ci:0.7.1" gitlab_runner="enpt_gitlab_CI_runner" # get sicor project diff --git a/tests/gitlab_CI_docker/context/enpt_ci.docker b/tests/gitlab_CI_docker/context/enpt_ci.docker index fe09f8c..411e876 100644 --- a/tests/gitlab_CI_docker/context/enpt_ci.docker +++ b/tests/gitlab_CI_docker/context/enpt_ci.docker @@ -46,3 +46,5 @@ RUN bash -i -c "source /root/miniconda3/bin/activate enpt; \ # -> sicor root directory is the default directory of these cache files if sicor_cache_dir is not set in EnPT options COPY *.zip /tmp/sicor/sicor +# copy AC LUT for EnMAP to SICOR tables directory +COPY EnMAP_LUT* /tmp/sicor/sicor/tables -- GitLab From c26f1297536fd2cd33842113811d24eefbf61ea3 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Thu, 7 Mar 2019 17:50:30 +0100 Subject: [PATCH 09/24] Switched SICOR branch in test_enpt CI job. --- .gitlab-ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b7a9e09..d2aca0b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,6 +17,13 @@ test_enpt: # update py_tools_ds - pip install 'py_tools_ds>=0.14.11' + # switch sicor branch + - rm -rf context/sicor + - git clone -b feature/improve_speed https://gitext.gfz-potsdam.de/EnMAP/sicor.git ./context/sicor + - cd ./context/sicor + - pip install -e . + - cd ../../ + # update sicor # - conda install -y -q -c conda-forge basemap # - rm -rf context/sicor -- GitLab From 927a1871fc0fd9150867d53aef71482deec7ddc8 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Thu, 7 Mar 2019 17:59:40 +0100 Subject: [PATCH 10/24] Changed method of SICOR branch switching. --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d2aca0b..28b9778 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,9 +18,9 @@ test_enpt: - pip install 'py_tools_ds>=0.14.11' # switch sicor branch - - rm -rf context/sicor - - git clone -b feature/improve_speed https://gitext.gfz-potsdam.de/EnMAP/sicor.git ./context/sicor - cd ./context/sicor + - git checkout -b feature/improve_speed + - git pull origin feature/improve_speed - pip install -e . - cd ../../ -- GitLab From 31fb091ca91475a0f862166364bbdc81a59cab92 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Thu, 7 Mar 2019 18:06:12 +0100 Subject: [PATCH 11/24] Changed method of SICOR branch switching once again. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 28b9778..9a9870c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,7 +18,7 @@ test_enpt: - pip install 'py_tools_ds>=0.14.11' # switch sicor branch - - cd ./context/sicor + - cd /tmp/sicor - git checkout -b feature/improve_speed - git pull origin feature/improve_speed - pip install -e . -- GitLab From 292aacafea6bd736f05873f50c589e3037928437 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Thu, 7 Mar 2019 18:15:53 +0100 Subject: [PATCH 12/24] Changed method of SICOR branch switching once again. --- .gitlab-ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9a9870c..c880e4e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,8 +19,12 @@ test_enpt: # switch sicor branch - cd /tmp/sicor - - git checkout -b feature/improve_speed - - git pull origin feature/improve_speed + - mv context/sicor/sicor/tables/*.h5 context/sicor_tables + - mv context/sicor/sicor/tables/*.pkl context/sicor_tables + - rm -rf context/sicor + - git clone -b feature/improve_speed https://gitext.gfz-potsdam.de/EnMAP/sicor.git context/sicor + - mv context/sicor_tables/* context/sicor/sicor/tables + - cd context/sicor - pip install -e . - cd ../../ -- GitLab From 1262f15af025cbf04e411a5dfa2bd3a84dbfdcc5 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Thu, 7 Mar 2019 18:39:53 +0100 Subject: [PATCH 13/24] Changed method of SICOR branch switching once again. --- .gitlab-ci.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c880e4e..a4cfa00 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,13 +18,14 @@ test_enpt: - pip install 'py_tools_ds>=0.14.11' # switch sicor branch + - mkdir /tmp/sicor_tables + - mv /tmp/sicor/sicor/tables/*.h5 /tmp/sicor_tables + - mv /tmp/sicor/sicor/tables/*.pkl /tmp/sicor_tables + - rm -rf /tmp/sicor + - git clone -b feature/improve_speed https://gitext.gfz-potsdam.de/EnMAP/sicor.git /tmp/sicor + - mv /tmp/sicor_tables/* /tmp/sicor/sicor/tables + - rm -rf /tmp/sicor_tables/ - cd /tmp/sicor - - mv context/sicor/sicor/tables/*.h5 context/sicor_tables - - mv context/sicor/sicor/tables/*.pkl context/sicor_tables - - rm -rf context/sicor - - git clone -b feature/improve_speed https://gitext.gfz-potsdam.de/EnMAP/sicor.git context/sicor - - mv context/sicor_tables/* context/sicor/sicor/tables - - cd context/sicor - pip install -e . - cd ../../ -- GitLab From 7414532933382167ca3a24df12871f132097898a Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Thu, 7 Mar 2019 18:49:27 +0100 Subject: [PATCH 14/24] Changed method of SICOR branch switching once again. --- .gitlab-ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a4cfa00..1187549 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,7 +15,7 @@ test_enpt: - export PYTHONPATH=$PYTHONPATH:/root # /root <- here are the sicor tables # update py_tools_ds - - pip install 'py_tools_ds>=0.14.11' + # - pip install 'py_tools_ds>=0.14.11' # switch sicor branch - mkdir /tmp/sicor_tables @@ -25,9 +25,7 @@ test_enpt: - git clone -b feature/improve_speed https://gitext.gfz-potsdam.de/EnMAP/sicor.git /tmp/sicor - mv /tmp/sicor_tables/* /tmp/sicor/sicor/tables - rm -rf /tmp/sicor_tables/ - - cd /tmp/sicor - - pip install -e . - - cd ../../ + - pip install -e /tmp/sicor # update sicor # - conda install -y -q -c conda-forge basemap -- GitLab From 2a813a9aa755a836dc16b6ea5f39f53af9191452 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Thu, 7 Mar 2019 19:42:02 +0100 Subject: [PATCH 15/24] Added log messages to orthorectifier. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1187549..797fc38 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,7 +20,7 @@ test_enpt: # switch sicor branch - mkdir /tmp/sicor_tables - mv /tmp/sicor/sicor/tables/*.h5 /tmp/sicor_tables - - mv /tmp/sicor/sicor/tables/*.pkl /tmp/sicor_tables + - mv /tmp/sicor/sicor/tables/EnMAP_LUT* /tmp/sicor_tables - rm -rf /tmp/sicor - git clone -b feature/improve_speed https://gitext.gfz-potsdam.de/EnMAP/sicor.git /tmp/sicor - mv /tmp/sicor_tables/* /tmp/sicor/sicor/tables -- GitLab From 4f6d78b2a67b5abc47856cbbb9593a360ee51bfc Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Mon, 18 Mar 2019 16:54:12 +0100 Subject: [PATCH 16/24] Fixed loggers failing to de-serialize. Added conda update to test_ept_install. --- .gitlab-ci.yml | 1 + enpt/model/srf.py | 4 ++-- enpt/utils/logging.py | 19 +++++++++++-------- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 797fc38..f934b4b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,6 +70,7 @@ test_enpt_install: stage: test script: - source /root/miniconda3/bin/activate + - conda update -n base -c defaults conda - conda create -y -q --name enpt_test python=3 - source activate enpt_test diff --git a/enpt/model/srf.py b/enpt/model/srf.py index 038fd18..4057743 100644 --- a/enpt/model/srf.py +++ b/enpt/model/srf.py @@ -43,8 +43,8 @@ class SRF(object): :param cwl: target center wavelength position :param fwhm: target band width (full width half maximum) - :param wvl_min: minimum wavelength to compute spectral respnse for - :param wvl_max: maximum wavelength to compute spectral respnse for + :param wvl_min: minimum wavelength to compute spectral response for + :param wvl_max: maximum wavelength to compute spectral response for :param wvl_res: spectral resolution at which spectral response is to be computed :param normalize: whether to normalize the output spectral response to values between 0 and 1 :return: 2D numpy.ndarray: rows: response per wavelength; columns: wavelength/response diff --git a/enpt/utils/logging.py b/enpt/utils/logging.py index 90d3513..0bfa14a 100644 --- a/enpt/utils/logging.py +++ b/enpt/utils/logging.py @@ -26,6 +26,12 @@ class EnPT_Logger(logging.Logger): # private attributes self._captured_stream = '' + # attributes that need to be present in order to unpickle the logger via __setstate_ + self.name_logfile = name_logfile + self.fmt_suffix = fmt_suffix + self.path_logfile = path_logfile + self.log_level = log_level + super(EnPT_Logger, self).__init__(name_logfile) self.path_logfile = path_logfile @@ -54,9 +60,9 @@ class EnPT_Logger(logging.Logger): # create StreamHandler self.streamObj = StringIO() - self.streamHandler = logging.StreamHandler(stream=self.streamObj) - self.streamHandler.setFormatter(self.formatter_fileH) - self.streamHandler.set_name('StringIO handler') + streamHandler = logging.StreamHandler(stream=self.streamObj) + streamHandler.setFormatter(self.formatter_fileH) + streamHandler.set_name('StringIO handler') # create ConsoleHandler for logging levels DEGUG and INFO -> logging to sys.stdout consoleHandler_out = logging.StreamHandler(stream=sys.stdout) # by default it would go to sys.stderr @@ -76,7 +82,7 @@ class EnPT_Logger(logging.Logger): if not self.handlers: if fileHandler: self.addHandler(fileHandler) - self.addHandler(self.streamHandler) + self.addHandler(streamHandler) self.addHandler(consoleHandler_out) self.addHandler(consoleHandler_err) @@ -118,9 +124,8 @@ class EnPT_Logger(logging.Logger): try: if handler.get_name() == 'StringIO handler': self.streamObj.flush() - self.streamHandler.flush() self.removeHandler(handler) # if not called with '[:]' the StreamHandlers are left open - # handler.flush() + handler.flush() handler.close() except PermissionError: warnings.warn('Could not properly close logfile due to a PermissionError: %s' % sys.exc_info()[1]) @@ -128,8 +133,6 @@ class EnPT_Logger(logging.Logger): if self.handlers[:]: warnings.warn('Not all logging handlers could be closed. Remaining handlers: %s' % self.handlers[:]) - # print('sh', self.streamHandler) - def view_logfile(self): """View the log file written to disk.""" with open(self.path_logfile) as inF: -- GitLab From 78380b13b9adedb34b68f8b320dde0b8a298fbd6 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Mon, 18 Mar 2019 17:10:45 +0100 Subject: [PATCH 17/24] Try to install pyresample from conda. --- .gitlab-ci.yml | 1 + enpt/processors/orthorectification/orthorectification.py | 1 + 2 files changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f934b4b..eb0d54d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -82,6 +82,7 @@ test_enpt_install: # - conda install -y -c conda-forge 'icu=58.*' # fixes bug for conda-forge gdal build # install py_tools_ds + - conda install -y -c conda-forge pyresample - pip install 'py_tools_ds>=0.14.11' # install sicor diff --git a/enpt/processors/orthorectification/orthorectification.py b/enpt/processors/orthorectification/orthorectification.py index 6401094..c3573ca 100644 --- a/enpt/processors/orthorectification/orthorectification.py +++ b/enpt/processors/orthorectification/orthorectification.py @@ -60,6 +60,7 @@ class Orthorectifier(object): tgt_extent = self._get_common_extent(enmap_ImageL1, tgt_epsg, enmap_grid=True) kw_init = dict(resamp_alg=self.cfg.ortho_resampAlg, nprocs=self.cfg.CPUs, + # nprocs=1, # FIXME radius_of_influence=30 if not self.cfg.ortho_resampAlg == 'bilinear' else 45) kw_trafo = dict(tgt_prj=tgt_epsg, tgt_extent=tgt_extent) -- GitLab From d24bdb6076c5e3bef221d34ea366de89fd65a130 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Wed, 20 Mar 2019 18:21:17 +0100 Subject: [PATCH 18/24] Added pyresample to conda packages. --- tests/gitlab_CI_docker/context/environment_enpt.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/gitlab_CI_docker/context/environment_enpt.yml b/tests/gitlab_CI_docker/context/environment_enpt.yml index 663248b..199268d 100644 --- a/tests/gitlab_CI_docker/context/environment_enpt.yml +++ b/tests/gitlab_CI_docker/context/environment_enpt.yml @@ -26,6 +26,7 @@ dependencies: - shapely - holoviews - bokeh + - pyresample # arosics - pyfftw -- GitLab From de58463e4ea553f66d0a560926a4c4d85d9e698f Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Wed, 20 Mar 2019 19:27:56 +0100 Subject: [PATCH 19/24] Changed GitLab pages URL. --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index b5da1c2..ab5a71f 100644 --- a/README.rst +++ b/README.rst @@ -33,6 +33,6 @@ This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypack .. _Cookiecutter: https://github.com/audreyr/cookiecutter .. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage -.. _documentation: http://segl.gitext.gfz-potsdam.de/EnPT/doc/ -.. _coverage: http://segl.gitext.gfz-potsdam.de/EnPT/coverage/ -.. _nosetests: http://segl.gitext.gfz-potsdam.de/EnPT/nosetests_reports/nosetests.html \ No newline at end of file +.. _documentation: https://enmap.gitext.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/EnPT/doc/ +.. _coverage: https://enmap.gitext.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/EnPT/coverage/ +.. _nosetests: https://enmap.gitext.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/EnPT/nosetests_reports/nosetests.html \ No newline at end of file -- GitLab From caca17972a31967ba8186a64f928d0ff04905f20 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Thu, 21 Mar 2019 00:20:45 +0100 Subject: [PATCH 20/24] Updated minimal version of py_tools_ds. --- .gitlab-ci.yml | 4 ++-- requirements.txt | 2 +- setup.py | 2 +- tests/gitlab_CI_docker/context/environment_enpt.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 92164cb..62e7851 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,7 +15,7 @@ test_enpt: - export PYTHONPATH=$PYTHONPATH:/root # /root <- here are the sicor tables # update py_tools_ds - - pip install 'py_tools_ds>=0.14.8' + - pip install 'py_tools_ds>=0.14.13' # update sicor # - conda install -y -q -c conda-forge basemap @@ -71,7 +71,7 @@ test_enpt_install: - conda install -y -c conda-forge 'icu=58.*' # fixes bug for conda-forge gdal build # install py_tools_ds - - pip install 'py_tools_ds>=0.14.8' + - pip install 'py_tools_ds>=0.14.13' # install sicor - conda install -y -q -c conda-forge pygrib h5py pytables pyfftw numba llvmlite scikit-learn diff --git a/requirements.txt b/requirements.txt index 2626286..344f8b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy pandas scipy geoarray>=0.8.9 -py_tools_ds>=0.14.8 +py_tools_ds>=0.14.13 cerberus jsmin matplotlib diff --git a/setup.py b/setup.py index b5aa6c8..77e9c7e 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ with open("enpt/version.py") as version_file: exec(version_file.read(), version) requirements = [ # put package requirements here - 'numpy', 'pandas', 'scipy', 'geoarray>=0.8.9', 'py_tools_ds>=0.14.8', 'cerberus', 'jsmin', 'matplotlib', 'tqdm', + 'numpy', 'pandas', 'scipy', 'geoarray>=0.8.9', 'py_tools_ds>=0.14.13', 'cerberus', 'jsmin', 'matplotlib', 'tqdm', 'utm', 'lxml', 'numpy-indexed' # 'sicor', # pip install git+https://gitext.gfz-potsdam.de/EnMAP/sicor.git ] diff --git a/tests/gitlab_CI_docker/context/environment_enpt.yml b/tests/gitlab_CI_docker/context/environment_enpt.yml index 00aa1be..403d6de 100644 --- a/tests/gitlab_CI_docker/context/environment_enpt.yml +++ b/tests/gitlab_CI_docker/context/environment_enpt.yml @@ -44,7 +44,7 @@ dependencies: - pip: - scipy - geoarray>=0.8.9 - - py_tools_ds>=0.14.8 + - py_tools_ds>=0.14.13 - cerberus - jsmin - tqdm -- GitLab From 366cd084e0e80e086b5ed76bd63acdc89a96c9a1 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Thu, 21 Mar 2019 15:40:42 +0100 Subject: [PATCH 21/24] Updated minimal version of py_tools_ds. --- .gitlab-ci.yml | 4 ++-- requirements.txt | 2 +- setup.py | 2 +- tests/gitlab_CI_docker/context/environment_enpt.yml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 62e7851..2163f8e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,7 +15,7 @@ test_enpt: - export PYTHONPATH=$PYTHONPATH:/root # /root <- here are the sicor tables # update py_tools_ds - - pip install 'py_tools_ds>=0.14.13' + - pip install 'py_tools_ds>=0.14.14' # update sicor # - conda install -y -q -c conda-forge basemap @@ -71,7 +71,7 @@ test_enpt_install: - conda install -y -c conda-forge 'icu=58.*' # fixes bug for conda-forge gdal build # install py_tools_ds - - pip install 'py_tools_ds>=0.14.13' + - pip install 'py_tools_ds>=0.14.14' # install sicor - conda install -y -q -c conda-forge pygrib h5py pytables pyfftw numba llvmlite scikit-learn diff --git a/requirements.txt b/requirements.txt index 344f8b8..34bae14 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy pandas scipy geoarray>=0.8.9 -py_tools_ds>=0.14.13 +py_tools_ds>=0.14.14 cerberus jsmin matplotlib diff --git a/setup.py b/setup.py index 77e9c7e..f0f60bd 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ with open("enpt/version.py") as version_file: exec(version_file.read(), version) requirements = [ # put package requirements here - 'numpy', 'pandas', 'scipy', 'geoarray>=0.8.9', 'py_tools_ds>=0.14.13', 'cerberus', 'jsmin', 'matplotlib', 'tqdm', + 'numpy', 'pandas', 'scipy', 'geoarray>=0.8.9', 'py_tools_ds>=0.14.14', 'cerberus', 'jsmin', 'matplotlib', 'tqdm', 'utm', 'lxml', 'numpy-indexed' # 'sicor', # pip install git+https://gitext.gfz-potsdam.de/EnMAP/sicor.git ] diff --git a/tests/gitlab_CI_docker/context/environment_enpt.yml b/tests/gitlab_CI_docker/context/environment_enpt.yml index 403d6de..22e395f 100644 --- a/tests/gitlab_CI_docker/context/environment_enpt.yml +++ b/tests/gitlab_CI_docker/context/environment_enpt.yml @@ -44,7 +44,7 @@ dependencies: - pip: - scipy - geoarray>=0.8.9 - - py_tools_ds>=0.14.13 + - py_tools_ds>=0.14.14 - cerberus - jsmin - tqdm -- GitLab From a717e67710f876329063b8c6fb6bb2df08a8d8eb Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Thu, 21 Mar 2019 18:01:16 +0100 Subject: [PATCH 22/24] Disabled AC to avoid deadlock (so far not solved issue within SICOR). --- enpt/options/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enpt/options/config.py b/enpt/options/config.py index 491d6e3..17ae3b6 100644 --- a/enpt/options/config.py +++ b/enpt/options/config.py @@ -65,7 +65,7 @@ config_for_testing_dlr = dict( n_lines_to_append=50, disable_progress_bars=True, is_dlr_dataformat=True, - enable_ac=True, + enable_ac=False, enable_ice_retrieval=False, CPUs=32, ortho_resampAlg='gauss', -- GitLab From 6f2da5a7e0b18bede6c32a0abf0e4a7c5d76efd2 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Thu, 21 Mar 2019 18:07:14 +0100 Subject: [PATCH 23/24] Disabled AC for Alpine data to avoid deadlock (so far not solved issue within SICOR). --- enpt/options/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enpt/options/config.py b/enpt/options/config.py index 17ae3b6..478e595 100644 --- a/enpt/options/config.py +++ b/enpt/options/config.py @@ -43,7 +43,7 @@ config_for_testing = dict( n_lines_to_append=50, disable_progress_bars=True, is_dlr_dataformat=False, - enable_ac=True, + enable_ac=False, enable_ice_retrieval=False, CPUs=16 ) -- GitLab From 42e649f350b30d090104db04631093d874979259 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Fri, 22 Mar 2019 19:22:26 +0100 Subject: [PATCH 24/24] Cleaned up. --- enpt/processors/orthorectification/orthorectification.py | 1 - 1 file changed, 1 deletion(-) diff --git a/enpt/processors/orthorectification/orthorectification.py b/enpt/processors/orthorectification/orthorectification.py index c3573ca..6401094 100644 --- a/enpt/processors/orthorectification/orthorectification.py +++ b/enpt/processors/orthorectification/orthorectification.py @@ -60,7 +60,6 @@ class Orthorectifier(object): tgt_extent = self._get_common_extent(enmap_ImageL1, tgt_epsg, enmap_grid=True) kw_init = dict(resamp_alg=self.cfg.ortho_resampAlg, nprocs=self.cfg.CPUs, - # nprocs=1, # FIXME radius_of_influence=30 if not self.cfg.ortho_resampAlg == 'bilinear' else 45) kw_trafo = dict(tgt_prj=tgt_epsg, tgt_extent=tgt_extent) -- GitLab