diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 61f52c86924a00aaf694094167de8facf879127e..8f68d19ca83954f88e9309a2f6c517576268640d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,6 +14,9 @@ test_enpt: - export GDAL_DATA=/root/miniconda3/envs/enpt/share/gdal - export PYTHONPATH=$PYTHONPATH:/root # /root <- here are the sicor tables + # update geoarray + - pip install 'geoarray>=0.8.11' + # update py_tools_ds - pip install 'py_tools_ds>=0.14.14' diff --git a/bin/enpt_cli.py b/bin/enpt_cli.py index e7f8496366bcf56874beef4e8b4b6a40bda11e37..f59fcbcbdf55126907adb7705ca38623e17d7db1 100644 --- a/bin/enpt_cli.py +++ b/bin/enpt_cli.py @@ -42,6 +42,9 @@ def get_enpt_argparser(): help='input path of digital elevation model in map or sensor geometry; GDAL compatible file format ' '(must cover the EnMAP L1B data completely if given in map geometry or must have the same pixel ' 'dimensions like the EnMAP L1B data if given in sensor geometry)') + add('-dummyfmt', '--is_dummy_dataformat', type=bool, default=False, + help='Set to true in case of the preliminary, GFZ-internal dataformat as used for the Alpine test dataset. ' + '(default: False. Note: This will be removed in future.)') add('-ele', '--average_elevation', type=int, default=0, help='average elevation in meters above sea level; may be provided if no DEM is available; ' 'ignored if DEM is given') diff --git a/enpt/execution/controller.py b/enpt/execution/controller.py index f2fd38ae7282dbddb58329695cd21ad3d148b1dd..97b37b23fe8d6eb21d6f697e0e1ff1c5eee3746c 100644 --- a/enpt/execution/controller.py +++ b/enpt/execution/controller.py @@ -42,7 +42,7 @@ class EnPT_Controller(object): :param subdir: subdirectory name to be created within temporary directory :return: /tmp/tmpk2qp0yri/rootdir/ """ - outdir = os.path.join(self.tempDir, subdir) if self.cfg.is_dlr_dataformat else self.tempDir + outdir = os.path.join(self.tempDir, subdir) if not self.cfg.is_dummy_dataformat else self.tempDir with zipfile.ZipFile(path_zipfile, "r") as zf: zf.extractall(outdir) @@ -50,7 +50,7 @@ class EnPT_Controller(object): if not os.path.isdir(outdir): raise NotADirectoryError(outdir) - if self.cfg.is_dlr_dataformat: + if not self.cfg.is_dummy_dataformat: return outdir else: return os.path.join(self.tempDir, os.path.basename(path_zipfile).split('.zip')[0]) diff --git a/enpt/model/images.py b/enpt/model/images.py index 048ac7affa9d152939bba991915673fb13bded28..b8896219c1674f2302a363845091cb59cca42dcb 100644 --- a/enpt/model/images.py +++ b/enpt/model/images.py @@ -445,7 +445,7 @@ class EnMAP_Detector_SensorGeo(_EnMAP_Image): :param n_lines: number of line to be added :return: None """ - if self.cfg.is_dlr_dataformat: + if not self.cfg.is_dummy_dataformat: basename_img1 = self.detector_meta.data_filename.split('-SPECTRAL_IMAGE')[0] + '::%s' % self.detector_name basename_img2 = img2.detector_meta.data_filename.split('-SPECTRAL_IMAGE')[0] + '::%s' % img2.detector_name else: @@ -489,7 +489,7 @@ class EnMAP_Detector_SensorGeo(_EnMAP_Image): self.detector_meta.nrows += n_lines # Create new corner coordinate - if self.cfg.is_dlr_dataformat: + if not self.cfg.is_dummy_dataformat: enmapIm_cornerCoords = tuple(zip(img2.detector_meta.lon_UL_UR_LL_LR, img2.detector_meta.lat_UL_UR_LL_LR)) dem_validated = DEM_Processor(img2.cfg.path_dem, @@ -532,7 +532,7 @@ class EnMAP_Detector_SensorGeo(_EnMAP_Image): # append the raster data self.data = np.append(self.data, img2.data[0:n_lines, :, :], axis=0) self.mask_clouds = np.append(self.mask_clouds, img2.mask_clouds[0:n_lines, :], axis=0) - if self.cfg.is_dlr_dataformat: + if not self.cfg.is_dummy_dataformat: self.deadpixelmap = np.append(self.deadpixelmap, img2.deadpixelmap[0:n_lines, :], axis=0) # TODO append remaining raster layers - additional cloud masks, ... @@ -620,7 +620,7 @@ class EnMAPL1Product_SensorGeo(object): self.logger = logger # Read metadata here in order to get all information needed by to get paths. - if self.cfg.is_dlr_dataformat: + if not self.cfg.is_dummy_dataformat: self.meta = EnMAP_Metadata_L1B_SensorGeo(glob(path.join(root_dir, "*METADATA.XML"))[0], config=self.cfg, logger=self.logger) else: diff --git a/enpt/model/metadata.py b/enpt/model/metadata.py index 4afb3595fc8bff77efea7dbd09647d4da523a3dd..3f428d7bd7c974fe922d4ff5ab54ef8ad10e608c 100644 --- a/enpt/model/metadata.py +++ b/enpt/model/metadata.py @@ -77,7 +77,7 @@ class EnMAP_Metadata_L1B_Detector_SensorGeo(object): """ self.cfg = config self.detector_name = detector_name # type: str - if self.cfg.is_dlr_dataformat: + if not self.cfg.is_dummy_dataformat: self.detector_label = L1B_product_props_DLR['xml_detector_label'][detector_name] else: self.detector_label = L1B_product_props['xml_detector_label'][detector_name] @@ -125,7 +125,7 @@ class EnMAP_Metadata_L1B_Detector_SensorGeo(object): """ xml = ElementTree.parse(path_xml).getroot() - if self.cfg.is_dlr_dataformat: + if not self.cfg.is_dummy_dataformat: lbl = self.detector_label self.logger.info("Reading metadata for %s detector..." % self.detector_name) @@ -472,7 +472,7 @@ class EnMAP_Metadata_L1B_SensorGeo(object): self.metaxml_filename = os.path.basename(path_xml) - if self.cfg.is_dlr_dataformat: + if not self.cfg.is_dummy_dataformat: # read processing level self.proc_level = xml.find("base/level").text if self.proc_level != 'L1B': @@ -558,7 +558,7 @@ class EnMAP_Metadata_L1B_SensorGeo(object): """ xml = ElementTree.parse(self.path_xml).getroot() - if self.cfg.is_dlr_dataformat: + if not self.cfg.is_dummy_dataformat: for detName, detMeta in zip(['VNIR', 'SWIR'], [self.vnir, self.swir]): lbl = L1B_product_props_DLR['xml_detector_label'][detName] xml.find("product/image/%s/dimension/rows" % lbl).text = str(detMeta.nrows) @@ -611,7 +611,7 @@ class EnMAP_Metadata_L2A_MapGeo(object): self.earthSunDist = meta_l1b.earthSunDist # type: float # earth-sun distance # generate file names for L2A output - if self.cfg.is_dlr_dataformat: + if not self.cfg.is_dummy_dataformat: self.scene_basename = meta_l1b.vnir.data_filename.split('-SPECTRAL_IMAGE')[0].replace('L1B-', 'L2A-') else: self.scene_basename = os.path.splitext(meta_l1b.vnir.data_filename)[0] @@ -644,7 +644,7 @@ class EnMAP_Metadata_L2A_MapGeo(object): self.smile_coef = np.vstack([meta_l1b.vnir.smile_coef, meta_l1b.swir.smile_coef])[bandidx_order, :] self.smile = np.hstack([meta_l1b.vnir.smile, meta_l1b.swir.smile])[:, bandidx_order] - if self.cfg.is_dlr_dataformat: + if not self.cfg.is_dummy_dataformat: self.rpc_coeffs = OrderedDict(zip( ['band_%d' % (i + 1) for i in range(dims_mapgeo[2])], [meta_l1b.vnir.rpc_coeffs['band_%d' % (i + 1)] if 'band_%d' % (i + 1) in meta_l1b.vnir.rpc_coeffs else @@ -730,7 +730,7 @@ class EnMAP_Metadata_L2A_MapGeo(object): # parse (use L1B metadata as template) xml = ElementTree.parse(self._meta_l1b.path_xml, parser).getroot() - if not self.cfg.is_dlr_dataformat: + if self.cfg.is_dummy_dataformat: self.logger.warning('No XML metadata conversion implemented for datasets different to the DLR format.' 'Metadata XML file will be empty.') return '' diff --git a/enpt/options/config.py b/enpt/options/config.py index 478e595d9da85e911c7a33103476bc20cef17524..0a1614eac2a8b50071b3927d05e5db799b3057d5 100644 --- a/enpt/options/config.py +++ b/enpt/options/config.py @@ -42,7 +42,7 @@ config_for_testing = dict( output_dir=os.path.join(path_enptlib, '..', 'tests', 'data', 'test_outputs'), n_lines_to_append=50, disable_progress_bars=True, - is_dlr_dataformat=False, + is_dummy_dataformat=True, enable_ac=False, enable_ice_retrieval=False, CPUs=16 @@ -64,7 +64,7 @@ config_for_testing_dlr = dict( output_dir=os.path.join(path_enptlib, '..', 'tests', 'data', 'test_outputs'), n_lines_to_append=50, disable_progress_bars=True, - is_dlr_dataformat=True, + is_dummy_dataformat=False, enable_ac=False, enable_ice_retrieval=False, CPUs=32, @@ -107,10 +107,12 @@ class EnPTConfig(object): # general options # ################### - try: - self.is_dlr_dataformat = gp('is_dlr_dataformat') - except: # noqa E722 # FIXME - self.is_dlr_dataformat = False + self.is_dummy_dataformat = gp('is_dummy_dataformat') + if 'is_dlr_dataformat' in user_opts: + warnings.warn("The 'is_dlr_dataformat' flag is deprectated and will not exist in future. " + "Please set 'is_dummy_dataformat' to False instead.", DeprecationWarning) + self.is_dummy_dataformat = user_opts['is_dlr_dataformat'] is False + self.CPUs = gp('CPUs', fallback=cpu_count()) self.log_level = gp('log_level') self.create_logfile = gp('create_logfile') diff --git a/enpt/options/options_default.json b/enpt/options/options_default.json index a1327714fd26ef5a1a0566b79090faf4443bf1d0..2531d343b4a70c3ebf11018e98671760e246bb6a 100644 --- a/enpt/options/options_default.json +++ b/enpt/options/options_default.json @@ -9,6 +9,8 @@ "path_dem": "", /*input path of digital elevation model in map or sensor geometry; GDAL compatible file format (must cover the EnMAP L1B data completely if given in map geometry or must have the same pixel dimensions like the EnMAP L1B data if given in sensor geometry)*/ + "is_dummy_dataformat": false, /*Set to true in case of the preliminary, GFZ-internal dataformat as used for the + Alpine test dataset. (default: False. Note: This will be removed in future.)*/ "average_elevation": 0, /*average elevation in meters above sea level; may be provided if no DEM is available; ignored if DEM is given*/ "path_l1b_snr_model": "./resources/EnMAP_Sensor/EnMAP_Level_1B_SNR.zip", /*input path of the EnMAP SNR model*/ diff --git a/enpt/options/options_schema.py b/enpt/options/options_schema.py index 1da9b4bcc39e2aa148ba388aff1437d331e33d9d..cc63d781f887ab03a5c99810ad383e86a92c042f 100644 --- a/enpt/options/options_schema.py +++ b/enpt/options/options_schema.py @@ -10,6 +10,7 @@ enpt_schema_input = dict( path_l1b_enmap_image=dict(type='string', required=False), path_l1b_enmap_image_gapfill=dict(type='string', required=False), path_dem=dict(type='string', required=False), + is_dummy_dataformat=dict(type='boolean', required=False), average_elevation=dict(type='integer', required=False), path_l1b_snr_model=dict(type='string', required=False), working_dir=dict(type='string', required=False, nullable=True), @@ -85,6 +86,7 @@ parameter_mapping = dict( path_l1b_enmap_image=('general_opts', 'path_l1b_enmap_image'), path_l1b_enmap_image_gapfill=('general_opts', 'path_l1b_enmap_image_gapfill'), path_dem=('general_opts', 'path_dem'), + is_dummy_dataformat=('general_opts', 'is_dummy_dataformat'), average_elevation=('general_opts', 'average_elevation'), path_l1b_snr_model=('general_opts', 'path_l1b_snr_model'), working_dir=('general_opts', 'working_dir'), diff --git a/enpt/version.py b/enpt/version.py index ec828856c552f9b2128c3578fc12ad6fada3b3fb..a1b766c2a8cbaa4c3d97537049b98317b346a226 100644 --- a/enpt/version.py +++ b/enpt/version.py @@ -1,2 +1,2 @@ -__version__ = '0.7.0' -__versionalias__ = '20180121.01' +__version__ = '0.7.1' +__versionalias__ = '20180305.01' diff --git a/requirements.txt b/requirements.txt index 34bae14158efdeb673a11fdd12d9f33051c6c2b8..8dc0cad9c3a6966884098ebc3864a17aad03d1ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ numpy pandas scipy -geoarray>=0.8.9 +geoarray>=0.8.11 py_tools_ds>=0.14.14 cerberus jsmin diff --git a/setup.py b/setup.py index f0f60bde6487ca3a650cfc62c92795b5c44797e5..3e2f0c4ac76fefaf2eccf5959b88902784552f0e 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ with open("enpt/version.py") as version_file: exec(version_file.read(), version) requirements = [ # put package requirements here - 'numpy', 'pandas', 'scipy', 'geoarray>=0.8.9', 'py_tools_ds>=0.14.14', 'cerberus', 'jsmin', 'matplotlib', 'tqdm', + 'numpy', 'pandas', 'scipy', 'geoarray>=0.8.11', 'py_tools_ds>=0.14.14', 'cerberus', 'jsmin', 'matplotlib', 'tqdm', 'utm', 'lxml', 'numpy-indexed' # 'sicor', # pip install git+https://gitext.gfz-potsdam.de/EnMAP/sicor.git ] diff --git a/tests/gitlab_CI_docker/context/environment_enpt.yml b/tests/gitlab_CI_docker/context/environment_enpt.yml index e00340468bb94d8c123f7355d1a221bb9c62956c..3a4f8d136675f7595e62d1065c88764cde34f89d 100644 --- a/tests/gitlab_CI_docker/context/environment_enpt.yml +++ b/tests/gitlab_CI_docker/context/environment_enpt.yml @@ -44,7 +44,7 @@ dependencies: - pip: - scipy - - geoarray>=0.8.9 + - geoarray>=0.8.11 - py_tools_ds>=0.14.14 - cerberus - jsmin diff --git a/tests/linting/pydocstyle.log b/tests/linting/pydocstyle.log index 402b1727f27c97ce746aa76883e0aa51c7908001..5c95b0ca4b4acb9563d0bef12236073013ddbbe9 100644 --- a/tests/linting/pydocstyle.log +++ b/tests/linting/pydocstyle.log @@ -88,15 +88,15 @@ enpt/utils/path_generator.py:15 in public class `PathGenL1BProduct`: D204: 1 blank line required after class docstring (found 0) enpt/utils/path_generator.py:58 in public function `get_path_ac_options`: D401: First line should be in imperative mood ('Return', not 'Returns') -enpt/utils/logging.py:83 in public method `__getstate__`: +enpt/utils/logging.py:89 in public method `__getstate__`: D105: Missing docstring in magic method -enpt/utils/logging.py:87 in public method `__setstate__`: +enpt/utils/logging.py:93 in public method `__setstate__`: D401: First line should be in imperative mood ('Define', not 'Defines') -enpt/utils/logging.py:138 in public method `__del__`: +enpt/utils/logging.py:141 in public method `__del__`: D105: Missing docstring in magic method -enpt/utils/logging.py:141 in public method `__enter__`: +enpt/utils/logging.py:144 in public method `__enter__`: D105: Missing docstring in magic method -enpt/utils/logging.py:144 in public method `__exit__`: +enpt/utils/logging.py:147 in public method `__exit__`: D105: Missing docstring in magic method enpt/processors/spatial_transform/spatial_transform.py:22 in public class `Geometry_Transformer`: D101: Missing docstring in public class @@ -148,9 +148,9 @@ enpt/processors/dem_preprocessing/dem_preprocessing.py:65 in public method `to_s D102: Missing docstring in public method enpt/processors/dead_pixel_correction/dead_pixel_correction.py:18 in public class `Dead_Pixel_Corrector`: D101: Missing docstring in public class -enpt/processors/atmospheric_correction/atmospheric_correction.py:25 in public method `get_ac_options`: +enpt/processors/atmospheric_correction/atmospheric_correction.py:27 in public method `get_ac_options`: D102: Missing docstring in public method -enpt/processors/atmospheric_correction/atmospheric_correction.py:47 in public method `run_ac`: +enpt/processors/atmospheric_correction/atmospheric_correction.py:50 in public method `run_ac`: D102: Missing docstring in public method enpt/processors/orthorectification/__init__.py:1 at module level: D205: 1 blank line required between summary line and description (found 0) @@ -176,37 +176,37 @@ enpt/options/config.py:81 in public class `EnPTConfig`: D101: Missing docstring in public class enpt/options/config.py:82 in public method `__init__`: D202: No blank lines allowed after function docstring (found 1) -enpt/options/config.py:172 in public method `absPath`: +enpt/options/config.py:174 in public method `absPath`: D102: Missing docstring in public method -enpt/options/config.py:175 in public method `get_parameter`: +enpt/options/config.py:177 in public method `get_parameter`: D102: Missing docstring in public method -enpt/options/config.py:238 in public method `to_dict`: +enpt/options/config.py:240 in public method `to_dict`: D202: No blank lines allowed after function docstring (found 1) -enpt/options/config.py:252 in public method `to_jsonable_dict`: +enpt/options/config.py:254 in public method `to_jsonable_dict`: D102: Missing docstring in public method -enpt/options/config.py:263 in public method `__repr__`: +enpt/options/config.py:265 in public method `__repr__`: D105: Missing docstring in magic method -enpt/options/config.py:267 in public function `json_to_python`: +enpt/options/config.py:269 in public function `json_to_python`: D103: Missing docstring in public function -enpt/options/config.py:300 in public function `python_to_json`: +enpt/options/config.py:302 in public function `python_to_json`: D103: Missing docstring in public function -enpt/options/config.py:322 in public class `EnPTValidator`: +enpt/options/config.py:324 in public class `EnPTValidator`: D101: Missing docstring in public class -enpt/options/config.py:323 in public method `__init__`: +enpt/options/config.py:325 in public method `__init__`: D205: 1 blank line required between summary line and description (found 0) -enpt/options/config.py:323 in public method `__init__`: +enpt/options/config.py:325 in public method `__init__`: D400: First line should end with a period (not 'r') -enpt/options/config.py:331 in public method `validate`: +enpt/options/config.py:333 in public method `validate`: D102: Missing docstring in public method -enpt/options/config.py:336 in public function `get_options`: +enpt/options/config.py:338 in public function `get_options`: D202: No blank lines allowed after function docstring (found 1) enpt/options/__init__.py:1 at module level: D104: Missing docstring in public package -enpt/options/options_schema.py:128 in public function `get_updated_schema`: +enpt/options/options_schema.py:130 in public function `get_updated_schema`: D103: Missing docstring in public function -enpt/options/options_schema.py:129 in private nested function `deep_update`: +enpt/options/options_schema.py:131 in private nested function `deep_update`: D202: No blank lines allowed after function docstring (found 1) -enpt/options/options_schema.py:129 in private nested function `deep_update`: +enpt/options/options_schema.py:131 in private nested function `deep_update`: D400: First line should end with a period (not 'e') -enpt/options/options_schema.py:148 in public function `get_param_from_json_config`: +enpt/options/options_schema.py:150 in public function `get_param_from_json_config`: D103: Missing docstring in public function