From 2b8ab22f0f74fb7d718b884baae2b664900bcdf6 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Fri, 18 Jun 2021 16:07:31 +0200 Subject: [PATCH 1/3] Fixed #79 (FileNotFoundError when using the official EnMAP test data downloaded from enmap.org). Signed-off-by: Daniel Scheffler --- HISTORY.rst | 6 ++++++ enpt/execution/controller.py | 10 ++++++++++ enpt/io/reader.py | 37 ++++++++++++++++++------------------ enpt/options/config.py | 9 ++++++--- 4 files changed, 41 insertions(+), 21 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2cdc841..dcc7da2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,12 @@ History ======= +0.18.3 (2021-06-18) +------------------- + +* Fixed #79 (FileNotFoundError when using the official EnMAP test data downloaded from enmap.org). + + 0.18.2 (2021-06-17) ------------------- diff --git a/enpt/execution/controller.py b/enpt/execution/controller.py index 14e3a5e..5203133 100644 --- a/enpt/execution/controller.py +++ b/enpt/execution/controller.py @@ -40,6 +40,7 @@ import pickle from typing import Optional from time import time from datetime import timedelta +from glob import glob from ..options.config import EnPTConfig from ..io.reader import L1B_Reader @@ -83,8 +84,17 @@ class EnPT_Controller(object): outdir = os.path.join(self.tempDir, subdir) if not self.cfg.is_dummy_dataformat else self.tempDir with zipfile.ZipFile(path_zipfile, "r") as zf: + for file in zf.namelist(): + f = file zf.extractall(outdir) + # move the data one level up in case they are within a sub-folder in the zip file + content = glob(os.path.join(outdir, '*')) + if len(content) == 1 and os.path.isdir(content[0]): + for fp in glob(os.path.join(outdir, '**', '*')): + shutil.move(fp, outdir) + shutil.rmtree(content[0]) + if not os.path.isdir(outdir): raise NotADirectoryError(outdir) diff --git a/enpt/io/reader.py b/enpt/io/reader.py index 4428731..8bd3190 100644 --- a/enpt/io/reader.py +++ b/enpt/io/reader.py @@ -127,24 +127,25 @@ class L1B_Reader(object): if not files: raise RuntimeError("The root directory of the EnMAP image %s is empty." % rootdir_l1b) - for pattern in ['*-HISTORY.XML', - '*-LOG.XML', - '*-METADATA.XML', - '*-QL_PIXELMASK_SWIR.TIF', - '*-QL_PIXELMASK_VNIR.TIF', - '*-QL_QUALITY_CIRRUS.TIF', - '*-QL_QUALITY_CLASSES.TIF', - '*-QL_QUALITY_CLOUD.TIF', - '*-QL_QUALITY_CLOUDSHADOW.TIF', - '*-QL_QUALITY_HAZE.TIF', - '*-QL_QUALITY_SNOW.TIF', - '*-QL_QUALITY_TESTFLAGS_SWIR.TIF', - '*-QL_QUALITY_TESTFLAGS_VNIR.TIF', - '*-QL_SWIR.TIF', - '*-QL_VNIR.TIF', - '*-SPECTRAL_IMAGE_SWIR.TIF', - '*-SPECTRAL_IMAGE_VNIR.TIF', - ]: + for pattern in [ + # '*-HISTORY.XML', # only included in internal DLR test data, not in the zip archive on enmap.org + # '*-LOG.XML', # only included in internal DLR test data, not in the zip archive on enmap.org + '*-METADATA.XML', + '*-QL_PIXELMASK_SWIR.TIF', + '*-QL_PIXELMASK_VNIR.TIF', + '*-QL_QUALITY_CIRRUS.TIF', + '*-QL_QUALITY_CLASSES.TIF', + '*-QL_QUALITY_CLOUD.TIF', + '*-QL_QUALITY_CLOUDSHADOW.TIF', + '*-QL_QUALITY_HAZE.TIF', + '*-QL_QUALITY_SNOW.TIF', + '*-QL_QUALITY_TESTFLAGS_SWIR.TIF', + '*-QL_QUALITY_TESTFLAGS_VNIR.TIF', + '*-QL_SWIR.TIF', + '*-QL_VNIR.TIF', + '*-SPECTRAL_IMAGE_SWIR.TIF', + '*-SPECTRAL_IMAGE_VNIR.TIF', + ]: if not filter(files, pattern) and not filter(files, pattern.replace('.TIF', '.GEOTIFF')): raise FileNotFoundError('The root directory of the EnMAP image %s misses a file with the pattern %s.' % (rootdir_l1b, pattern)) diff --git a/enpt/options/config.py b/enpt/options/config.py index 8c4d4d6..e7901dc 100644 --- a/enpt/options/config.py +++ b/enpt/options/config.py @@ -139,7 +139,7 @@ config_for_testing_dlr = dict( # 'ENMAP01-____L1B-DT000000987_20130205T105307Z_001_V000101_20190426T143700Z.zip' # Arcachon - 'ENMAP01-____L1B-DT000400126_20170218T110115Z_002_V000204_20200206T182719Z__rows700-799.zip' + # 'ENMAP01-____L1B-DT000400126_20170218T110115Z_002_V000204_20200206T182719Z__rows700-799.zip' # Arcachon 1000x30 # 'ENMAP01-____L1B-DT000400126_20170218T110115Z_002_V000204_20200206T182719Z__rows700-730.zip' @@ -149,6 +149,9 @@ config_for_testing_dlr = dict( # Arcachon full tile 3, reprocessed 05/2020 # 'ENMAP01-____L1B-DT000400126_20170218T110119Z_003_V000204_20200508T124425Z.zip' + + # Arcachon tile 3 (full), downloaded from enmap.org + 'L1B_Arcachon_3__enmap.org.zip', )), # path_l1b_enmap_image_gapfill=os.path.abspath( # os.path.join(path_enptlib, '..', 'tests', 'data', 'EnMAP_Level_1B', @@ -164,10 +167,10 @@ config_for_testing_dlr = dict( # 'DLR_L2A_DEM_UTM32.bsq' # Arcachon tile 2 ASTER DEM (02/2020) - 'ENMAP01-____L1B-DT000400126_20170218T110115Z_002_V000204_20200206T182719Z__tile2__DEM_ASTER.bsq' + # 'ENMAP01-____L1B-DT000400126_20170218T110115Z_002_V000204_20200206T182719Z__tile2__DEM_ASTER.bsq' # Arcachon tile 3 ASTER DEM (05/2020) - # 'ENMAP01-____L1B-DT000400126_20170218T110119Z_003_V000204_20200508T124425Z__tile3__DEM_ASTER.bsq' + 'ENMAP01-____L1B-DT000400126_20170218T110119Z_003_V000204_20200508T124425Z__tile3__DEM_ASTER.bsq' # '15_DEM_UTM__with_prj.tif' )), log_level='DEBUG', -- GitLab From 1c26bc8e50459b74382eaf32faa608bc09544970 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Fri, 18 Jun 2021 16:15:06 +0200 Subject: [PATCH 2/3] Use small Arcachon test data for CI. Signed-off-by: Daniel Scheffler --- enpt/options/config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/enpt/options/config.py b/enpt/options/config.py index e7901dc..dae693a 100644 --- a/enpt/options/config.py +++ b/enpt/options/config.py @@ -139,7 +139,7 @@ config_for_testing_dlr = dict( # 'ENMAP01-____L1B-DT000000987_20130205T105307Z_001_V000101_20190426T143700Z.zip' # Arcachon - # 'ENMAP01-____L1B-DT000400126_20170218T110115Z_002_V000204_20200206T182719Z__rows700-799.zip' + 'ENMAP01-____L1B-DT000400126_20170218T110115Z_002_V000204_20200206T182719Z__rows700-799.zip' # Arcachon 1000x30 # 'ENMAP01-____L1B-DT000400126_20170218T110115Z_002_V000204_20200206T182719Z__rows700-730.zip' @@ -151,7 +151,7 @@ config_for_testing_dlr = dict( # 'ENMAP01-____L1B-DT000400126_20170218T110119Z_003_V000204_20200508T124425Z.zip' # Arcachon tile 3 (full), downloaded from enmap.org - 'L1B_Arcachon_3__enmap.org.zip', + # 'L1B_Arcachon_3__enmap.org.zip', )), # path_l1b_enmap_image_gapfill=os.path.abspath( # os.path.join(path_enptlib, '..', 'tests', 'data', 'EnMAP_Level_1B', @@ -167,10 +167,10 @@ config_for_testing_dlr = dict( # 'DLR_L2A_DEM_UTM32.bsq' # Arcachon tile 2 ASTER DEM (02/2020) - # 'ENMAP01-____L1B-DT000400126_20170218T110115Z_002_V000204_20200206T182719Z__tile2__DEM_ASTER.bsq' + 'ENMAP01-____L1B-DT000400126_20170218T110115Z_002_V000204_20200206T182719Z__tile2__DEM_ASTER.bsq' # Arcachon tile 3 ASTER DEM (05/2020) - 'ENMAP01-____L1B-DT000400126_20170218T110119Z_003_V000204_20200508T124425Z__tile3__DEM_ASTER.bsq' + # 'ENMAP01-____L1B-DT000400126_20170218T110119Z_003_V000204_20200508T124425Z__tile3__DEM_ASTER.bsq' # '15_DEM_UTM__with_prj.tif' )), log_level='DEBUG', -- GitLab From c7992cbdad1584e92dfa5d32a4c7b99dadb4dd12 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Fri, 18 Jun 2021 17:03:42 +0200 Subject: [PATCH 3/3] Revised the fix to make it also work with the dummy data format. Signed-off-by: Daniel Scheffler --- enpt/execution/controller.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/enpt/execution/controller.py b/enpt/execution/controller.py index 5203133..b519963 100644 --- a/enpt/execution/controller.py +++ b/enpt/execution/controller.py @@ -84,16 +84,16 @@ class EnPT_Controller(object): outdir = os.path.join(self.tempDir, subdir) if not self.cfg.is_dummy_dataformat else self.tempDir with zipfile.ZipFile(path_zipfile, "r") as zf: - for file in zf.namelist(): - f = file zf.extractall(outdir) # move the data one level up in case they are within a sub-folder in the zip file - content = glob(os.path.join(outdir, '*')) - if len(content) == 1 and os.path.isdir(content[0]): - for fp in glob(os.path.join(outdir, '**', '*')): - shutil.move(fp, outdir) - shutil.rmtree(content[0]) + if not self.cfg.is_dummy_dataformat: + content = glob(os.path.join(outdir, '*')) + + if len(content) == 1 and os.path.isdir(content[0]): + for fp in glob(os.path.join(outdir, '**', '*')): + shutil.move(fp, outdir) + shutil.rmtree(content[0]) if not os.path.isdir(outdir): raise NotADirectoryError(outdir) -- GitLab