Skip to content
GitLab
Commit cfb10541 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Added Sentinel-2B compatibility. Added Sentinel-2B test. Added Sentinel-2B... 

Added Sentinel-2B compatibility. Added Sentinel-2B test. Added Sentinel-2B test data. Replaced Sentinel-2A new style test data.
parent f644bb2a
Pipeline #1676 passed with stage
in 11 minutes and 15 seconds
Hide whitespace changes
Inline Side-by-side
gms_preprocessing/algorithms/cloud_masking.py
View file @ cfb10541
......@@ -332,7 +332,7 @@ class FMASK_Runner_Sentinel2(_FMASK_Runner):
'Sentinel-2B': dict(opticalOLDStyle='%s*_B0[1-8].jp2 %s*_B8A.jp2 %s*_B09.jp2 %s*_B1[0-2].jp2'
% (oldStPref, oldStPref, oldStPref, oldStPref),
opticalNEWStyle='*_B0[1-8].jp2 *_B8A.jp2 *_B09.jp2 *_B1[0-2].jp2',
metaOLDStyle='%sS2A*.xml' % oldStPref,
metaOLDStyle='%sS2B*.xml' % oldStPref,
metaNEWStyle='*MTD_TL.xml'),
}[satellite]
......
gms_preprocessing/misc/definition_dicts.py
View file @ cfb10541
......@@ -177,6 +177,7 @@ def is_dataset_provided_as_fullScene(GMS_identifier):
'S2A10': False,
'S2A20': False,
'S2A60': False,
'S2B_full': False,
'S2B10': False,
'S2B20': False,
'S2B60': False, }
......
gms_preprocessing/misc/exception_handler.py
View file @ cfb10541
......@@ -145,7 +145,7 @@ class ExceptionHandler(object):
update statistics column in jobs table of postgreSQL database to 'started'"""
if isinstance(self.GMS_objs, collections.OrderedDict) and self.GMS_objs['proc_level'] is None:
if not self.GMS_objs['subsystem'] or self.GMS_objs['subsystem'] in ['VNIR1', 'S2A10']:
if not self.GMS_objs['subsystem'] or self.GMS_objs['subsystem'] in ['VNIR1', 'S2A10', 'S2B10']:
self.logger.debug("Setting job statistics array to 'STARTED'.")
# update statistics column ONLY in case of full cube or first subsystem
......@@ -162,7 +162,7 @@ class ExceptionHandler(object):
# NOTE: in case GMS_obj represents a subsystem and another one has already been marked as FAILED the
# failed_sceneids column and the statistics column is NOT updated once more
# check if another subsystem of the same scene ID already failed - don't increment the stats anymore
if not GMS_obj.subsystem or GMS_obj.subsystem in ['VNIR1', 'S2A10']:
if not GMS_obj.subsystem or GMS_obj.subsystem in ['VNIR1', 'S2A10', 'S2B10']:
another_ss_failed = False
if GMS_obj.subsystem:
......
gms_preprocessing/model/metadata.py
View file @ cfb10541
......@@ -127,8 +127,8 @@ class METADATA(object):
self.Read_SPOT_dimap2()
elif re.search("Terra", self.Satellite, re.I):
self.Read_ASTER_hdffile(self.Subsystem)
elif re.search("Sentinel-2A", self.Satellite, re.I):
self.Read_Sentinel2A_xmls()
elif re.search("Sentinel-2A", self.Satellite, re.I) or re.search("Sentinel-2B", self.Satellite, re.I):
self.Read_Sentinel2_xmls()
elif re.search("LANDSAT", self.Satellite, re.I):
self.Read_LANDSAT_mtltxt(self.LayerBandsAssignment)
elif re.search("RapidEye", self.Satellite, re.I):
......@@ -137,7 +137,7 @@ class METADATA(object):
self.Read_ALOS_summary()
self.Read_ALOS_LEADER() # for gains and offsets
else:
self.logger.critical("\tERROR: %s is not a supported sensor or sensorname is misspelled." % self.Satellite)
raise NotImplementedError("%s is not a supported sensor or sensorname is misspelled." % self.Satellite)
return self
......@@ -1215,17 +1215,18 @@ class METADATA(object):
self.Offsets = [float(lead_[4680 * 3 + 2711:4680 * 3 + 2718]), float(lead_[4680 * 3 + 2727:4680 * 3 + 2734]),
float(lead_[4680 * 3 + 2743:4680 * 3 + 2750]), float(lead_[4680 * 3 + 2759:4680 * 3 + 2766])]
def Read_Sentinel2A_xmls(self):
"""Read metadata from Sentinel-2A generic xml and granule xml"""
def Read_Sentinel2_xmls(self):
"""Read metadata from Sentinel-2 generic xml and granule xml"""
# query database to get entityid
assert self.SceneID and self.SceneID != -9999, "Read_Sentinel2A_xmls(): Missing scene ID. "
assert self.SceneID and self.SceneID != -9999, "Read_Sentinel2_xmls(): Missing scene ID. "
res = DB_T.get_info_from_postgreSQLdb(CFG.conn_database, 'scenes', ['entityid'], {'id': self.SceneID})
assert len(res) != 0, \
"Invalid SceneID given - no corresponding scene with the ID=%s found in database.\n" % self.SceneID
assert len(res) == 1, "Error in database. The sceneid %s exists more than once. \n" % self.SceneID
S2AgranuleID = res[0][0]
S2AB = 'S2A' if re.search("Sentinel-2A", self.Satellite, re.I) else 'S2B'
S2ABgranuleID = res[0][0]
#################
# get XML roots #
......@@ -1236,21 +1237,22 @@ class METADATA(object):
#####################################
# get xml_Scene_root (contains scene metadata)
glob_res = glob.glob(os.path.join(self.FolderOrArchive, 'S2A*.xml'))
glob_res = glob.glob(os.path.join(self.FolderOrArchive, '%s*.xml' % S2AB))
if not glob_res:
# new style packaging
glob_res = glob.glob(os.path.join(self.FolderOrArchive, 'MTD*.xml'))
assert len(glob_res) > 0, 'No S2A*.xml or MTD*.xml file can be found in %s/!' % self.FolderOrArchive
assert len(glob_res) > 0, 'No %s*.xml or MTD*.xml file can be found in %s/!' % (S2AB, self.FolderOrArchive)
self.Metafile = glob_res[0]
xml_Scene_root = ET.parse(glob_res[0]).getroot() # xml_parser from file
# get xml_GR_root (contains granule metadata)
glob_res = glob.glob(os.path.join(self.FolderOrArchive, 'GRANULE/' + S2AgranuleID + '/S2A*.xml'))
glob_res = glob.glob(os.path.join(self.FolderOrArchive, 'GRANULE/' + S2ABgranuleID + '/%s*.xml' % S2AB))
if not glob_res:
# new style packaging
glob_res = glob.glob(os.path.join(self.FolderOrArchive, 'GRANULE/' + S2AgranuleID + '/MTD*.xml'))
glob_res = glob.glob(os.path.join(self.FolderOrArchive, 'GRANULE/' + S2ABgranuleID + '/MTD*.xml'))
assert len(glob_res) > 0, \
'No /GRANULE/<S2AgranuleID>/S2A*.xml or MTD*.xml file can be found in %s/!' % self.FolderOrArchive
'No /GRANULE/<%sgranuleID>/%s*.xml or MTD*.xml file can be found in %s/!' \
% (S2AB, S2AB, self.FolderOrArchive)
self.Metafile = self.Metafile + ", " + glob_res[0]
xml_GR_root = ET.parse(glob_res).getroot() # xml_parser from file
......@@ -1261,14 +1263,16 @@ class METADATA(object):
# get xml_Scene_root and xml_GR_root (contain scene and granule metadata)
try:
# old style packaging
xml_SC_str_, self.Metafile = open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/S2A*.xml')
xml_GR_str_, Metafile_ = open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/GRANULE/' +
S2AgranuleID + '/S2A*.xml')
xml_SC_str_, self.Metafile = \
open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/%s*.xml' % S2AB)
xml_GR_str_, Metafile_ = \
open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/GRANULE/' +
S2ABgranuleID + '/%s*.xml' % S2AB)
except AssertionError:
# new style packaging
xml_SC_str_, self.Metafile = open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/MTD*.xml')
xml_GR_str_, Metafile_ = open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/GRANULE/' +
S2AgranuleID + '/MTD*.xml')
S2ABgranuleID + '/MTD*.xml')
xml_Scene_root = ET.fromstring(xml_SC_str_)
xml_GR_root = ET.fromstring(xml_GR_str_)
......@@ -1278,7 +1282,7 @@ class METADATA(object):
# EXTRACT METADATA FROM XML ROOTS #
###################################
# define Sentinel 2A metadata (hard coded)
# define Sentinel 2 metadata (hard coded)
self.Sensor = "MSI"
# extract metadata from xml_Scene_root #
......@@ -1286,7 +1290,7 @@ class METADATA(object):
# get the namespace within the xml_Scene_root
m = re.match('{(.*)\}', xml_Scene_root.tag) # extract namespace from "{https://....}Level-1C_Tile_ID"
assert m, 'XML Namespace could not be identified within metadata XML file.'
assert m, 'XML Namespace could not be identified within Sentinel-2 metadata XML file.'
namespace = m.group(1)
self.EntityID = \
......@@ -1343,7 +1347,7 @@ class METADATA(object):
-int(tmp[:-1]) if tmp[-1] == 'S' else self.CS_UTM_ZONE
# corner coords () #
subsytem_Res_dic = {"S2A10": 10, "S2A20": 20, "S2A60": 60}
subsytem_Res_dic = {"%s10" % S2AB: 10, "%s20" % S2AB: 20, "%s60" % S2AB: 60}
if self.CS_TYPE == 'UTM':
spatial_samplings = {float(size.get("resolution")): {key: int(size.find(key).text)
for key in ["NROWS", "NCOLS"]} for size in
......@@ -1948,21 +1952,27 @@ def get_special_values(GMS_identifier):
'SPOT1a': 0, 'SPOT2a': 0, 'SPOT3a': 0, 'SPOT4a': 0, 'SPOT5a': 0,
'SPOT1b': 0, 'SPOT2b': 0, 'SPOT3b': 0, 'SPOT4b': 0, 'SPOT5b': 0,
'RE5': 0,
'S2A10': 0, 'S2A20': 0, 'S2A60': 0}
'S2A10': 0, 'S2A20': 0, 'S2A60': 0,
'S2B10': 0, 'S2B20': 0, 'S2B60': 0,
}
dict_zero = {'AVNIR-2': None,
'AST_V1': 1, 'AST_V2': 1, 'AST_S': 1, 'AST_T': 1,
'TM4': None, 'TM5': None, 'TM7': None, 'LDCM': None,
'SPOT1a': None, 'SPOT2a': None, 'SPOT3a': None, 'SPOT4a': None, 'SPOT5a': None,
'SPOT1b': None, 'SPOT2b': None, 'SPOT3b': None, 'SPOT4b': None, 'SPOT5b': None,
'RE5': None,
'S2A10': None, 'S2A20': None, 'S2A60': None}
'S2A10': None, 'S2A20': None, 'S2A60': None,
'S2B10': None, 'S2B20': None, 'S2B60': None,
}
dict_saturated = {'AVNIR-2': None,
'AST_V1': 255, 'AST_V2': 255, 'AST_S': 255, 'AST_T': 65535,
'TM4': None, 'TM5': None, 'TM7': None, 'LDCM': 65535,
'SPOT1a': 255, 'SPOT2a': 255, 'SPOT3a': 255, 'SPOT4a': 255, 'SPOT5a': 255,
'SPOT1b': 255, 'SPOT2b': 255, 'SPOT3b': 255, 'SPOT4b': 255, 'SPOT5b': 255,
'RE5': None,
'S2A10': 65535, 'S2A20': 65535, 'S2A60': 65535}
'S2A10': 65535, 'S2A20': 65535, 'S2A60': 65535,
'S2B10': 65535, 'S2B20': 65535, 'S2B60': 65535
}
return {'fill': dict_fill[GMS_sensorcode],
'zero': dict_zero[GMS_sensorcode],
'saturated': dict_saturated[GMS_sensorcode]}
......
gms_preprocessing/options/config.py
View file @ cfb10541
......@@ -608,6 +608,11 @@ class JobConfig(object):
sub_ds = ds.copy()
sub_ds['subsystem'] = subsystem
data_list.append(sub_ds)
elif re.search("Sentinel-2B", ds['satellite'], re.I):
for subsystem in ['S2B10', 'S2B20', 'S2B60']:
sub_ds = ds.copy()
sub_ds['subsystem'] = subsystem
data_list.append(sub_ds)
elif re.search("Terra", ds['satellite'], re.I):
for subsystem in ['VNIR1', 'VNIR2', 'SWIR', 'TIR']:
sub_ds = ds.copy()
......
tests/test_gms_preprocessing.py
View file @ cfb10541
......@@ -240,6 +240,16 @@ class Test_Sentinel2A_MultiGranuleFormat(BaseTestCases.TestAll):
cls.create_job(26186272, job_config_kwargs)
class Test_Sentinel2B_SingleGranuleFormat(BaseTestCases.TestAll):
"""
Parametrized testclass. Tests the level-processes on a Sentinel-2B MSI scene (1 granule in archive: > 2017).
More information on the dataset will be output after the tests-classes are executed.
"""
@classmethod
def setUpClass(cls):
cls.create_job(26186937, job_config_kwargs)
class Test_MultipleDatasetsInOneJob(BaseTestCases.TestAll):
"""
Parametrized testclass. Tests the level-processes on a job containing a Landsat-5 (pre-collection data),
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment