Added Sentinel-2B compatibility. Added Sentinel-2B test. Added Sentinel-2B...

Added Sentinel-2B compatibility. Added Sentinel-2B test. Added Sentinel-2B test data. Replaced Sentinel-2A new style test data.


Former-commit-id: cfb10541
Former-commit-id: b4b6ea2e

gms_preprocessing/algorithms/cloud_masking.py

@@ -332,7 +332,7 @@ class FMASK_Runner_Sentinel2(_FMASK_Runner):
             'Sentinel-2B': dict(opticalOLDStyle='%s*_B0[1-8].jp2 %s*_B8A.jp2 %s*_B09.jp2 %s*_B1[0-2].jp2'
                                                 % (oldStPref, oldStPref, oldStPref, oldStPref),
                                 opticalNEWStyle='*_B0[1-8].jp2 *_B8A.jp2 *_B09.jp2 *_B1[0-2].jp2',
-                                metaOLDStyle='%sS2A*.xml' % oldStPref,
+                                metaOLDStyle='%sS2B*.xml' % oldStPref,
                                 metaNEWStyle='*MTD_TL.xml'),
         }[satellite]
 

gms_preprocessing/misc/definition_dicts.py

@@ -177,6 +177,7 @@ def is_dataset_provided_as_fullScene(GMS_identifier):
         'S2A10': False,
         'S2A20': False,
         'S2A60': False,
+        'S2B_full': False,
         'S2B10': False,
         'S2B20': False,
         'S2B60': False, }

gms_preprocessing/misc/exception_handler.py

@@ -145,7 +145,7 @@ class ExceptionHandler(object):
         update statistics column in jobs table of postgreSQL database to 'started'"""
         if isinstance(self.GMS_objs, collections.OrderedDict) and self.GMS_objs['proc_level'] is None:
 
-            if not self.GMS_objs['subsystem'] or self.GMS_objs['subsystem'] in ['VNIR1', 'S2A10']:
+            if not self.GMS_objs['subsystem'] or self.GMS_objs['subsystem'] in ['VNIR1', 'S2A10', 'S2B10']:
                 self.logger.debug("Setting job statistics array to 'STARTED'.")
 
                 # update statistics column ONLY in case of full cube or first subsystem
@@ -162,7 +162,7 @@ class ExceptionHandler(object):
         # NOTE: in case GMS_obj represents a subsystem and another one has already been marked as FAILED the
         # failed_sceneids column and the statistics column is NOT updated once more
         # check if another subsystem of the same scene ID already failed - don't increment the stats anymore
-        if not GMS_obj.subsystem or GMS_obj.subsystem in ['VNIR1', 'S2A10']:
+        if not GMS_obj.subsystem or GMS_obj.subsystem in ['VNIR1', 'S2A10', 'S2B10']:
             another_ss_failed = False
 
             if GMS_obj.subsystem:

gms_preprocessing/model/metadata.py

@@ -127,8 +127,8 @@ class METADATA(object):
             self.Read_SPOT_dimap2()
         elif re.search("Terra", self.Satellite, re.I):
             self.Read_ASTER_hdffile(self.Subsystem)
-        elif re.search("Sentinel-2A", self.Satellite, re.I):
-            self.Read_Sentinel2A_xmls()
+        elif re.search("Sentinel-2A", self.Satellite, re.I) or re.search("Sentinel-2B", self.Satellite, re.I):
+            self.Read_Sentinel2_xmls()
         elif re.search("LANDSAT", self.Satellite, re.I):
             self.Read_LANDSAT_mtltxt(self.LayerBandsAssignment)
         elif re.search("RapidEye", self.Satellite, re.I):
@@ -137,7 +137,7 @@ class METADATA(object):
             self.Read_ALOS_summary()
             self.Read_ALOS_LEADER()  # for gains and offsets
         else:
-            self.logger.critical("\tERROR: %s is not a supported sensor or sensorname is misspelled." % self.Satellite)
+            raise NotImplementedError("%s is not a supported sensor or sensorname is misspelled." % self.Satellite)
 
         return self
 
@@ -1215,17 +1215,18 @@ class METADATA(object):
         self.Offsets = [float(lead_[4680 * 3 + 2711:4680 * 3 + 2718]), float(lead_[4680 * 3 + 2727:4680 * 3 + 2734]),
                         float(lead_[4680 * 3 + 2743:4680 * 3 + 2750]), float(lead_[4680 * 3 + 2759:4680 * 3 + 2766])]
 
-    def Read_Sentinel2A_xmls(self):
-        """Read metadata from Sentinel-2A generic xml and granule xml"""
+    def Read_Sentinel2_xmls(self):
+        """Read metadata from Sentinel-2 generic xml and granule xml"""
 
         # query database to get entityid
-        assert self.SceneID and self.SceneID != -9999, "Read_Sentinel2A_xmls(): Missing scene ID. "
+        assert self.SceneID and self.SceneID != -9999, "Read_Sentinel2_xmls(): Missing scene ID. "
         res = DB_T.get_info_from_postgreSQLdb(CFG.conn_database, 'scenes', ['entityid'], {'id': self.SceneID})
         assert len(res) != 0, \
             "Invalid SceneID given - no corresponding scene with the ID=%s found in database.\n" % self.SceneID
         assert len(res) == 1, "Error in database. The sceneid %s exists more than once. \n" % self.SceneID
 
-        S2AgranuleID = res[0][0]
+        S2AB = 'S2A' if re.search("Sentinel-2A", self.Satellite, re.I) else 'S2B'
+        S2ABgranuleID = res[0][0]
 
         #################
         # get XML roots #
@@ -1236,21 +1237,22 @@ class METADATA(object):
             #####################################
 
             # get xml_Scene_root (contains scene metadata)
-            glob_res = glob.glob(os.path.join(self.FolderOrArchive, 'S2A*.xml'))
+            glob_res = glob.glob(os.path.join(self.FolderOrArchive, '%s*.xml' % S2AB))
             if not glob_res:
                 # new style packaging
                 glob_res = glob.glob(os.path.join(self.FolderOrArchive, 'MTD*.xml'))
-            assert len(glob_res) > 0, 'No S2A*.xml or MTD*.xml file can be found in %s/!' % self.FolderOrArchive
+            assert len(glob_res) > 0, 'No %s*.xml or MTD*.xml file can be found in %s/!' % (S2AB, self.FolderOrArchive)
             self.Metafile = glob_res[0]
             xml_Scene_root = ET.parse(glob_res[0]).getroot()  # xml_parser from file
 
             # get xml_GR_root (contains granule metadata)
-            glob_res = glob.glob(os.path.join(self.FolderOrArchive, 'GRANULE/' + S2AgranuleID + '/S2A*.xml'))
+            glob_res = glob.glob(os.path.join(self.FolderOrArchive, 'GRANULE/' + S2ABgranuleID + '/%s*.xml' % S2AB))
             if not glob_res:
                 # new style packaging
-                glob_res = glob.glob(os.path.join(self.FolderOrArchive, 'GRANULE/' + S2AgranuleID + '/MTD*.xml'))
+                glob_res = glob.glob(os.path.join(self.FolderOrArchive, 'GRANULE/' + S2ABgranuleID + '/MTD*.xml'))
             assert len(glob_res) > 0, \
-                'No  /GRANULE/<S2AgranuleID>/S2A*.xml or MTD*.xml file can be found in %s/!' % self.FolderOrArchive
+                'No  /GRANULE/<%sgranuleID>/%s*.xml or MTD*.xml file can be found in %s/!' \
+                % (S2AB, S2AB, self.FolderOrArchive)
             self.Metafile = self.Metafile + ", " + glob_res[0]
             xml_GR_root = ET.parse(glob_res).getroot()  # xml_parser from file
 
@@ -1261,14 +1263,16 @@ class METADATA(object):
             # get xml_Scene_root and xml_GR_root (contain scene and granule metadata)
             try:
                 # old style packaging
-                xml_SC_str_, self.Metafile = open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/S2A*.xml')
-                xml_GR_str_, Metafile_ = open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/GRANULE/' +
-                                                                           S2AgranuleID + '/S2A*.xml')
+                xml_SC_str_, self.Metafile = \
+                    open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/%s*.xml' % S2AB)
+                xml_GR_str_, Metafile_ = \
+                    open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/GRANULE/' +
+                                                      S2ABgranuleID + '/%s*.xml' % S2AB)
             except AssertionError:
                 # new style packaging
                 xml_SC_str_, self.Metafile = open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/MTD*.xml')
                 xml_GR_str_, Metafile_ = open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/GRANULE/' +
-                                                                           S2AgranuleID + '/MTD*.xml')
+                                                                           S2ABgranuleID + '/MTD*.xml')
 
             xml_Scene_root = ET.fromstring(xml_SC_str_)
             xml_GR_root = ET.fromstring(xml_GR_str_)
@@ -1278,7 +1282,7 @@ class METADATA(object):
         # EXTRACT METADATA FROM XML ROOTS #
         ###################################
 
-        # define Sentinel 2A metadata (hard coded)
+        # define Sentinel 2 metadata (hard coded)
         self.Sensor = "MSI"
 
         # extract metadata from xml_Scene_root #
@@ -1286,7 +1290,7 @@ class METADATA(object):
 
         # get the namespace within the xml_Scene_root
         m = re.match('{(.*)\}', xml_Scene_root.tag)  # extract namespace from "{https://....}Level-1C_Tile_ID"
-        assert m, 'XML Namespace could not be identified within metadata XML file.'
+        assert m, 'XML Namespace could not be identified within Sentinel-2 metadata XML file.'
         namespace = m.group(1)
 
         self.EntityID = \
@@ -1343,7 +1347,7 @@ class METADATA(object):
                 -int(tmp[:-1]) if tmp[-1] == 'S' else self.CS_UTM_ZONE
 
         # corner coords () #
-        subsytem_Res_dic = {"S2A10": 10, "S2A20": 20, "S2A60": 60}
+        subsytem_Res_dic = {"%s10" % S2AB: 10, "%s20" % S2AB: 20, "%s60" % S2AB: 60}
         if self.CS_TYPE == 'UTM':
             spatial_samplings = {float(size.get("resolution")): {key: int(size.find(key).text)
                                                                  for key in ["NROWS", "NCOLS"]} for size in
@@ -1948,21 +1952,27 @@ def get_special_values(GMS_identifier):
                  'SPOT1a': 0, 'SPOT2a': 0, 'SPOT3a': 0, 'SPOT4a': 0, 'SPOT5a': 0,
                  'SPOT1b': 0, 'SPOT2b': 0, 'SPOT3b': 0, 'SPOT4b': 0, 'SPOT5b': 0,
                  'RE5': 0,
-                 'S2A10': 0, 'S2A20': 0, 'S2A60': 0}
+                 'S2A10': 0, 'S2A20': 0, 'S2A60': 0,
+                 'S2B10': 0, 'S2B20': 0, 'S2B60': 0,
+                 }
     dict_zero = {'AVNIR-2': None,
                  'AST_V1': 1, 'AST_V2': 1, 'AST_S': 1, 'AST_T': 1,
                  'TM4': None, 'TM5': None, 'TM7': None, 'LDCM': None,
                  'SPOT1a': None, 'SPOT2a': None, 'SPOT3a': None, 'SPOT4a': None, 'SPOT5a': None,
                  'SPOT1b': None, 'SPOT2b': None, 'SPOT3b': None, 'SPOT4b': None, 'SPOT5b': None,
                  'RE5': None,
-                 'S2A10': None, 'S2A20': None, 'S2A60': None}
+                 'S2A10': None, 'S2A20': None, 'S2A60': None,
+                 'S2B10': None, 'S2B20': None, 'S2B60': None,
+                 }
     dict_saturated = {'AVNIR-2': None,
                       'AST_V1': 255, 'AST_V2': 255, 'AST_S': 255, 'AST_T': 65535,
                       'TM4': None, 'TM5': None, 'TM7': None, 'LDCM': 65535,
                       'SPOT1a': 255, 'SPOT2a': 255, 'SPOT3a': 255, 'SPOT4a': 255, 'SPOT5a': 255,
                       'SPOT1b': 255, 'SPOT2b': 255, 'SPOT3b': 255, 'SPOT4b': 255, 'SPOT5b': 255,
                       'RE5': None,
-                      'S2A10': 65535, 'S2A20': 65535, 'S2A60': 65535}
+                      'S2A10': 65535, 'S2A20': 65535, 'S2A60': 65535,
+                      'S2B10': 65535, 'S2B20': 65535, 'S2B60': 65535
+                      }
     return {'fill': dict_fill[GMS_sensorcode],
             'zero': dict_zero[GMS_sensorcode],
             'saturated': dict_saturated[GMS_sensorcode]}

gms_preprocessing/options/config.py

@@ -608,6 +608,11 @@ class JobConfig(object):
                             sub_ds = ds.copy()
                             sub_ds['subsystem'] = subsystem
                             data_list.append(sub_ds)
+                    elif re.search("Sentinel-2B", ds['satellite'], re.I):
+                        for subsystem in ['S2B10', 'S2B20', 'S2B60']:
+                            sub_ds = ds.copy()
+                            sub_ds['subsystem'] = subsystem
+                            data_list.append(sub_ds)
                     elif re.search("Terra", ds['satellite'], re.I):
                         for subsystem in ['VNIR1', 'VNIR2', 'SWIR', 'TIR']:
                             sub_ds = ds.copy()

tests/data/archive_data/Sentinel-2A/MSI/S2A_MSIL1C_20161231T103432_N0204_R108_T32TLT_20161231T103428.zip.REMOVED.git-id

-62339a73197f189e00bec8df51e56c1901633d61
\ No newline at end of file

tests/test_gms_preprocessing.py

@@ -240,6 +240,16 @@ class Test_Sentinel2A_MultiGranuleFormat(BaseTestCases.TestAll):
         cls.create_job(26186272, job_config_kwargs)
 
 
+class Test_Sentinel2B_SingleGranuleFormat(BaseTestCases.TestAll):
+    """
+    Parametrized testclass. Tests the level-processes on a Sentinel-2B MSI scene (1 granule in archive: > 2017).
+    More information on the dataset will be output after the tests-classes are executed.
+    """
+    @classmethod
+    def setUpClass(cls):
+        cls.create_job(26186937, job_config_kwargs)
+
+
 class Test_MultipleDatasetsInOneJob(BaseTestCases.TestAll):
     """
     Parametrized testclass. Tests the level-processes on a job containing a Landsat-5 (pre-collection data),