Fixed exceptions during parsing of most recent Sentinel-2A metadata XMLs.

Former-commit-id: 405fe519

bin/run_gms.py

@@ -197,7 +197,7 @@ def get_gms_argparser():
                     'downloaded satellite image archives!',
         help="Run a GeoMultiSens preprocessing job for a given list of filenames of downloaded satellite "
              "image archives! (Sub-Parser).",
-        formatter_class = argparse.ArgumentDefaultsHelpFormatter)
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
     parser_constraints = subparsers.add_parser(
         'constraints', parents=[general_opts_parser],

gms_preprocessing/model/metadata.py

@@ -1218,41 +1218,54 @@ class METADATA(object):
     def Read_Sentinel2A_xmls(self):
         """Read metadata from Sentinel-2A generic xml and granule xml"""
 
-        # self.default_attr()
-
-        assert self.SceneID is not None and self.SceneID != -9999, "Read_Sentinel2A_xmls(): Missing scene ID. "
+        # query database to get entityid
+        assert self.SceneID and self.SceneID != -9999, "Read_Sentinel2A_xmls(): Missing scene ID. "
         res = DB_T.get_info_from_postgreSQLdb(CFG.conn_database, 'scenes', ['entityid'], {'id': self.SceneID})
         assert len(res) != 0, \
             "Invalid SceneID given - no corresponding scene with the ID=%s found in database.\n" % self.SceneID
         assert len(res) == 1, "Error in database. The sceneid %s exists more than once. \n" % self.SceneID
+
         S2AgranuleID = res[0][0]
 
+        #################
+        # get XML roots #
+        #################
+
         if os.path.isdir(self.FolderOrArchive):
+            # metadata has to be read from folder
+            #####################################
+
+            # get xml_Scene_root (contains scene metadata)
             glob_res = glob.glob(os.path.join(self.FolderOrArchive, 'S2A*.xml'))
             if not glob_res:
-                # new stype packaging
+                # new style packaging
                 glob_res = glob.glob(os.path.join(self.FolderOrArchive, 'MTD*.xml'))
             assert len(glob_res) > 0, 'No S2A*.xml or MTD*.xml file can be found in %s/!' % self.FolderOrArchive
             self.Metafile = glob_res[0]
             xml_Scene_root = ET.parse(glob_res[0]).getroot()  # xml_parser from file
 
+            # get xml_GR_root (contains granule metadata)
             glob_res = glob.glob(os.path.join(self.FolderOrArchive, 'GRANULE/' + S2AgranuleID + '/S2A*.xml'))
             if not glob_res:
-                # new stype packaging
+                # new style packaging
                 glob_res = glob.glob(os.path.join(self.FolderOrArchive, 'GRANULE/' + S2AgranuleID + '/MTD*.xml'))
             assert len(glob_res) > 0, \
                 'No  /GRANULE/<S2AgranuleID>/S2A*.xml or MTD*.xml file can be found in %s/!' % self.FolderOrArchive
             self.Metafile = self.Metafile + ", " + glob_res[0]
             xml_GR_root = ET.parse(glob_res).getroot()  # xml_parser from file
 
-        else:  # archive
+        else:
+            # metadata has to be read from within archive
+            #############################################
+
+            # get xml_Scene_root and xml_GR_root (contain scene and granule metadata)
             try:
                 # old style packaging
                 xml_SC_str_, self.Metafile = open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/S2A*.xml')
                 xml_GR_str_, Metafile_ = open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/GRANULE/' +
                                                                            S2AgranuleID + '/S2A*.xml')
             except AssertionError:
-                # new stype packaging
+                # new style packaging
                 xml_SC_str_, self.Metafile = open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/MTD*.xml')
                 xml_GR_str_, Metafile_ = open_specific_file_within_archive(self.FolderOrArchive, '*.SAFE/GRANULE/' +
                                                                            S2AgranuleID + '/MTD*.xml')
@@ -1261,13 +1274,23 @@ class METADATA(object):
             xml_GR_root = ET.fromstring(xml_GR_str_)
             self.Metafile = self.Metafile + ", " + Metafile_
 
+        ###################################
+        # EXTRACT METADATA FROM XML ROOTS #
+        ###################################
+
         # define Sentinel 2A metadata (hard coded)
         self.Sensor = "MSI"
 
-        # extract metadata from xml_Scene_root
-        namespace = "https://psd-13.sentinel2.eo.esa.int/PSD/User_Product_Level-1C.xsd"
-        self.EntityID = xml_Scene_root.find(".//Datatake").attrib[
-            'datatakeIdentifier']  # FIXME tileID (Granule) oder scene ID???
+        # extract metadata from xml_Scene_root #
+        ########################################
+
+        # get the namespace within the xml_Scene_root
+        m = re.match('{(.*)\}', xml_Scene_root.tag)  # extract namespace from "{https://....}Level-1C_Tile_ID"
+        assert m, 'XML Namespace could not be identified within metadata XML file.'
+        namespace = m.group(1)
+
+        self.EntityID = \
+            xml_Scene_root.find(".//Datatake").attrib['datatakeIdentifier']  # FIXME tileID (Granule) oder scene ID???
         self.Satellite = xml_Scene_root.find(".//SPACECRAFT_NAME").text
 
         # AcqDate, AcqTime
@@ -1292,8 +1315,13 @@ class METADATA(object):
 
         self.ProcLCode = xml_Scene_root.find(".//PROCESSING_LEVEL").text
 
-        # extract metadata from xml_GR_root
-        namespace = "https://psd-12.sentinel2.eo.esa.int/PSD/S2_PDI_Level-1C_Tile_Metadata.xsd"
+        # extract metadata from xml_GR_root #
+        #####################################
+
+        # get the namespace within the xml_GR_root
+        m = re.match('{(.*)\}', xml_GR_root.tag)  # extract namespace from "{https://....}Level-1C_Tile_ID"
+        assert m, 'XML Namespace could not be identified within metadata XML file.'
+        namespace = m.group(1)  # e.g., "https://psd-12.sentinel2.eo.esa.int/PSD/S2_PDI_Level-1C_Tile_Metadata.xsd"
 
         # set self.AcqDateTime as well as self.AcqDate and self.AcqTime
         self.AcqDateTime = iso8601.parse_date(xml_GR_root.find(".//SENSING_TIME").text)
@@ -1385,14 +1413,23 @@ class METADATA(object):
 
         # Quality flags # FIXME does not work (at least with old data)
         Quality_temp = (xml_Scene_root.find(".//Technical_Quality_Assessment"))
-        Quality_temp.find("./DEGRADED_ANC_DATA_PERCENTAGE").text
         self.Quality.append(["DEGRADED_ANC_DATA_PERCENTAGE", Quality_temp.find("./DEGRADED_ANC_DATA_PERCENTAGE").text])
         self.Quality.append(["DEGRADED_MSI_DATA_PERCENTAGE", Quality_temp.find("./DEGRADED_MSI_DATA_PERCENTAGE").text])
         Quality_temp2 = xml_Scene_root.find(".//Quality_Inspections")
         quality_flags = ["SENSOR_QUALITY_FLAG", "GEOMETRIC_QUALITY_FLAG", "GENERAL_QUALITY_FLAG",
                          "FORMAT_CORRECTNESS_FLAG", "RADIOMETRIC_QUALITY_FLAG"]
-        for ql in quality_flags:
-            self.Quality.append([ql, Quality_temp2.find("./" + ql).text])
+
+        try:
+            # layout example: <SENSOR_QUALITY_FLAG>PASSED</SENSOR_QUALITY_FLAG>
+            for ql in quality_flags:
+                self.Quality.append([ql, Quality_temp2.find("./" + ql).text])
+        except AttributeError:
+            # since ~11/2017 the quality checks layout in the XML has changed:
+            # layout example: <quality_check checkType="SENSOR_QUALITY">PASSED</quality_check>
+            elements = Quality_temp2.findall('quality_check')
+            checkTypeValDict = {ele.attrib['checkType']: ele.text for ele in elements}
+            for ql in quality_flags:
+                self.Quality.append([ql, checkTypeValDict[ql.split('_FLAG')[0]]])
 
         self.FOV = get_FieldOfView(self.GMS_identifier)
         self.orbitParams = get_orbit_params(self.GMS_identifier)

tests/test_gms_preprocessing.py

@@ -286,10 +286,11 @@ summary_testResults, summary_errors, summary_failures, summary_skipped, jobstatu
 class Test_in_normal_mode(unittest.TestCase):
     def setUp(self):
         # self.job_id = 26186740  # Testjob Landsat-8
-        self.job_id = 26186906  # Bug Input Validator
+        # self.job_id = 26186906  # Bug Input Validator
+        self.job_id = 26186925  # 1 Sentinel-2A, Bug NoneType' object has no attribute 'find'
 
         self.PC = process_controller(self.job_id, **dict(is_test=False, parallelization_level='scenes', db_host='geoms',
-                                                         delete_old_output=True,))
+                                                         delete_old_output=True, disable_exception_handler=True))
 
     def test(self):
         self.PC.run_all_processors()