Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
geomultisens
gms_preprocessing
Commits
449af678
Commit
449af678
authored
Dec 04, 2017
by
Daniel Scheffler
Browse files
Fixed exceptions during parsing of most recent Sentinel-2A metadata XMLs.
Former-commit-id:
405fe519
parent
d9cc25d9
Changes
3
Hide whitespace changes
Inline
Side-by-side
bin/run_gms.py
View file @
449af678
...
...
@@ -197,7 +197,7 @@ def get_gms_argparser():
'downloaded satellite image archives!'
,
help
=
"Run a GeoMultiSens preprocessing job for a given list of filenames of downloaded satellite "
"image archives! (Sub-Parser)."
,
formatter_class
=
argparse
.
ArgumentDefaultsHelpFormatter
)
formatter_class
=
argparse
.
ArgumentDefaultsHelpFormatter
)
parser_constraints
=
subparsers
.
add_parser
(
'constraints'
,
parents
=
[
general_opts_parser
],
...
...
gms_preprocessing/model/metadata.py
View file @
449af678
...
...
@@ -1218,41 +1218,54 @@ class METADATA(object):
def
Read_Sentinel2A_xmls
(
self
):
"""Read metadata from Sentinel-2A generic xml and granule xml"""
# self.default_attr()
assert
self
.
SceneID
is
not
None
and
self
.
SceneID
!=
-
9999
,
"Read_Sentinel2A_xmls(): Missing scene ID. "
# query database to get entityid
assert
self
.
SceneID
and
self
.
SceneID
!=
-
9999
,
"Read_Sentinel2A_xmls(): Missing scene ID. "
res
=
DB_T
.
get_info_from_postgreSQLdb
(
CFG
.
conn_database
,
'scenes'
,
[
'entityid'
],
{
'id'
:
self
.
SceneID
})
assert
len
(
res
)
!=
0
,
\
"Invalid SceneID given - no corresponding scene with the ID=%s found in database.
\n
"
%
self
.
SceneID
assert
len
(
res
)
==
1
,
"Error in database. The sceneid %s exists more than once.
\n
"
%
self
.
SceneID
S2AgranuleID
=
res
[
0
][
0
]
#################
# get XML roots #
#################
if
os
.
path
.
isdir
(
self
.
FolderOrArchive
):
# metadata has to be read from folder
#####################################
# get xml_Scene_root (contains scene metadata)
glob_res
=
glob
.
glob
(
os
.
path
.
join
(
self
.
FolderOrArchive
,
'S2A*.xml'
))
if
not
glob_res
:
# new sty
p
e packaging
# new sty
l
e packaging
glob_res
=
glob
.
glob
(
os
.
path
.
join
(
self
.
FolderOrArchive
,
'MTD*.xml'
))
assert
len
(
glob_res
)
>
0
,
'No S2A*.xml or MTD*.xml file can be found in %s/!'
%
self
.
FolderOrArchive
self
.
Metafile
=
glob_res
[
0
]
xml_Scene_root
=
ET
.
parse
(
glob_res
[
0
]).
getroot
()
# xml_parser from file
# get xml_GR_root (contains granule metadata)
glob_res
=
glob
.
glob
(
os
.
path
.
join
(
self
.
FolderOrArchive
,
'GRANULE/'
+
S2AgranuleID
+
'/S2A*.xml'
))
if
not
glob_res
:
# new sty
p
e packaging
# new sty
l
e packaging
glob_res
=
glob
.
glob
(
os
.
path
.
join
(
self
.
FolderOrArchive
,
'GRANULE/'
+
S2AgranuleID
+
'/MTD*.xml'
))
assert
len
(
glob_res
)
>
0
,
\
'No /GRANULE/<S2AgranuleID>/S2A*.xml or MTD*.xml file can be found in %s/!'
%
self
.
FolderOrArchive
self
.
Metafile
=
self
.
Metafile
+
", "
+
glob_res
[
0
]
xml_GR_root
=
ET
.
parse
(
glob_res
).
getroot
()
# xml_parser from file
else
:
# archive
else
:
# metadata has to be read from within archive
#############################################
# get xml_Scene_root and xml_GR_root (contain scene and granule metadata)
try
:
# old style packaging
xml_SC_str_
,
self
.
Metafile
=
open_specific_file_within_archive
(
self
.
FolderOrArchive
,
'*.SAFE/S2A*.xml'
)
xml_GR_str_
,
Metafile_
=
open_specific_file_within_archive
(
self
.
FolderOrArchive
,
'*.SAFE/GRANULE/'
+
S2AgranuleID
+
'/S2A*.xml'
)
except
AssertionError
:
# new sty
p
e packaging
# new sty
l
e packaging
xml_SC_str_
,
self
.
Metafile
=
open_specific_file_within_archive
(
self
.
FolderOrArchive
,
'*.SAFE/MTD*.xml'
)
xml_GR_str_
,
Metafile_
=
open_specific_file_within_archive
(
self
.
FolderOrArchive
,
'*.SAFE/GRANULE/'
+
S2AgranuleID
+
'/MTD*.xml'
)
...
...
@@ -1261,13 +1274,23 @@ class METADATA(object):
xml_GR_root
=
ET
.
fromstring
(
xml_GR_str_
)
self
.
Metafile
=
self
.
Metafile
+
", "
+
Metafile_
###################################
# EXTRACT METADATA FROM XML ROOTS #
###################################
# define Sentinel 2A metadata (hard coded)
self
.
Sensor
=
"MSI"
# extract metadata from xml_Scene_root
namespace
=
"https://psd-13.sentinel2.eo.esa.int/PSD/User_Product_Level-1C.xsd"
self
.
EntityID
=
xml_Scene_root
.
find
(
".//Datatake"
).
attrib
[
'datatakeIdentifier'
]
# FIXME tileID (Granule) oder scene ID???
# extract metadata from xml_Scene_root #
########################################
# get the namespace within the xml_Scene_root
m
=
re
.
match
(
'{(.*)\}'
,
xml_Scene_root
.
tag
)
# extract namespace from "{https://....}Level-1C_Tile_ID"
assert
m
,
'XML Namespace could not be identified within metadata XML file.'
namespace
=
m
.
group
(
1
)
self
.
EntityID
=
\
xml_Scene_root
.
find
(
".//Datatake"
).
attrib
[
'datatakeIdentifier'
]
# FIXME tileID (Granule) oder scene ID???
self
.
Satellite
=
xml_Scene_root
.
find
(
".//SPACECRAFT_NAME"
).
text
# AcqDate, AcqTime
...
...
@@ -1292,8 +1315,13 @@ class METADATA(object):
self
.
ProcLCode
=
xml_Scene_root
.
find
(
".//PROCESSING_LEVEL"
).
text
# extract metadata from xml_GR_root
namespace
=
"https://psd-12.sentinel2.eo.esa.int/PSD/S2_PDI_Level-1C_Tile_Metadata.xsd"
# extract metadata from xml_GR_root #
#####################################
# get the namespace within the xml_GR_root
m
=
re
.
match
(
'{(.*)\}'
,
xml_GR_root
.
tag
)
# extract namespace from "{https://....}Level-1C_Tile_ID"
assert
m
,
'XML Namespace could not be identified within metadata XML file.'
namespace
=
m
.
group
(
1
)
# e.g., "https://psd-12.sentinel2.eo.esa.int/PSD/S2_PDI_Level-1C_Tile_Metadata.xsd"
# set self.AcqDateTime as well as self.AcqDate and self.AcqTime
self
.
AcqDateTime
=
iso8601
.
parse_date
(
xml_GR_root
.
find
(
".//SENSING_TIME"
).
text
)
...
...
@@ -1385,14 +1413,23 @@ class METADATA(object):
# Quality flags # FIXME does not work (at least with old data)
Quality_temp
=
(
xml_Scene_root
.
find
(
".//Technical_Quality_Assessment"
))
Quality_temp
.
find
(
"./DEGRADED_ANC_DATA_PERCENTAGE"
).
text
self
.
Quality
.
append
([
"DEGRADED_ANC_DATA_PERCENTAGE"
,
Quality_temp
.
find
(
"./DEGRADED_ANC_DATA_PERCENTAGE"
).
text
])
self
.
Quality
.
append
([
"DEGRADED_MSI_DATA_PERCENTAGE"
,
Quality_temp
.
find
(
"./DEGRADED_MSI_DATA_PERCENTAGE"
).
text
])
Quality_temp2
=
xml_Scene_root
.
find
(
".//Quality_Inspections"
)
quality_flags
=
[
"SENSOR_QUALITY_FLAG"
,
"GEOMETRIC_QUALITY_FLAG"
,
"GENERAL_QUALITY_FLAG"
,
"FORMAT_CORRECTNESS_FLAG"
,
"RADIOMETRIC_QUALITY_FLAG"
]
for
ql
in
quality_flags
:
self
.
Quality
.
append
([
ql
,
Quality_temp2
.
find
(
"./"
+
ql
).
text
])
try
:
# layout example: <SENSOR_QUALITY_FLAG>PASSED</SENSOR_QUALITY_FLAG>
for
ql
in
quality_flags
:
self
.
Quality
.
append
([
ql
,
Quality_temp2
.
find
(
"./"
+
ql
).
text
])
except
AttributeError
:
# since ~11/2017 the quality checks layout in the XML has changed:
# layout example: <quality_check checkType="SENSOR_QUALITY">PASSED</quality_check>
elements
=
Quality_temp2
.
findall
(
'quality_check'
)
checkTypeValDict
=
{
ele
.
attrib
[
'checkType'
]:
ele
.
text
for
ele
in
elements
}
for
ql
in
quality_flags
:
self
.
Quality
.
append
([
ql
,
checkTypeValDict
[
ql
.
split
(
'_FLAG'
)[
0
]]])
self
.
FOV
=
get_FieldOfView
(
self
.
GMS_identifier
)
self
.
orbitParams
=
get_orbit_params
(
self
.
GMS_identifier
)
...
...
tests/test_gms_preprocessing.py
View file @
449af678
...
...
@@ -286,10 +286,11 @@ summary_testResults, summary_errors, summary_failures, summary_skipped, jobstatu
class
Test_in_normal_mode
(
unittest
.
TestCase
):
def
setUp
(
self
):
# self.job_id = 26186740 # Testjob Landsat-8
self
.
job_id
=
26186906
# Bug Input Validator
# self.job_id = 26186906 # Bug Input Validator
self
.
job_id
=
26186925
# 1 Sentinel-2A, Bug NoneType' object has no attribute 'find'
self
.
PC
=
process_controller
(
self
.
job_id
,
**
dict
(
is_test
=
False
,
parallelization_level
=
'scenes'
,
db_host
=
'geoms'
,
delete_old_output
=
True
,))
delete_old_output
=
True
,
disable_exception_handler
=
True
))
def
test
(
self
):
self
.
PC
.
run_all_processors
()
...
...
Daniel Scheffler
@danschef
mentioned in commit
b801513f
·
Mar 31, 2020
mentioned in commit
b801513f
mentioned in commit b801513fc0a094de3b3495bf5ec597ddd616546d
Toggle commit list
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment