Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
geomultisens
gms_preprocessing
Commits
a58ecd3e
Commit
a58ecd3e
authored
Dec 05, 2017
by
Daniel Scheffler
Browse files
Merge branch 'feature/add_sentinel2b_compatibility'
Former-commit-id:
8c2702d2
Former-commit-id:
6769ab20
parents
613ed806
cf78945a
Pipeline
#2503
canceled with stage
Changes
9
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
gms_preprocessing/algorithms/cloud_masking.py
View file @
a58ecd3e
...
...
@@ -332,7 +332,7 @@ class FMASK_Runner_Sentinel2(_FMASK_Runner):
'Sentinel-2B'
:
dict
(
opticalOLDStyle
=
'%s*_B0[1-8].jp2 %s*_B8A.jp2 %s*_B09.jp2 %s*_B1[0-2].jp2'
%
(
oldStPref
,
oldStPref
,
oldStPref
,
oldStPref
),
opticalNEWStyle
=
'*_B0[1-8].jp2 *_B8A.jp2 *_B09.jp2 *_B1[0-2].jp2'
,
metaOLDStyle
=
'%sS2
A
*.xml'
%
oldStPref
,
metaOLDStyle
=
'%sS2
B
*.xml'
%
oldStPref
,
metaNEWStyle
=
'*MTD_TL.xml'
),
}[
satellite
]
...
...
gms_preprocessing/misc/definition_dicts.py
View file @
a58ecd3e
...
...
@@ -177,6 +177,7 @@ def is_dataset_provided_as_fullScene(GMS_identifier):
'S2A10'
:
False
,
'S2A20'
:
False
,
'S2A60'
:
False
,
'S2B_full'
:
False
,
'S2B10'
:
False
,
'S2B20'
:
False
,
'S2B60'
:
False
,
}
...
...
gms_preprocessing/misc/exception_handler.py
View file @
a58ecd3e
...
...
@@ -145,7 +145,7 @@ class ExceptionHandler(object):
update statistics column in jobs table of postgreSQL database to 'started'"""
if
isinstance
(
self
.
GMS_objs
,
collections
.
OrderedDict
)
and
self
.
GMS_objs
[
'proc_level'
]
is
None
:
if
not
self
.
GMS_objs
[
'subsystem'
]
or
self
.
GMS_objs
[
'subsystem'
]
in
[
'VNIR1'
,
'S2A10'
]:
if
not
self
.
GMS_objs
[
'subsystem'
]
or
self
.
GMS_objs
[
'subsystem'
]
in
[
'VNIR1'
,
'S2A10'
,
'S2B10'
]:
self
.
logger
.
debug
(
"Setting job statistics array to 'STARTED'."
)
# update statistics column ONLY in case of full cube or first subsystem
...
...
@@ -162,7 +162,7 @@ class ExceptionHandler(object):
# NOTE: in case GMS_obj represents a subsystem and another one has already been marked as FAILED the
# failed_sceneids column and the statistics column is NOT updated once more
# check if another subsystem of the same scene ID already failed - don't increment the stats anymore
if
not
GMS_obj
.
subsystem
or
GMS_obj
.
subsystem
in
[
'VNIR1'
,
'S2A10'
]:
if
not
GMS_obj
.
subsystem
or
GMS_obj
.
subsystem
in
[
'VNIR1'
,
'S2A10'
,
'S2B10'
]:
another_ss_failed
=
False
if
GMS_obj
.
subsystem
:
...
...
gms_preprocessing/model/metadata.py
View file @
a58ecd3e
...
...
@@ -127,8 +127,8 @@ class METADATA(object):
self
.
Read_SPOT_dimap2
()
elif
re
.
search
(
"Terra"
,
self
.
Satellite
,
re
.
I
):
self
.
Read_ASTER_hdffile
(
self
.
Subsystem
)
elif
re
.
search
(
"Sentinel-2A"
,
self
.
Satellite
,
re
.
I
):
self
.
Read_Sentinel2
A
_xmls
()
elif
re
.
search
(
"Sentinel-2A"
,
self
.
Satellite
,
re
.
I
)
or
re
.
search
(
"Sentinel-2B"
,
self
.
Satellite
,
re
.
I
):
self
.
Read_Sentinel2_xmls
()
elif
re
.
search
(
"LANDSAT"
,
self
.
Satellite
,
re
.
I
):
self
.
Read_LANDSAT_mtltxt
(
self
.
LayerBandsAssignment
)
elif
re
.
search
(
"RapidEye"
,
self
.
Satellite
,
re
.
I
):
...
...
@@ -137,7 +137,7 @@ class METADATA(object):
self
.
Read_ALOS_summary
()
self
.
Read_ALOS_LEADER
()
# for gains and offsets
else
:
self
.
logger
.
critical
(
"
\t
ERROR:
%s is not a supported sensor or sensorname is misspelled."
%
self
.
Satellite
)
raise
NotImplementedError
(
"
%s is not a supported sensor or sensorname is misspelled."
%
self
.
Satellite
)
return
self
...
...
@@ -1215,17 +1215,18 @@ class METADATA(object):
self
.
Offsets
=
[
float
(
lead_
[
4680
*
3
+
2711
:
4680
*
3
+
2718
]),
float
(
lead_
[
4680
*
3
+
2727
:
4680
*
3
+
2734
]),
float
(
lead_
[
4680
*
3
+
2743
:
4680
*
3
+
2750
]),
float
(
lead_
[
4680
*
3
+
2759
:
4680
*
3
+
2766
])]
def
Read_Sentinel2
A
_xmls
(
self
):
"""Read metadata from Sentinel-2
A
generic xml and granule xml"""
def
Read_Sentinel2_xmls
(
self
):
"""Read metadata from Sentinel-2 generic xml and granule xml"""
# query database to get entityid
assert
self
.
SceneID
and
self
.
SceneID
!=
-
9999
,
"Read_Sentinel2
A
_xmls(): Missing scene ID. "
assert
self
.
SceneID
and
self
.
SceneID
!=
-
9999
,
"Read_Sentinel2_xmls(): Missing scene ID. "
res
=
DB_T
.
get_info_from_postgreSQLdb
(
CFG
.
conn_database
,
'scenes'
,
[
'entityid'
],
{
'id'
:
self
.
SceneID
})
assert
len
(
res
)
!=
0
,
\
"Invalid SceneID given - no corresponding scene with the ID=%s found in database.
\n
"
%
self
.
SceneID
assert
len
(
res
)
==
1
,
"Error in database. The sceneid %s exists more than once.
\n
"
%
self
.
SceneID
S2AgranuleID
=
res
[
0
][
0
]
S2AB
=
'S2A'
if
re
.
search
(
"Sentinel-2A"
,
self
.
Satellite
,
re
.
I
)
else
'S2B'
S2ABgranuleID
=
res
[
0
][
0
]
#################
# get XML roots #
...
...
@@ -1236,21 +1237,22 @@ class METADATA(object):
#####################################
# get xml_Scene_root (contains scene metadata)
glob_res
=
glob
.
glob
(
os
.
path
.
join
(
self
.
FolderOrArchive
,
'
S2A
*.xml'
))
glob_res
=
glob
.
glob
(
os
.
path
.
join
(
self
.
FolderOrArchive
,
'
%s
*.xml'
%
S2AB
))
if
not
glob_res
:
# new style packaging
glob_res
=
glob
.
glob
(
os
.
path
.
join
(
self
.
FolderOrArchive
,
'MTD*.xml'
))
assert
len
(
glob_res
)
>
0
,
'No
S2A
*.xml or MTD*.xml file can be found in %s/!'
%
self
.
FolderOrArchive
assert
len
(
glob_res
)
>
0
,
'No
%s
*.xml or MTD*.xml file can be found in %s/!'
%
(
S2AB
,
self
.
FolderOrArchive
)
self
.
Metafile
=
glob_res
[
0
]
xml_Scene_root
=
ET
.
parse
(
glob_res
[
0
]).
getroot
()
# xml_parser from file
# get xml_GR_root (contains granule metadata)
glob_res
=
glob
.
glob
(
os
.
path
.
join
(
self
.
FolderOrArchive
,
'GRANULE/'
+
S2AgranuleID
+
'/
S2A
*.xml'
))
glob_res
=
glob
.
glob
(
os
.
path
.
join
(
self
.
FolderOrArchive
,
'GRANULE/'
+
S2A
B
granuleID
+
'/
%s
*.xml'
%
S2AB
))
if
not
glob_res
:
# new style packaging
glob_res
=
glob
.
glob
(
os
.
path
.
join
(
self
.
FolderOrArchive
,
'GRANULE/'
+
S2AgranuleID
+
'/MTD*.xml'
))
glob_res
=
glob
.
glob
(
os
.
path
.
join
(
self
.
FolderOrArchive
,
'GRANULE/'
+
S2A
B
granuleID
+
'/MTD*.xml'
))
assert
len
(
glob_res
)
>
0
,
\
'No /GRANULE/<S2AgranuleID>/S2A*.xml or MTD*.xml file can be found in %s/!'
%
self
.
FolderOrArchive
'No /GRANULE/<%sgranuleID>/%s*.xml or MTD*.xml file can be found in %s/!'
\
%
(
S2AB
,
S2AB
,
self
.
FolderOrArchive
)
self
.
Metafile
=
self
.
Metafile
+
", "
+
glob_res
[
0
]
xml_GR_root
=
ET
.
parse
(
glob_res
).
getroot
()
# xml_parser from file
...
...
@@ -1261,14 +1263,16 @@ class METADATA(object):
# get xml_Scene_root and xml_GR_root (contain scene and granule metadata)
try
:
# old style packaging
xml_SC_str_
,
self
.
Metafile
=
open_specific_file_within_archive
(
self
.
FolderOrArchive
,
'*.SAFE/S2A*.xml'
)
xml_GR_str_
,
Metafile_
=
open_specific_file_within_archive
(
self
.
FolderOrArchive
,
'*.SAFE/GRANULE/'
+
S2AgranuleID
+
'/S2A*.xml'
)
xml_SC_str_
,
self
.
Metafile
=
\
open_specific_file_within_archive
(
self
.
FolderOrArchive
,
'*.SAFE/%s*.xml'
%
S2AB
)
xml_GR_str_
,
Metafile_
=
\
open_specific_file_within_archive
(
self
.
FolderOrArchive
,
'*.SAFE/GRANULE/'
+
S2ABgranuleID
+
'/%s*.xml'
%
S2AB
)
except
AssertionError
:
# new style packaging
xml_SC_str_
,
self
.
Metafile
=
open_specific_file_within_archive
(
self
.
FolderOrArchive
,
'*.SAFE/MTD*.xml'
)
xml_GR_str_
,
Metafile_
=
open_specific_file_within_archive
(
self
.
FolderOrArchive
,
'*.SAFE/GRANULE/'
+
S2AgranuleID
+
'/MTD*.xml'
)
S2A
B
granuleID
+
'/MTD*.xml'
)
xml_Scene_root
=
ET
.
fromstring
(
xml_SC_str_
)
xml_GR_root
=
ET
.
fromstring
(
xml_GR_str_
)
...
...
@@ -1278,7 +1282,7 @@ class METADATA(object):
# EXTRACT METADATA FROM XML ROOTS #
###################################
# define Sentinel 2
A
metadata (hard coded)
# define Sentinel 2 metadata (hard coded)
self
.
Sensor
=
"MSI"
# extract metadata from xml_Scene_root #
...
...
@@ -1286,7 +1290,7 @@ class METADATA(object):
# get the namespace within the xml_Scene_root
m
=
re
.
match
(
'{(.*)\}'
,
xml_Scene_root
.
tag
)
# extract namespace from "{https://....}Level-1C_Tile_ID"
assert
m
,
'XML Namespace could not be identified within metadata XML file.'
assert
m
,
'XML Namespace could not be identified within
Sentinel-2
metadata XML file.'
namespace
=
m
.
group
(
1
)
self
.
EntityID
=
\
...
...
@@ -1343,7 +1347,7 @@ class METADATA(object):
-
int
(
tmp
[:
-
1
])
if
tmp
[
-
1
]
==
'S'
else
self
.
CS_UTM_ZONE
# corner coords () #
subsytem_Res_dic
=
{
"
S2A10"
:
10
,
"S2A20"
:
20
,
"S2A60"
:
60
}
subsytem_Res_dic
=
{
"
%s10"
%
S2AB
:
10
,
"%s20"
%
S2AB
:
20
,
"%s60"
%
S2AB
:
60
}
if
self
.
CS_TYPE
==
'UTM'
:
spatial_samplings
=
{
float
(
size
.
get
(
"resolution"
)):
{
key
:
int
(
size
.
find
(
key
).
text
)
for
key
in
[
"NROWS"
,
"NCOLS"
]}
for
size
in
...
...
@@ -1948,21 +1952,27 @@ def get_special_values(GMS_identifier):
'SPOT1a': 0, 'SPOT2a': 0, 'SPOT3a': 0, 'SPOT4a': 0, 'SPOT5a': 0,
'SPOT1b': 0, 'SPOT2b': 0, 'SPOT3b': 0, 'SPOT4b': 0, 'SPOT5b': 0,
'RE5': 0,
'S2A10': 0, 'S2A20': 0, 'S2A60': 0}
'S2A10': 0, 'S2A20': 0, 'S2A60': 0,
'S2B10': 0, 'S2B20': 0, 'S2B60': 0,
}
dict_zero = {'AVNIR-2': None,
'AST_V1': 1, 'AST_V2': 1, 'AST_S': 1, 'AST_T': 1,
'TM4': None, 'TM5': None, 'TM7': None, 'LDCM': None,
'SPOT1a': None, 'SPOT2a': None, 'SPOT3a': None, 'SPOT4a': None, 'SPOT5a': None,
'SPOT1b': None, 'SPOT2b': None, 'SPOT3b': None, 'SPOT4b': None, 'SPOT5b': None,
'RE5': None,
'S2A10': None, 'S2A20': None, 'S2A60': None}
'S2A10': None, 'S2A20': None, 'S2A60': None,
'S2B10': None, 'S2B20': None, 'S2B60': None,
}
dict_saturated = {'AVNIR-2': None,
'AST_V1': 255, 'AST_V2': 255, 'AST_S': 255, 'AST_T': 65535,
'TM4': None, 'TM5': None, 'TM7': None, 'LDCM': 65535,
'SPOT1a': 255, 'SPOT2a': 255, 'SPOT3a': 255, 'SPOT4a': 255, 'SPOT5a': 255,
'SPOT1b': 255, 'SPOT2b': 255, 'SPOT3b': 255, 'SPOT4b': 255, 'SPOT5b': 255,
'RE5': None,
'S2A10': 65535, 'S2A20': 65535, 'S2A60': 65535}
'S2A10': 65535, 'S2A20': 65535, 'S2A60': 65535,
'S2B10': 65535, 'S2B20': 65535, 'S2B60': 65535
}
return {'fill': dict_fill[GMS_sensorcode],
'zero': dict_zero[GMS_sensorcode],
'saturated': dict_saturated[GMS_sensorcode]}
...
...
gms_preprocessing/options/config.py
View file @
a58ecd3e
...
...
@@ -608,6 +608,11 @@ class JobConfig(object):
sub_ds
=
ds
.
copy
()
sub_ds
[
'subsystem'
]
=
subsystem
data_list
.
append
(
sub_ds
)
elif
re
.
search
(
"Sentinel-2B"
,
ds
[
'satellite'
],
re
.
I
):
for
subsystem
in
[
'S2B10'
,
'S2B20'
,
'S2B60'
]:
sub_ds
=
ds
.
copy
()
sub_ds
[
'subsystem'
]
=
subsystem
data_list
.
append
(
sub_ds
)
elif
re
.
search
(
"Terra"
,
ds
[
'satellite'
],
re
.
I
):
for
subsystem
in
[
'VNIR1'
,
'VNIR2'
,
'SWIR'
,
'TIR'
]:
sub_ds
=
ds
.
copy
()
...
...
tests/data/archive_data/Sentinel-2A/MSI/S2A_MSIL1C_20161231T103432_N0204_R108_T32TLT_20161231T103428.zip
0 → 100644
View file @
a58ecd3e
File added
tests/data/archive_data/Sentinel-2A/MSI/S2A_MSIL1C_20161231T103432_N0204_R108_T32TLT_20161231T103428.zip.REMOVED.git-id
deleted
100644 → 0
View file @
613ed806
62339a73197f189e00bec8df51e56c1901633d61
\ No newline at end of file
tests/data/archive_data/Sentinel-2B/MSI/S2B_MSIL1C_20171110T074119_N0206_R092_T38SNE_20171110T113706.zip
0 → 100644
View file @
a58ecd3e
File added
tests/test_gms_preprocessing.py
View file @
a58ecd3e
...
...
@@ -240,6 +240,16 @@ class Test_Sentinel2A_MultiGranuleFormat(BaseTestCases.TestAll):
cls
.
create_job
(
26186272
,
job_config_kwargs
)
class
Test_Sentinel2B_SingleGranuleFormat
(
BaseTestCases
.
TestAll
):
"""
Parametrized testclass. Tests the level-processes on a Sentinel-2B MSI scene (1 granule in archive: > 2017).
More information on the dataset will be output after the tests-classes are executed.
"""
@
classmethod
def
setUpClass
(
cls
):
cls
.
create_job
(
26186937
,
job_config_kwargs
)
class
Test_MultipleDatasetsInOneJob
(
BaseTestCases
.
TestAll
):
"""
Parametrized testclass. Tests the level-processes on a job containing a Landsat-5 (pre-collection data),
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment