Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
geomultisens
gms_preprocessing
Commits
6f07aa50
Commit
6f07aa50
authored
Dec 06, 2017
by
Daniel Scheffler
Browse files
Merge branch 'master' into feature/spectral_homogenization
parents
979bdc22
3a439762
Changes
60
Expand all
Hide whitespace changes
Inline
Side-by-side
Makefile
View file @
6f07aa50
...
...
@@ -61,7 +61,7 @@ test-all: ## run tests on every Python version with tox
tox
coverage
:
clean-test
##
check code coverage quickly with the default Python
coverage run
--source
gms_preprocessing setup.py
test
coverage run
--source
gms_preprocessing
--source
bin
setup.py
test
coverage combine
# must be called in order to make coverage work in multiprocessing
coverage report
-m
coverage html
...
...
@@ -71,8 +71,8 @@ nosetests: clean-test ## Runs nosetests with coverage, xUnit and nose-html-outpu
## - puts the coverage results in the folder 'htmlcov'
## - generates 'nosetests.html'
(
--with-html
)
## - generates 'nosetests.xml'
(
--with-xunit
)
which is currently not visualizable by GitLab
nosetests
-vv
--with-coverage
--cover-package
=
gms_preprocessing
--cover-
erase
--cover-
html
--cover-html
-dir
=
htmlcov
\
--with-html
--with-xunit
--rednose
--force-color
nosetests
-vv
--with-coverage
--cover-package
=
gms_preprocessing
--cover-
package
=
bin
--cover-
erase
--cover-html
\
--cover-html-dir
=
htmlcov
--with-html
--with-xunit
--rednose
--force-color
docs
:
##
generate Sphinx HTML documentation
,
including API docs
rm
-f
docs/gms_preprocessing.rst
...
...
bin/run_gms.py
View file @
6f07aa50
...
...
@@ -3,6 +3,7 @@ __author__ = 'Daniel Scheffler'
import
argparse
import
warnings
import
os
import
matplotlib
...
...
@@ -10,6 +11,17 @@ matplotlib.use('Agg', warn=False) # switch matplotlib backend to 'Agg' and disa
from
gms_preprocessing
import
process_controller
,
__version__
# noqa: E402
from
gms_preprocessing.misc.database_tools
import
GMS_JOB
# noqa: E402
from
gms_preprocessing.options.config
import
get_conn_database
# noqa: E402
from
gms_preprocessing.options.config
import
path_options_default
# noqa: E402
from
gms_preprocessing.options.config
import
get_options
# noqa: E402
from
gms_preprocessing.options.config
import
get_config_kwargs_default
# noqa: E402
options_default
=
get_options
(
path_options_default
,
validation
=
True
)
# type: dict
config_kwargs_default
=
get_config_kwargs_default
()
# type: dict
def
get_config_kwargs_from_cli_args
(
cli_args
):
return
{
k
:
v
for
k
,
v
in
cli_args
.
__dict__
.
items
()
if
k
in
config_kwargs_default
.
keys
()}
def
run_from_jobid
(
args
):
...
...
@@ -19,12 +31,15 @@ def run_from_jobid(args):
# TODO download: run only the downloader
# set up process controller instance
PC
=
process_controller
(
args
.
jobid
,
parallelization_level
=
'scenes'
,
db_host
=
'geoms'
)
# FIXME hardcoded host
# PC.job.path_procdata_scenes = '/geoms/data/processed_scenes_dev'
# PC.job.path_procdata_MGRS = '/geoms/data/processed_mgrs_tiles_dev'
PC
=
process_controller
(
args
.
jobid
,
**
get_config_kwargs_from_cli_args
(
args
))
# run the job
PC
.
run_all_processors
()
if
'GMS_IS_TEST'
in
os
.
environ
and
os
.
environ
[
'GMS_IS_TEST'
]
==
'True'
:
# in case of software test, it is enough to get an instance of process controller because all inputs are
# validated within options.config.Job_Config (indirectly called by ProcessController.__init__() )
pass
else
:
PC
.
run_all_processors
()
def
run_from_sceneids
(
args
):
...
...
@@ -32,19 +47,12 @@ def run_from_sceneids(args):
warnings
.
warn
(
'Currently the console argument parser expects the given scenes as already downloaded.'
)
# TODO
# create a new processing job from scene IDs
db_connection
=
"dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3"
# TODO
warnings
.
warn
(
'Currently the console argument parser expects the database at localhost.'
)
# TODO
virtual_sensor_id
=
1
# TODO
warnings
.
warn
(
'Currently the console argument parser sets the virtual sensor ID to 1.'
)
# TODO
datasetid_spatial_ref
=
249
# TODO
warnings
.
warn
(
'Currently the console argument parser sets the dataset ID of the spatial reference to 249.'
)
# TODO
dbJob
=
GMS_JOB
(
db_connection
)
dbJob
=
GMS_JOB
(
get_conn_database
(
args
.
db_host
))
dbJob
.
from_sceneIDlist
(
list_sceneIDs
=
args
.
sceneids
,
virtual_sensor_id
=
virtual_sensor_id
,
datasetid_spatial_ref
=
datasetid_spatial_ref
,
comment
=
''
)
_run_job
(
dbJob
)
virtual_sensor_id
=
args
.
virtual_sensor_id
,
datasetid_spatial_ref
=
args
.
datasetid_spatial_ref
,
comment
=
args
.
comment
)
_run_job
(
dbJob
,
**
get_config_kwargs_from_cli_args
(
args
)
)
def
run_from_entityids
(
args
):
...
...
@@ -53,20 +61,12 @@ def run_from_entityids(args):
:param args:
:return:
"""
db_connection
=
"dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3"
# TODO
warnings
.
warn
(
'Currently the console argument parser expects the database at localhost.'
)
# TODO
virtual_sensor_id
=
1
# TODO
warnings
.
warn
(
'Currently the console argument parser sets the virtual sensor ID to 1.'
)
# TODO
datasetid_spatial_ref
=
249
# TODO
warnings
.
warn
(
'Currently the console argument parser sets the dataset ID of the spatial reference to 249.'
)
# TODO
dbJob
=
GMS_JOB
(
db_connection
)
dbJob
=
GMS_JOB
(
get_conn_database
(
args
.
db_host
))
dbJob
.
from_entityIDlist
(
list_entityids
=
args
.
entityids
,
virtual_sensor_id
=
virtual_sensor_id
,
datasetid_spatial_ref
=
datasetid_spatial_ref
,
comment
=
''
)
_run_job
(
dbJob
)
virtual_sensor_id
=
args
.
virtual_sensor_id
,
datasetid_spatial_ref
=
args
.
datasetid_spatial_ref
,
comment
=
args
.
comment
)
_run_job
(
dbJob
,
**
get_config_kwargs_from_cli_args
(
args
)
)
def
run_from_filenames
(
args
):
...
...
@@ -75,20 +75,12 @@ def run_from_filenames(args):
:param args:
:return:
"""
db_connection
=
"dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3"
# TODO
warnings
.
warn
(
'Currently the console argument parser expects the database at localhost.'
)
# TODO
virtual_sensor_id
=
1
# TODO
warnings
.
warn
(
'Currently the console argument parser sets the virtual sensor ID to 1.'
)
# TODO
datasetid_spatial_ref
=
249
# TODO
warnings
.
warn
(
'Currently the console argument parser sets the dataset ID of the spatial reference to 249.'
)
# TODO
dbJob
=
GMS_JOB
(
db_connection
)
dbJob
=
GMS_JOB
(
get_conn_database
(
args
.
db_host
))
dbJob
.
from_filenames
(
list_filenames
=
args
.
filenames
,
virtual_sensor_id
=
virtual_sensor_id
,
datasetid_spatial_ref
=
datasetid_spatial_ref
,
comment
=
''
)
_run_job
(
dbJob
)
virtual_sensor_id
=
args
.
virtual_sensor_id
,
datasetid_spatial_ref
=
args
.
datasetid_spatial_ref
,
comment
=
args
.
comment
)
_run_job
(
dbJob
,
**
get_config_kwargs_from_cli_args
(
args
)
)
def
run_from_constraints
(
args
):
...
...
@@ -97,32 +89,35 @@ def run_from_constraints(args):
raise
NotImplementedError
def
_run_job
(
dbJob
,
parallelization_level
=
'scenes'
):
def
_run_job
(
dbJob
,
**
config_kwargs
):
# type: (GMS_JOB) -> None
"""
:param dbJob:
:return:
"""
# create a database record for the given job
dbJob
.
create
()
jobid
=
dbJob
.
id
# set up process controller instance
warnings
.
warn
(
"Currently the console argument parser sets the parallelization level to 'scenes'."
)
# TODO
PC
=
process_controller
(
jobid
,
parallelization_level
=
parallelization_level
)
# PC.job.path_procdata_scenes = '/geoms/data/processed_scenes_dev'
# PC.job.path_procdata_MGRS = '/geoms/data/processed_mgrs_tiles_dev'
PC
=
process_controller
(
dbJob
.
id
,
**
config_kwargs
)
# run the job
PC
.
run_all_processors
()
if
'GMS_IS_TEST'
in
os
.
environ
and
os
.
environ
[
'GMS_IS_TEST'
]
==
'True'
:
# in case of software test, it is enough to get an instance of process controller because all inputs are
# validated within options.config.Job_Config (indirectly called by ProcessController.__init__() )
pass
else
:
PC
.
run_all_processors
()
def
get_gms_argparser
():
"""Return argument parser for run_gms.py program."""
# CONFIGURE MAIN PARSER FOR THE GEOMULTISENS PREPROCESSING CHAIN
##################################################################
# CONFIGURE MAIN PARSER FOR THE GEOMULTISENS PREPROCESSING CHAIN #
##################################################################
parser
=
argparse
.
ArgumentParser
(
prog
=
'run_gms.py'
,
description
=
'='
*
70
+
'
\n
'
+
'GeoMultiSens preprocessing console argument parser. '
...
...
@@ -133,40 +128,95 @@ def get_gms_argparser():
parser
.
add_argument
(
'--version'
,
action
=
'version'
,
version
=
__version__
)
subparsers
=
parser
.
add_subparsers
()
#################################################################
# CONFIGURE SUBPARSERS FOR THE GEOMULTISENS PREPROCESSING CHAIN #
#################################################################
##############################################
# define parsers containing common arguments #
##############################################
general_opts_parser
=
argparse
.
ArgumentParser
(
add_help
=
False
)
gop_p
=
general_opts_parser
.
add_argument
gop_p
(
'-jc'
,
'--json_config'
,
nargs
=
'?'
,
type
=
str
,
help
=
'file path of a JSON file containing options. See here for an example: '
'https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing/'
'blob/master/gms_preprocessing/options/options_default.json'
)
# '-exec_L1AP': dict(nargs=3, type=bool, help="L1A Processor configuration",
# metavar=tuple("[run processor, write output, delete output]".split(' ')), default=[1, 1, 1]),
# CONFIGURE SUBPARSERS FOR THE GEOMULTISENS PREPROCESSING CHAIN
parser_jobid
=
subparsers
\
.
add_parser
(
'jobid'
,
description
=
'Run a GeoMultiSens preprocessing job using an already existing job ID.'
,
help
=
"Run a GeoMultiSens preprocessing job using an already existing job ID (Sub-Parser)."
)
gop_p
(
'-DH'
,
'--db_host'
,
nargs
=
'?'
,
type
=
str
,
default
=
options_default
[
"global_opts"
][
"db_host"
],
help
=
'host name of the server that runs the postgreSQL database'
)
parser_sceneids
=
subparsers
\
.
add_parser
(
'sceneids'
,
description
=
'Run a GeoMultiSens preprocessing job for a given list of scene IDs.'
,
help
=
"Run a GeoMultiSens preprocessing job for a given list of scene IDs (Sub-Parser)."
)
gop_p
(
'-DOO'
,
'--delete_old_output'
,
nargs
=
'?'
,
type
=
bool
,
default
=
options_default
[
"global_opts"
][
"delete_old_output"
]
,
help
=
'delete previously created output of the given job ID before running the job'
)
parser_entityids
=
subparsers
\
.
add_parser
(
'entityids'
,
description
=
'Run a GeoMultiSens preprocessing job for a given list of entity IDs.'
,
help
=
"Run a GeoMultiSens preprocessing job for a given list of entity IDs (Sub-Parser)."
)
gop_p
(
'-vid'
,
'--virtual_sensor_id'
,
type
=
int
,
default
=
options_default
[
"usecase"
][
"virtual_sensor_id"
]
,
help
=
'ID of the target (virtual) sensor'
)
parser_filenames
=
subparsers
\
.
add_parser
(
'filenames'
,
description
=
'Run a GeoMultiSens preprocessing job for a given list of filenames of '
'downloaded satellite image archives!'
,
help
=
"Run a GeoMultiSens preprocessing job for a given list of filenames of downloaded satellite "
"image archives! (Sub-Parser)."
)
gop_p
(
'-dsid_spat'
,
'--datasetid_spatial_ref'
,
type
=
int
,
default
=
options_default
[
"usecase"
][
"datasetid_spatial_ref"
],
help
=
'dataset ID of the spatial reference'
)
parser_constraints
=
subparsers
\
.
add_parser
(
'constraints'
,
description
=
'Run a GeoMultiSens preprocessing job matching the given constraints.
'
,
help
=
"Run a GeoMultiSens preprocessing job matching the given constraints (Sub-Parser)."
)
gop_p
(
'-c'
,
'--comment'
,
nargs
=
'?'
,
type
=
str
,
default
=
'
'
,
help
=
'comment concerning the job'
)
# parse_from_sceneids = subparsers.add_parser('from_sceneids',
# description='Run a GeoMultiSens preprocessing job for a given list of scene IDs.',
# help="use '>>> python /path/to/GeMultiSens/run_gms.py from_sceneids -h' for documentation and usage hints")
##################
# add subparsers #
##################
subparsers
=
parser
.
add_subparsers
()
parser_jobid
=
subparsers
.
add_parser
(
'jobid'
,
parents
=
[
general_opts_parser
],
description
=
'Run a GeoMultiSens preprocessing job using an already existing job ID.'
,
help
=
"Run a GeoMultiSens preprocessing job using an already existing job ID (Sub-Parser)."
,
formatter_class
=
argparse
.
ArgumentDefaultsHelpFormatter
)
parser_sceneids
=
subparsers
.
add_parser
(
'sceneids'
,
parents
=
[
general_opts_parser
],
description
=
'Run a GeoMultiSens preprocessing job for a given list of scene IDs.'
,
help
=
"Run a GeoMultiSens preprocessing job for a given list of scene IDs (Sub-Parser)."
,
formatter_class
=
argparse
.
ArgumentDefaultsHelpFormatter
)
parser_entityids
=
subparsers
.
add_parser
(
'entityids'
,
parents
=
[
general_opts_parser
],
description
=
'Run a GeoMultiSens preprocessing job for a given list of entity IDs.'
,
help
=
"Run a GeoMultiSens preprocessing job for a given list of entity IDs (Sub-Parser)."
,
formatter_class
=
argparse
.
ArgumentDefaultsHelpFormatter
)
parser_filenames
=
subparsers
.
add_parser
(
'filenames'
,
parents
=
[
general_opts_parser
],
description
=
'Run a GeoMultiSens preprocessing job for a given list of filenames of '
'downloaded satellite image archives!'
,
help
=
"Run a GeoMultiSens preprocessing job for a given list of filenames of downloaded satellite "
"image archives! (Sub-Parser)."
,
formatter_class
=
argparse
.
ArgumentDefaultsHelpFormatter
)
parser_constraints
=
subparsers
.
add_parser
(
'constraints'
,
parents
=
[
general_opts_parser
],
description
=
'Run a GeoMultiSens preprocessing job matching the given constraints.'
,
help
=
"Run a GeoMultiSens preprocessing job matching the given constraints (Sub-Parser)."
,
formatter_class
=
argparse
.
ArgumentDefaultsHelpFormatter
)
#################
# ADD ARGUMENTS #
#################
##########################
# add indivial arguments #
##########################
# ADD ARGUMENTS
# add arguments to parser_jobid
jid_p
=
parser_jobid
.
add_argument
jid_p
(
'jobid'
,
type
=
int
,
help
=
'job ID of an already created GeoMultiSens preprocessing job
(must be present in the
'
'jobs table of the database)'
)
jid_p
(
'jobid'
,
type
=
int
,
help
=
'job ID of an already created GeoMultiSens preprocessing job '
'
(must be present in the
jobs table of the database)'
)
# add arguments to parser_sceneids
sid_p
=
parser_sceneids
.
add_argument
...
...
@@ -191,27 +241,10 @@ def get_gms_argparser():
# con_p('constraints', nargs='+', type=str, help="list of entity IDs corresponding to valid records within the "
# "'scenes' table of the database")
# add general arguments # TODO add these configs to each subparser
general_opts
=
{
'-db_host'
:
dict
(),
'-exec_mode'
:
dict
(
nargs
=
3
,
type
=
bool
,
help
=
"L1A Processor configuration"
,
metavar
=
tuple
(
"[run processor, write output, delete output]"
.
split
(
' '
)),
default
=
[
1
,
1
,
1
]),
'-exec_L1AP'
:
dict
(),
'-exec_L1BP'
:
dict
(),
'-exec_L1CP'
:
dict
(),
'-exec_L2AP'
:
dict
(),
'-exec_L2BP'
:
dict
(),
'-exec_L2CP'
:
dict
(),
'-sub_multiProc'
:
dict
(),
'-exc_handler'
:
dict
(),
'-blocksize'
:
dict
(),
'-profiling'
:
dict
(),
'-bench_all'
:
dict
(),
'-bench_cloudMask'
:
dict
(),
}
# LINK PARSERS TO RUN FUNCTIONS
#################################
# LINK PARSERS TO RUN FUNCTIONS #
#################################
parser_jobid
.
set_defaults
(
func
=
run_from_jobid
)
parser_sceneids
.
set_defaults
(
func
=
run_from_sceneids
)
parser_entityids
.
set_defaults
(
func
=
run_from_entityids
)
...
...
gms_preprocessing/__init__.py
View file @
6f07aa50
...
...
@@ -8,17 +8,21 @@ from . import algorithms # noqa: E402
from
.
import
io
# noqa: E402
from
.
import
misc
# noqa: E402
from
.
import
processing
# noqa: E402
from
.
import
config
# noqa: E402
from
.
import
options
# noqa: F401 (imported but unused)
from
.options
import
config
# noqa: F401 (imported but unused)
from
.options.config
import
set_config
# noqa: F401 (imported but unused)
from
.processing.process_controller
import
process_controller
# noqa: E402
__author__
=
"""Daniel Scheffler"""
__email__
=
'daniel.scheffler@gfz-potsdam.de'
__version__
=
'0.
9.1
0'
__versionalias__
=
'20171
1
20.0
2
'
__version__
=
'0.
11.
0'
__versionalias__
=
'2017120
5
.0
1
'
__all__
=
[
'algorithms'
,
'io'
,
'misc'
,
'processing'
,
'config'
,
'config'
# only to keep compatibility with HU-INF codes
'options'
,
'set_config'
,
'process_controller'
,
]
gms_preprocessing/algorithms/L1A_P.py
View file @
6f07aa50
...
...
@@ -19,7 +19,7 @@ from py_tools_ds.geo.coord_trafo import pixelToLatLon
from
py_tools_ds.geo.map_info
import
mapinfo2geotransform
from
py_tools_ds.geo.projection
import
EPSG2WKT
from
..config
import
GMS_config
as
CFG
from
..
options.
config
import
GMS_config
as
CFG
from
.
import
geoprocessing
as
GEOP
from
..io
import
output_writer
as
OUT_W
from
..misc
import
helper_functions
as
HLP_F
...
...
@@ -143,7 +143,7 @@ class L1A_object(GMS_object):
subset
=
[
'block'
,
[[
sub_dim
[
0
],
sub_dim
[
1
]
+
1
],
[
sub_dim
[
2
],
sub_dim
[
3
]
+
1
]]]
rasObj
=
GEOP
.
GEOPROCESSING
(
paths_files2stack
[
0
],
self
.
logger
,
subset
=
subset
)
if
CFG
.
job
.
exec_mode
==
'Flink'
and
path_output
is
None
:
# numpy array output
if
CFG
.
exec_mode
==
'Flink'
and
path_output
is
None
:
# numpy array output
self
.
arr
=
rasObj
.
Layerstacking
(
paths_files2stack
)
self
.
path_InFilePreprocessor
=
paths_files2stack
[
0
]
else
:
# 'MEMORY' or physical output
...
...
@@ -162,7 +162,7 @@ class L1A_object(GMS_object):
subset
=
[
'block'
,
[[
sub_dim
[
0
],
sub_dim
[
1
]
+
1
],
[
sub_dim
[
2
],
sub_dim
[
3
]
+
1
]]]
rasObj
=
GEOP
.
GEOPROCESSING
(
path_file2load
,
self
.
logger
,
subset
=
subset
)
if
CFG
.
job
.
exec_mode
==
'Flink'
and
path_output
is
None
:
# numpy array output
if
CFG
.
exec_mode
==
'Flink'
and
path_output
is
None
:
# numpy array output
self
.
arr
=
gdalnumeric
.
LoadFile
(
path_file2load
)
if
subset
is
None
else
\
gdalnumeric
.
LoadFile
(
path_file2load
,
rasObj
.
colStart
,
rasObj
.
rowStart
,
rasObj
.
cols
,
rasObj
.
rows
)
self
.
path_InFilePreprocessor
=
path_file2load
...
...
@@ -190,7 +190,7 @@ class L1A_object(GMS_object):
data_arr
=
np
.
empty
(
data
.
shape
+
(
len
(
self
.
LayerBandsAssignment
),),
data
.
dtype
)
data_arr
[:,
:,
bidx
]
=
data
if
CFG
.
job
.
exec_mode
==
'Flink'
and
path_output
is
None
:
# numpy array output
if
CFG
.
exec_mode
==
'Flink'
and
path_output
is
None
:
# numpy array output
self
.
arr
=
data_arr
else
:
GEOP
.
ndarray2gdal
(
data_arr
,
path_output
,
geotransform
=
ds
.
GetGeoTransform
(),
...
...
@@ -221,7 +221,7 @@ class L1A_object(GMS_object):
data_arr
=
np
.
empty
(
data
.
shape
+
(
len
(
self
.
LayerBandsAssignment
),),
data
.
dtype
)
data_arr
[:,
:,
i
]
=
data
if
CFG
.
job
.
exec_mode
==
'Flink'
and
path_output
is
None
:
# numpy array output
if
CFG
.
exec_mode
==
'Flink'
and
path_output
is
None
:
# numpy array output
self
.
arr
=
data_arr
else
:
GEOP
.
ndarray2gdal
(
data_arr
,
path_output
,
direction
=
3
)
...
...
@@ -271,7 +271,7 @@ class L1A_object(GMS_object):
def
calc_TOARadRefTemp
(
self
,
subset
=
None
):
"""Convert DN, Rad or TOA_Ref data to TOA Reflectance, to Radiance or to Surface Temperature
(depending on CFG.
usecase.conversion_type_optical and conversion_type
_thermal).
(depending on CFG.
target_radunit_optical and target_radunit
_thermal).
The function can be executed by a L1A_object representing a full scene or a tile. To process a file from disk
in tiles, provide an item of self.tile_pos as the 'subset' argument."""
...
...
@@ -305,7 +305,7 @@ class L1A_object(GMS_object):
for
optical_thermal
in
[
'optical'
,
'thermal'
]:
if
optical_thermal
not
in
self
.
dict_LayerOptTherm
.
values
():
continue
conv
=
getattr
(
CFG
.
usecase
,
'conversion_type
_%s'
%
optical_thermal
)
conv
=
getattr
(
CFG
,
'target_radunit
_%s'
%
optical_thermal
)
conv
=
conv
if
conv
!=
'BOA_Ref'
else
'TOA_Ref'
assert
conv
in
[
'Rad'
,
'TOA_Ref'
,
'Temp'
],
'Unsupported conversion type: %s'
%
conv
arr_desc
=
self
.
arr_desc
.
split
(
'/'
)[
0
]
if
optical_thermal
==
'optical'
else
self
.
arr_desc
.
split
(
'/'
)[
-
1
]
...
...
@@ -330,7 +330,7 @@ class L1A_object(GMS_object):
inSaturated
)
if
conv
==
'TOA_Ref'
else
\
GEOP
.
DN2DegreesCelsius_fastforward
(
inArray
,
OFF
,
GAI
,
K1
,
K2
,
0.95
,
inFill
,
inZero
,
inSaturated
)
if
conv
==
'TOA_Ref'
:
self
.
MetaObj
.
ScaleFactor
=
CFG
.
usecase
.
scale_factor_TOARef
self
.
MetaObj
.
ScaleFactor
=
CFG
.
scale_factor_TOARef
elif
arr_desc
==
'Rad'
:
raise
NotImplementedError
(
"Conversion Rad to %s is currently not supported."
%
conv
)
...
...
@@ -349,16 +349,16 @@ class L1A_object(GMS_object):
'13 bands and it not clear for which bands the gains are provided.'
)
raise
NotImplementedError
(
"Conversion TOA_Ref to %s is currently not supported."
%
conv
)
else
:
# conv=='TOA_Ref'
if
self
.
MetaObj
.
ScaleFactor
!=
CFG
.
usecase
.
scale_factor_TOARef
:
res
=
self
.
rescale_array
(
inArray
,
CFG
.
usecase
.
scale_factor_TOARef
,
self
.
MetaObj
.
ScaleFactor
)
self
.
MetaObj
.
ScaleFactor
=
CFG
.
usecase
.
scale_factor_TOARef
if
self
.
MetaObj
.
ScaleFactor
!=
CFG
.
scale_factor_TOARef
:
res
=
self
.
rescale_array
(
inArray
,
CFG
.
scale_factor_TOARef
,
self
.
MetaObj
.
ScaleFactor
)
self
.
MetaObj
.
ScaleFactor
=
CFG
.
scale_factor_TOARef
self
.
log_for_fullArr_or_firstTile
(
'Rescaling Ref data to scaling factor %d.'
%
CFG
.
usecase
.
scale_factor_TOARef
)
'Rescaling Ref data to scaling factor %d.'
%
CFG
.
scale_factor_TOARef
)
else
:
res
=
inArray
self
.
log_for_fullArr_or_firstTile
(
'The input data already represents TOA '
'reflectance with the desired scale factor of %d.'
%
CFG
.
usecase
.
scale_factor_TOARef
)
%
CFG
.
scale_factor_TOARef
)
else
:
# arr_desc == 'Temp'
raise
NotImplementedError
(
"Conversion Temp to %s is currently not supported."
%
conv
)
...
...
@@ -390,8 +390,8 @@ class L1A_object(GMS_object):
self
.
update_spec_vals_according_to_dtype
(
'int16'
)
tiles_desc
=
'_'
.
join
([
desc
for
op_th
,
desc
in
zip
([
'optical'
,
'thermal'
],
[
CFG
.
usecase
.
conversion_type
_optical
,
CFG
.
usecase
.
conversion_type
_thermal
])
[
CFG
.
target_radunit
_optical
,
CFG
.
target_radunit
_thermal
])
if
desc
in
self
.
dict_LayerOptTherm
.
values
()])
self
.
arr
=
dataOut
...
...
@@ -452,12 +452,12 @@ class L1A_object(GMS_object):
dst_CS_datum
=
'WGS84'
,
mode
=
'GDAL'
,
use_workspace
=
True
,
inFill
=
self
.
MetaObj
.
spec_vals
[
'fill'
])
if
CFG
.
job
.
exec_mode
==
'Python'
:
if
CFG
.
exec_mode
==
'Python'
:
path_warped
=
os
.
path
.
join
(
self
.
ExtractedFolder
,
self
.
baseN
+
'__'
+
self
.
arr_desc
)
GEOP
.
ndarray2gdal
(
rasObj
.
tondarray
(
direction
=
3
),
path_warped
,
importFile
=
rasObj
.
desc
,
direction
=
3
)
self
.
MetaObj
.
Dataname
=
path_warped
self
.
arr
=
path_warped
else
:
# CFG.
job.
exec_mode=='Flink':
else
:
# CFG.exec_mode=='Flink':
self
.
arr
=
rasObj
.
tondarray
(
direction
=
3
)
self
.
shape_fullArr
=
[
rasObj
.
rows
,
rasObj
.
cols
,
rasObj
.
bands
]
...
...
@@ -473,7 +473,7 @@ class L1A_object(GMS_object):
self
.
MetaObj
.
CornerTieP_UTM
=
rasObj
.
get_corner_coordinates
(
'UTM'
)
self
.
meta_odict
=
self
.
MetaObj
.
to_odict
()
# important in order to keep geotransform/projection
if
CFG
.
job
.
exec_mode
==
'Flink'
:
if
CFG
.
exec_mode
==
'Flink'
:
self
.
delete_tempFiles
()
# these files are needed later in Python execution mode
self
.
MetaObj
.
Dataname
=
previous_dataname
# /vsi.. pointing directly to raw data archive (which exists)
...
...
gms_preprocessing/algorithms/L1B_P.py
View file @
6f07aa50
...
...
@@ -27,7 +27,7 @@ from py_tools_ds.geo.coord_trafo import reproject_shapelyGeometry, transform_any
from
py_tools_ds.geo.projection
import
prj_equal
,
EPSG2WKT
,
WKT2EPSG
from
py_tools_ds.geo.vector.topology
import
get_overlap_polygon
from
..config
import
GMS_config
as
CFG
from
..
options.
config
import
GMS_config
as
CFG
from
..model.gms_object
import
GMS_object
from
.L1A_P
import
L1A_object
from
..misc
import
database_tools
as
DB_T
...
...
@@ -40,9 +40,23 @@ __author__ = 'Daniel Scheffler'
class
Scene_finder
(
object
):
"""Scene_finder class to query the postgreSQL database to find a suitable reference scene for co-registration."""
def
__init__
(
self
,
src_boundsLonLat
,
src_AcqDate
,
src_prj
,
src_footprint_poly
,
sceneID_excluded
=
None
,
min_overlap
=
20
,
min_cloudcov
=
0
,
max_cloudcov
=
20
,
plusminus_days
=
30
,
plusminus_years
=
10
):
"""Initialize Scene_finder.
:param src_boundsLonLat:
:param src_AcqDate:
:param src_prj:
:param src_footprint_poly:
:param sceneID_excluded:
:param min_overlap: minimum overlap of reference scene in percent
:param min_cloudcov: minimum cloud cover of reference scene in percent
:param max_cloudcov: maximum cloud cover of reference scene in percent
:param plusminus_days: maximum time interval between target and reference scene in days
:param plusminus_years: maximum time interval between target and reference scene in years
"""
self
.
boundsLonLat
=
src_boundsLonLat
self
.
src_AcqDate
=
src_AcqDate
self
.
src_prj
=
src_prj
...
...
@@ -76,7 +90,7 @@ class Scene_finder(object):
SpIM
=
SpatialIndexMediator
(
timeout
=
timeout
)
self
.
possib_ref_scenes
=
\
SpIM
.
getFullSceneDataForDataset
(
self
.
boundsLonLat
,
self
.
timeStart
,
self
.
timeEnd
,
self
.
min_cloudcov
,
self
.
max_cloudcov
,
CFG
.
usecase
.
datasetid_spatial_ref
,
self
.
max_cloudcov
,
CFG
.
datasetid_spatial_ref
,
refDate
=
self
.
src_AcqDate
,
maxDaysDelta
=
self
.
plusminus_days
)
break
except
socket
.
timeout
:
...
...
@@ -116,7 +130,7 @@ class Scene_finder(object):
# get processing level of reference scenes
procL
=
GeoDataFrame
(
DB_T
.
get_info_from_postgreSQLdb
(
CFG
.
job
.
conn_database
,
'scenes_proc'
,
[
'sceneid'
,
'proc_level'
],
DB_T
.
get_info_from_postgreSQLdb
(
CFG
.
conn_database
,
'scenes_proc'
,
[
'sceneid'
,
'proc_level'
],
{
'sceneid'
:
list
(
GDF
.
sceneid
)}),
columns
=
[
'sceneid'
,
'proc_level'
])
GDF
=
GDF
.
merge
(
procL
,
on
=
'sceneid'
,
how
=
'left'
)
GDF
=
GDF
.
where
(
GDF
.
notnull
(),
None
)
# replace NaN values with None
...
...
@@ -129,7 +143,7 @@ class Scene_finder(object):
GDF
[
'refDs_exists'
]
=
list
(
GDF
[
'path_ref'
].
map
(
lambda
p
:
os
.
path
.
exists
(
p
)
if
p
else
False
))
# check if a proper entity ID can be gathered from database
eID
=
GeoDataFrame
(
DB_T
.
get_info_from_postgreSQLdb
(
CFG
.
job
.
conn_database
,
'scenes'
,
[
'id'
,
'entityid'
],
eID
=
GeoDataFrame
(
DB_T
.
get_info_from_postgreSQLdb
(
CFG
.
conn_database
,
'scenes'
,
[
'id'
,
'entityid'
],
{
'id'
:
list
(
GDF
.
sceneid
)}),
columns
=
[
'sceneid'
,
'entityid'
])
GDF
=
GDF
.
merge
(
eID
,
on
=
'sceneid'
,
how
=
'left'
)
self
.
GDF_ref_scenes
=
GDF
.
where
(
GDF
.
notnull
(),
None
)
...
...
@@ -245,7 +259,12 @@ class L1B_object(L1A_object):
boundsLonLat
=
corner_coord_to_minmax
(
self
.
trueDataCornerLonLat
)
footprint_poly
=
HLP_F
.
CornerLonLat_to_shapelyPoly
(
self
.
trueDataCornerLonLat
)
RSF
=
Scene_finder
(
boundsLonLat
,
self
.
acq_datetime
,
self
.
meta_odict
[
'coordinate system string'
],
footprint_poly
,
self
.
scene_ID
,
20
,
0
,
20
,
30
,
10
)
footprint_poly
,
self
.
scene_ID
,
min_overlap
=
CFG
.
spatial_ref_min_overlap
,
min_cloudcov
=
CFG
.
spatial_ref_min_cloudcov
,
max_cloudcov
=
CFG
.
spatial_ref_max_cloudcov
,
plusminus_days
=
CFG
.
spatial_ref_plusminus_days
,
plusminus_years
=
CFG
.
spatial_ref_plusminus_years
)
# run spatial query
self
.
logger
.
info
(
'Querying database in order to find a suitable reference scene for co-registration.'
)
...
...
@@ -284,7 +303,7 @@ class L1B_object(L1A_object):
plusminus_days
=
30
AcqDate
=
self
.
im2shift_objDict
[
'acquisition_date'
]
date_minmax
=
[
AcqDate
-
timedelta
(
days
=
plusminus_days
),
AcqDate
+
timedelta
(
days
=
plusminus_days
)]
dataset_cond
=
'datasetid=%s'
%
CFG
.
usecase
.
datasetid_spatial_ref
dataset_cond
=
'datasetid=%s'
%
CFG
.
datasetid_spatial_ref
cloudcov_cond
=
'cloudcover < %s'
%
max_cloudcov
# FIXME cloudcover noch nicht für alle scenes im proc_level METADATA verfügbar
dayrange_cond
=
"(EXTRACT(MONTH FROM scenes.acquisitiondate), EXTRACT(DAY FROM scenes.acquisitiondate)) "
\
...
...
@@ -294,7 +313,7 @@ class L1B_object(L1A_object):
def
query_scenes
(
condlist
):
return
DB_T
.
get_overlapping_scenes_from_postgreSQLdb
(
CFG
.
job
.
conn_database
,
CFG
.
conn_database
,
table
=
'scenes'
,
tgt_corners_lonlat
=
self
.
trueDataCornerLonLat
,
conditions
=
condlist
,
...
...
@@ -311,9 +330,9 @@ class L1B_object(L1A_object):
# das ist nur Ergebnis aus scenes_proc
# -> dort liegt nur eine referenz, wenn die szene schon bei CFG.job-Beginn in Datensatzliste drin war
res
=
DB_T
.
get_overlapping_scenes_from_postgreSQLdb
(
CFG
.
job
.
conn_database
,
CFG
.
conn_database
,
tgt_corners_lonlat
=
self
.
trueDataCornerLonLat
,
conditions
=
[
'datasetid=%s'
%
CFG
.
usecase
.
datasetid_spatial_ref
],
conditions
=
[
'datasetid=%s'
%
CFG
.
datasetid_spatial_ref
],
add_cmds
=
'ORDER BY scenes.cloudcover ASC'
,
timeout
=
25000
)
filt_overlap_scenes
=
self
.
_sceneIDList_to_filt_overlap_scenes
([
i
[
0
]
for
i
in
res
[:
50
]],
20.
)
...
...
@@ -354,8 +373,8 @@ class L1B_object(L1A_object):
# start download of scene data not available and start L1A processing
def
dl_cmd
(
scene_ID
):
print
(
'%s %s %s'
%
(
CFG
.
job
.
java_commands
[
'keyword'
].
strip
(),
# FIXME CFG.
job.
java_commands is deprecated
CFG
.
job
.
java_commands
[
"value_download"
].
strip
(),
scene_ID
))
CFG
.
java_commands
[
'keyword'
].
strip
(),
# FIXME CFG.java_commands is deprecated
CFG
.
java_commands
[
"value_download"
].
strip
(),
scene_ID
))
path
=
PG
.
path_generator
(
scene_ID
=
sc
[
'scene_ID'
]).
get_path_imagedata
()