Commit aca9a730 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Merge branch 'master' into feature/spectral_homogenization


Former-commit-id: 6f07aa50
Former-commit-id: b2348cdf
parents a33d52f2 2bee3d1b
......@@ -61,7 +61,7 @@ test-all: ## run tests on every Python version with tox
tox
coverage: clean-test ## check code coverage quickly with the default Python
coverage run --source gms_preprocessing setup.py test
coverage run --source gms_preprocessing --source bin setup.py test
coverage combine # must be called in order to make coverage work in multiprocessing
coverage report -m
coverage html
......@@ -71,8 +71,8 @@ nosetests: clean-test ## Runs nosetests with coverage, xUnit and nose-html-outpu
## - puts the coverage results in the folder 'htmlcov'
## - generates 'nosetests.html' (--with-html)
## - generates 'nosetests.xml' (--with-xunit) which is currently not visualizable by GitLab
nosetests -vv --with-coverage --cover-package=gms_preprocessing --cover-erase --cover-html --cover-html-dir=htmlcov \
--with-html --with-xunit --rednose --force-color
nosetests -vv --with-coverage --cover-package=gms_preprocessing --cover-package=bin --cover-erase --cover-html \
--cover-html-dir=htmlcov --with-html --with-xunit --rednose --force-color
docs: ## generate Sphinx HTML documentation, including API docs
rm -f docs/gms_preprocessing.rst
......
......@@ -3,6 +3,7 @@ __author__ = 'Daniel Scheffler'
import argparse
import warnings
import os
import matplotlib
......@@ -10,6 +11,17 @@ matplotlib.use('Agg', warn=False) # switch matplotlib backend to 'Agg' and disa
from gms_preprocessing import process_controller, __version__ # noqa: E402
from gms_preprocessing.misc.database_tools import GMS_JOB # noqa: E402
from gms_preprocessing.options.config import get_conn_database # noqa: E402
from gms_preprocessing.options.config import path_options_default # noqa: E402
from gms_preprocessing.options.config import get_options # noqa: E402
from gms_preprocessing.options.config import get_config_kwargs_default # noqa: E402
options_default = get_options(path_options_default, validation=True) # type: dict
config_kwargs_default = get_config_kwargs_default() # type: dict
def get_config_kwargs_from_cli_args(cli_args):
return {k: v for k, v in cli_args.__dict__.items() if k in config_kwargs_default.keys()}
def run_from_jobid(args):
......@@ -19,12 +31,15 @@ def run_from_jobid(args):
# TODO download: run only the downloader
# set up process controller instance
PC = process_controller(args.jobid, parallelization_level='scenes', db_host='geoms') # FIXME hardcoded host
# PC.job.path_procdata_scenes = '/geoms/data/processed_scenes_dev'
# PC.job.path_procdata_MGRS = '/geoms/data/processed_mgrs_tiles_dev'
PC = process_controller(args.jobid, **get_config_kwargs_from_cli_args(args))
# run the job
PC.run_all_processors()
if 'GMS_IS_TEST' in os.environ and os.environ['GMS_IS_TEST'] == 'True':
# in case of software test, it is enough to get an instance of process controller because all inputs are
# validated within options.config.Job_Config (indirectly called by ProcessController.__init__() )
pass
else:
PC.run_all_processors()
def run_from_sceneids(args):
......@@ -32,19 +47,12 @@ def run_from_sceneids(args):
warnings.warn('Currently the console argument parser expects the given scenes as already downloaded.') # TODO
# create a new processing job from scene IDs
db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO
warnings.warn('Currently the console argument parser expects the database at localhost.') # TODO
virtual_sensor_id = 1 # TODO
warnings.warn('Currently the console argument parser sets the virtual sensor ID to 1.') # TODO
datasetid_spatial_ref = 249 # TODO
warnings.warn('Currently the console argument parser sets the dataset ID of the spatial reference to 249.') # TODO
dbJob = GMS_JOB(db_connection)
dbJob = GMS_JOB(get_conn_database(args.db_host))
dbJob.from_sceneIDlist(list_sceneIDs=args.sceneids,
virtual_sensor_id=virtual_sensor_id,
datasetid_spatial_ref=datasetid_spatial_ref,
comment='')
_run_job(dbJob)
virtual_sensor_id=args.virtual_sensor_id,
datasetid_spatial_ref=args.datasetid_spatial_ref,
comment=args.comment)
_run_job(dbJob, **get_config_kwargs_from_cli_args(args))
def run_from_entityids(args):
......@@ -53,20 +61,12 @@ def run_from_entityids(args):
:param args:
:return:
"""
db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO
warnings.warn('Currently the console argument parser expects the database at localhost.') # TODO
virtual_sensor_id = 1 # TODO
warnings.warn('Currently the console argument parser sets the virtual sensor ID to 1.') # TODO
datasetid_spatial_ref = 249 # TODO
warnings.warn('Currently the console argument parser sets the dataset ID of the spatial reference to 249.') # TODO
dbJob = GMS_JOB(db_connection)
dbJob = GMS_JOB(get_conn_database(args.db_host))
dbJob.from_entityIDlist(list_entityids=args.entityids,
virtual_sensor_id=virtual_sensor_id,
datasetid_spatial_ref=datasetid_spatial_ref,
comment='')
_run_job(dbJob)
virtual_sensor_id=args.virtual_sensor_id,
datasetid_spatial_ref=args.datasetid_spatial_ref,
comment=args.comment)
_run_job(dbJob, **get_config_kwargs_from_cli_args(args))
def run_from_filenames(args):
......@@ -75,20 +75,12 @@ def run_from_filenames(args):
:param args:
:return:
"""
db_connection = "dbname='geomultisens' user='gmsdb' password='gmsdb' host='localhost' connect_timeout=3" # TODO
warnings.warn('Currently the console argument parser expects the database at localhost.') # TODO
virtual_sensor_id = 1 # TODO
warnings.warn('Currently the console argument parser sets the virtual sensor ID to 1.') # TODO
datasetid_spatial_ref = 249 # TODO
warnings.warn('Currently the console argument parser sets the dataset ID of the spatial reference to 249.') # TODO
dbJob = GMS_JOB(db_connection)
dbJob = GMS_JOB(get_conn_database(args.db_host))
dbJob.from_filenames(list_filenames=args.filenames,
virtual_sensor_id=virtual_sensor_id,
datasetid_spatial_ref=datasetid_spatial_ref,
comment='')
_run_job(dbJob)
virtual_sensor_id=args.virtual_sensor_id,
datasetid_spatial_ref=args.datasetid_spatial_ref,
comment=args.comment)
_run_job(dbJob, **get_config_kwargs_from_cli_args(args))
def run_from_constraints(args):
......@@ -97,32 +89,35 @@ def run_from_constraints(args):
raise NotImplementedError
def _run_job(dbJob, parallelization_level='scenes'):
def _run_job(dbJob, **config_kwargs):
# type: (GMS_JOB) -> None
"""
:param dbJob:
:return:
"""
# create a database record for the given job
dbJob.create()
jobid = dbJob.id
# set up process controller instance
warnings.warn("Currently the console argument parser sets the parallelization level to 'scenes'.") # TODO
PC = process_controller(jobid, parallelization_level=parallelization_level)
# PC.job.path_procdata_scenes = '/geoms/data/processed_scenes_dev'
# PC.job.path_procdata_MGRS = '/geoms/data/processed_mgrs_tiles_dev'
PC = process_controller(dbJob.id, **config_kwargs)
# run the job
PC.run_all_processors()
if 'GMS_IS_TEST' in os.environ and os.environ['GMS_IS_TEST'] == 'True':
# in case of software test, it is enough to get an instance of process controller because all inputs are
# validated within options.config.Job_Config (indirectly called by ProcessController.__init__() )
pass
else:
PC.run_all_processors()
def get_gms_argparser():
"""Return argument parser for run_gms.py program."""
# CONFIGURE MAIN PARSER FOR THE GEOMULTISENS PREPROCESSING CHAIN
##################################################################
# CONFIGURE MAIN PARSER FOR THE GEOMULTISENS PREPROCESSING CHAIN #
##################################################################
parser = argparse.ArgumentParser(
prog='run_gms.py',
description='=' * 70 + '\n' + 'GeoMultiSens preprocessing console argument parser. '
......@@ -133,40 +128,95 @@ def get_gms_argparser():
parser.add_argument('--version', action='version', version=__version__)
subparsers = parser.add_subparsers()
#################################################################
# CONFIGURE SUBPARSERS FOR THE GEOMULTISENS PREPROCESSING CHAIN #
#################################################################
##############################################
# define parsers containing common arguments #
##############################################
general_opts_parser = argparse.ArgumentParser(add_help=False)
gop_p = general_opts_parser.add_argument
gop_p('-jc', '--json_config', nargs='?', type=str,
help='file path of a JSON file containing options. See here for an example: '
'https://gitext.gfz-potsdam.de/geomultisens/gms_preprocessing/'
'blob/master/gms_preprocessing/options/options_default.json')
# '-exec_L1AP': dict(nargs=3, type=bool, help="L1A Processor configuration",
# metavar=tuple("[run processor, write output, delete output]".split(' ')), default=[1, 1, 1]),
# CONFIGURE SUBPARSERS FOR THE GEOMULTISENS PREPROCESSING CHAIN
parser_jobid = subparsers\
.add_parser('jobid', description='Run a GeoMultiSens preprocessing job using an already existing job ID.',
help="Run a GeoMultiSens preprocessing job using an already existing job ID (Sub-Parser).")
gop_p('-DH', '--db_host', nargs='?', type=str,
default=options_default["global_opts"]["db_host"],
help='host name of the server that runs the postgreSQL database')
parser_sceneids = subparsers\
.add_parser('sceneids', description='Run a GeoMultiSens preprocessing job for a given list of scene IDs.',
help="Run a GeoMultiSens preprocessing job for a given list of scene IDs (Sub-Parser).")
gop_p('-DOO', '--delete_old_output', nargs='?', type=bool,
default=options_default["global_opts"]["delete_old_output"],
help='delete previously created output of the given job ID before running the job')
parser_entityids = subparsers\
.add_parser('entityids', description='Run a GeoMultiSens preprocessing job for a given list of entity IDs.',
help="Run a GeoMultiSens preprocessing job for a given list of entity IDs (Sub-Parser).")
gop_p('-vid', '--virtual_sensor_id', type=int,
default=options_default["usecase"]["virtual_sensor_id"],
help='ID of the target (virtual) sensor')
parser_filenames = subparsers\
.add_parser('filenames', description='Run a GeoMultiSens preprocessing job for a given list of filenames of '
'downloaded satellite image archives!',
help="Run a GeoMultiSens preprocessing job for a given list of filenames of downloaded satellite "
"image archives! (Sub-Parser).")
gop_p('-dsid_spat', '--datasetid_spatial_ref', type=int,
default=options_default["usecase"]["datasetid_spatial_ref"],
help='dataset ID of the spatial reference')
parser_constraints = subparsers\
.add_parser('constraints', description='Run a GeoMultiSens preprocessing job matching the given constraints.',
help="Run a GeoMultiSens preprocessing job matching the given constraints (Sub-Parser).")
gop_p('-c', '--comment', nargs='?', type=str,
default='',
help='comment concerning the job')
# parse_from_sceneids = subparsers.add_parser('from_sceneids',
# description='Run a GeoMultiSens preprocessing job for a given list of scene IDs.',
# help="use '>>> python /path/to/GeMultiSens/run_gms.py from_sceneids -h' for documentation and usage hints")
##################
# add subparsers #
##################
subparsers = parser.add_subparsers()
parser_jobid = subparsers.add_parser(
'jobid', parents=[general_opts_parser],
description='Run a GeoMultiSens preprocessing job using an already existing job ID.',
help="Run a GeoMultiSens preprocessing job using an already existing job ID (Sub-Parser).",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser_sceneids = subparsers.add_parser(
'sceneids', parents=[general_opts_parser],
description='Run a GeoMultiSens preprocessing job for a given list of scene IDs.',
help="Run a GeoMultiSens preprocessing job for a given list of scene IDs (Sub-Parser).",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser_entityids = subparsers.add_parser(
'entityids', parents=[general_opts_parser],
description='Run a GeoMultiSens preprocessing job for a given list of entity IDs.',
help="Run a GeoMultiSens preprocessing job for a given list of entity IDs (Sub-Parser).",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser_filenames = subparsers.add_parser(
'filenames', parents=[general_opts_parser],
description='Run a GeoMultiSens preprocessing job for a given list of filenames of '
'downloaded satellite image archives!',
help="Run a GeoMultiSens preprocessing job for a given list of filenames of downloaded satellite "
"image archives! (Sub-Parser).",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser_constraints = subparsers.add_parser(
'constraints', parents=[general_opts_parser],
description='Run a GeoMultiSens preprocessing job matching the given constraints.',
help="Run a GeoMultiSens preprocessing job matching the given constraints (Sub-Parser).",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
#################
# ADD ARGUMENTS #
#################
##########################
# add indivial arguments #
##########################
# ADD ARGUMENTS
# add arguments to parser_jobid
jid_p = parser_jobid.add_argument
jid_p('jobid', type=int, help='job ID of an already created GeoMultiSens preprocessing job (must be present in the '
'jobs table of the database)')
jid_p('jobid', type=int, help='job ID of an already created GeoMultiSens preprocessing job '
'(must be present in the jobs table of the database)')
# add arguments to parser_sceneids
sid_p = parser_sceneids.add_argument
......@@ -191,27 +241,10 @@ def get_gms_argparser():
# con_p('constraints', nargs='+', type=str, help="list of entity IDs corresponding to valid records within the "
# "'scenes' table of the database")
# add general arguments # TODO add these configs to each subparser
general_opts = {
'-db_host': dict(),
'-exec_mode': dict(nargs=3, type=bool, help="L1A Processor configuration",
metavar=tuple("[run processor, write output, delete output]".split(' ')), default=[1, 1, 1]),
'-exec_L1AP': dict(),
'-exec_L1BP': dict(),
'-exec_L1CP': dict(),
'-exec_L2AP': dict(),
'-exec_L2BP': dict(),
'-exec_L2CP': dict(),
'-sub_multiProc': dict(),
'-exc_handler': dict(),
'-blocksize': dict(),
'-profiling': dict(),
'-bench_all': dict(),
'-bench_cloudMask': dict(),
}
# LINK PARSERS TO RUN FUNCTIONS
#################################
# LINK PARSERS TO RUN FUNCTIONS #
#################################
parser_jobid.set_defaults(func=run_from_jobid)
parser_sceneids.set_defaults(func=run_from_sceneids)
parser_entityids.set_defaults(func=run_from_entityids)
......
......@@ -8,17 +8,21 @@ from . import algorithms # noqa: E402
from . import io # noqa: E402
from . import misc # noqa: E402
from . import processing # noqa: E402
from . import config # noqa: E402
from . import options # noqa: F401 (imported but unused)
from .options import config # noqa: F401 (imported but unused)
from .options.config import set_config # noqa: F401 (imported but unused)
from .processing.process_controller import process_controller # noqa: E402
__author__ = """Daniel Scheffler"""
__email__ = 'daniel.scheffler@gfz-potsdam.de'
__version__ = '0.9.10'
__versionalias__ = '20171120.02'
__version__ = '0.11.0'
__versionalias__ = '20171205.01'
__all__ = ['algorithms',
'io',
'misc',
'processing',
'config',
'config' # only to keep compatibility with HU-INF codes
'options',
'set_config',
'process_controller',
]
......@@ -19,7 +19,7 @@ from py_tools_ds.geo.coord_trafo import pixelToLatLon
from py_tools_ds.geo.map_info import mapinfo2geotransform
from py_tools_ds.geo.projection import EPSG2WKT
from ..config import GMS_config as CFG
from ..options.config import GMS_config as CFG
from . import geoprocessing as GEOP
from ..io import output_writer as OUT_W
from ..misc import helper_functions as HLP_F
......@@ -143,7 +143,7 @@ class L1A_object(GMS_object):
subset = ['block', [[sub_dim[0], sub_dim[1] + 1], [sub_dim[2], sub_dim[3] + 1]]]
rasObj = GEOP.GEOPROCESSING(paths_files2stack[0], self.logger, subset=subset)
if CFG.job.exec_mode == 'Flink' and path_output is None: # numpy array output
if CFG.exec_mode == 'Flink' and path_output is None: # numpy array output
self.arr = rasObj.Layerstacking(paths_files2stack)
self.path_InFilePreprocessor = paths_files2stack[0]
else: # 'MEMORY' or physical output
......@@ -162,7 +162,7 @@ class L1A_object(GMS_object):
subset = ['block', [[sub_dim[0], sub_dim[1] + 1], [sub_dim[2], sub_dim[3] + 1]]]
rasObj = GEOP.GEOPROCESSING(path_file2load, self.logger, subset=subset)
if CFG.job.exec_mode == 'Flink' and path_output is None: # numpy array output
if CFG.exec_mode == 'Flink' and path_output is None: # numpy array output
self.arr = gdalnumeric.LoadFile(path_file2load) if subset is None else \
gdalnumeric.LoadFile(path_file2load, rasObj.colStart, rasObj.rowStart, rasObj.cols, rasObj.rows)
self.path_InFilePreprocessor = path_file2load
......@@ -190,7 +190,7 @@ class L1A_object(GMS_object):
data_arr = np.empty(data.shape + (len(self.LayerBandsAssignment),), data.dtype)
data_arr[:, :, bidx] = data
if CFG.job.exec_mode == 'Flink' and path_output is None: # numpy array output
if CFG.exec_mode == 'Flink' and path_output is None: # numpy array output
self.arr = data_arr
else:
GEOP.ndarray2gdal(data_arr, path_output, geotransform=ds.GetGeoTransform(),
......@@ -221,7 +221,7 @@ class L1A_object(GMS_object):
data_arr = np.empty(data.shape + (len(self.LayerBandsAssignment),), data.dtype)
data_arr[:, :, i] = data
if CFG.job.exec_mode == 'Flink' and path_output is None: # numpy array output
if CFG.exec_mode == 'Flink' and path_output is None: # numpy array output
self.arr = data_arr
else:
GEOP.ndarray2gdal(data_arr, path_output, direction=3)
......@@ -271,7 +271,7 @@ class L1A_object(GMS_object):
def calc_TOARadRefTemp(self, subset=None):
"""Convert DN, Rad or TOA_Ref data to TOA Reflectance, to Radiance or to Surface Temperature
(depending on CFG.usecase.conversion_type_optical and conversion_type_thermal).
(depending on CFG.target_radunit_optical and target_radunit_thermal).
The function can be executed by a L1A_object representing a full scene or a tile. To process a file from disk
in tiles, provide an item of self.tile_pos as the 'subset' argument."""
......@@ -305,7 +305,7 @@ class L1A_object(GMS_object):
for optical_thermal in ['optical', 'thermal']:
if optical_thermal not in self.dict_LayerOptTherm.values():
continue
conv = getattr(CFG.usecase, 'conversion_type_%s' % optical_thermal)
conv = getattr(CFG, 'target_radunit_%s' % optical_thermal)
conv = conv if conv != 'BOA_Ref' else 'TOA_Ref'
assert conv in ['Rad', 'TOA_Ref', 'Temp'], 'Unsupported conversion type: %s' % conv
arr_desc = self.arr_desc.split('/')[0] if optical_thermal == 'optical' else self.arr_desc.split('/')[-1]
......@@ -330,7 +330,7 @@ class L1A_object(GMS_object):
inSaturated) if conv == 'TOA_Ref' else \
GEOP.DN2DegreesCelsius_fastforward(inArray, OFF, GAI, K1, K2, 0.95, inFill, inZero, inSaturated)
if conv == 'TOA_Ref':
self.MetaObj.ScaleFactor = CFG.usecase.scale_factor_TOARef
self.MetaObj.ScaleFactor = CFG.scale_factor_TOARef
elif arr_desc == 'Rad':
raise NotImplementedError("Conversion Rad to %s is currently not supported." % conv)
......@@ -349,16 +349,16 @@ class L1A_object(GMS_object):
'13 bands and it not clear for which bands the gains are provided.')
raise NotImplementedError("Conversion TOA_Ref to %s is currently not supported." % conv)
else: # conv=='TOA_Ref'
if self.MetaObj.ScaleFactor != CFG.usecase.scale_factor_TOARef:
res = self.rescale_array(inArray, CFG.usecase.scale_factor_TOARef, self.MetaObj.ScaleFactor)
self.MetaObj.ScaleFactor = CFG.usecase.scale_factor_TOARef
if self.MetaObj.ScaleFactor != CFG.scale_factor_TOARef:
res = self.rescale_array(inArray, CFG.scale_factor_TOARef, self.MetaObj.ScaleFactor)
self.MetaObj.ScaleFactor = CFG.scale_factor_TOARef
self.log_for_fullArr_or_firstTile(
'Rescaling Ref data to scaling factor %d.' % CFG.usecase.scale_factor_TOARef)
'Rescaling Ref data to scaling factor %d.' % CFG.scale_factor_TOARef)
else:
res = inArray
self.log_for_fullArr_or_firstTile('The input data already represents TOA '
'reflectance with the desired scale factor of %d.'
% CFG.usecase.scale_factor_TOARef)
% CFG.scale_factor_TOARef)
else: # arr_desc == 'Temp'
raise NotImplementedError("Conversion Temp to %s is currently not supported." % conv)
......@@ -390,8 +390,8 @@ class L1A_object(GMS_object):
self.update_spec_vals_according_to_dtype('int16')
tiles_desc = '_'.join([desc for op_th, desc in zip(['optical', 'thermal'],
[CFG.usecase.conversion_type_optical,
CFG.usecase.conversion_type_thermal])
[CFG.target_radunit_optical,
CFG.target_radunit_thermal])
if desc in self.dict_LayerOptTherm.values()])
self.arr = dataOut
......@@ -452,12 +452,12 @@ class L1A_object(GMS_object):
dst_CS_datum='WGS84', mode='GDAL', use_workspace=True,
inFill=self.MetaObj.spec_vals['fill'])
if CFG.job.exec_mode == 'Python':
if CFG.exec_mode == 'Python':
path_warped = os.path.join(self.ExtractedFolder, self.baseN + '__' + self.arr_desc)
GEOP.ndarray2gdal(rasObj.tondarray(direction=3), path_warped, importFile=rasObj.desc, direction=3)
self.MetaObj.Dataname = path_warped
self.arr = path_warped
else: # CFG.job.exec_mode=='Flink':
else: # CFG.exec_mode=='Flink':
self.arr = rasObj.tondarray(direction=3)
self.shape_fullArr = [rasObj.rows, rasObj.cols, rasObj.bands]
......@@ -473,7 +473,7 @@ class L1A_object(GMS_object):
self.MetaObj.CornerTieP_UTM = rasObj.get_corner_coordinates('UTM')
self.meta_odict = self.MetaObj.to_odict() # important in order to keep geotransform/projection
if CFG.job.exec_mode == 'Flink':
if CFG.exec_mode == 'Flink':
self.delete_tempFiles() # these files are needed later in Python execution mode
self.MetaObj.Dataname = previous_dataname # /vsi.. pointing directly to raw data archive (which exists)
......
......@@ -27,7 +27,7 @@ from py_tools_ds.geo.coord_trafo import reproject_shapelyGeometry, transform_any
from py_tools_ds.geo.projection import prj_equal, EPSG2WKT, WKT2EPSG
from py_tools_ds.geo.vector.topology import get_overlap_polygon
from ..config import GMS_config as CFG
from ..options.config import GMS_config as CFG
from ..model.gms_object import GMS_object
from .L1A_P import L1A_object
from ..misc import database_tools as DB_T
......@@ -40,9 +40,23 @@ __author__ = 'Daniel Scheffler'
class Scene_finder(object):
"""Scene_finder class to query the postgreSQL database to find a suitable reference scene for co-registration."""
def __init__(self, src_boundsLonLat, src_AcqDate, src_prj, src_footprint_poly, sceneID_excluded=None,
min_overlap=20, min_cloudcov=0, max_cloudcov=20, plusminus_days=30, plusminus_years=10):
"""Initialize Scene_finder.
:param src_boundsLonLat:
:param src_AcqDate:
:param src_prj:
:param src_footprint_poly:
:param sceneID_excluded:
:param min_overlap: minimum overlap of reference scene in percent
:param min_cloudcov: minimum cloud cover of reference scene in percent
:param max_cloudcov: maximum cloud cover of reference scene in percent
:param plusminus_days: maximum time interval between target and reference scene in days
:param plusminus_years: maximum time interval between target and reference scene in years
"""
self.boundsLonLat = src_boundsLonLat
self.src_AcqDate = src_AcqDate
self.src_prj = src_prj
......@@ -76,7 +90,7 @@ class Scene_finder(object):
SpIM = SpatialIndexMediator(timeout=timeout)
self.possib_ref_scenes = \
SpIM.getFullSceneDataForDataset(self.boundsLonLat, self.timeStart, self.timeEnd, self.min_cloudcov,
self.max_cloudcov, CFG.usecase.datasetid_spatial_ref,
self.max_cloudcov, CFG.datasetid_spatial_ref,
refDate=self.src_AcqDate, maxDaysDelta=self.plusminus_days)
break
except socket.timeout:
......@@ -116,7 +130,7 @@ class Scene_finder(object):
# get processing level of reference scenes
procL = GeoDataFrame(
DB_T.get_info_from_postgreSQLdb(CFG.job.conn_database, 'scenes_proc', ['sceneid', 'proc_level'],
DB_T.get_info_from_postgreSQLdb(CFG.conn_database, 'scenes_proc', ['sceneid', 'proc_level'],
{'sceneid': list(GDF.sceneid)}), columns=['sceneid', 'proc_level'])
GDF = GDF.merge(procL, on='sceneid', how='left')
GDF = GDF.where(GDF.notnull(), None) # replace NaN values with None
......@@ -129,7 +143,7 @@ class Scene_finder(object):
GDF['refDs_exists'] = list(GDF['path_ref'].map(lambda p: os.path.exists(p) if p else False))
# check if a proper entity ID can be gathered from database
eID = GeoDataFrame(DB_T.get_info_from_postgreSQLdb(CFG.job.conn_database, 'scenes', ['id', 'entityid'],
eID = GeoDataFrame(DB_T.get_info_from_postgreSQLdb(CFG.conn_database, 'scenes', ['id', 'entityid'],
{'id': list(GDF.sceneid)}), columns=['sceneid', 'entityid'])
GDF = GDF.merge(eID, on='sceneid', how='left')
self.GDF_ref_scenes = GDF.where(GDF.notnull(), None)
......@@ -245,7 +259,12 @@ class L1B_object(L1A_object):
boundsLonLat = corner_coord_to_minmax(self.trueDataCornerLonLat)
footprint_poly = HLP_F.CornerLonLat_to_shapelyPoly(self.trueDataCornerLonLat)
RSF = Scene_finder(boundsLonLat, self.acq_datetime, self.meta_odict['coordinate system string'],
footprint_poly, self.scene_ID, 20, 0, 20, 30, 10)
footprint_poly, self.scene_ID,
min_overlap=CFG.spatial_ref_min_overlap,
min_cloudcov=CFG.spatial_ref_min_cloudcov,
max_cloudcov=CFG.spatial_ref_max_cloudcov,
plusminus_days=CFG.spatial_ref_plusminus_days,
plusminus_years=CFG.spatial_ref_plusminus_years)
# run spatial query
self.logger.info('Querying database in order to find a suitable reference scene for co-registration.')
......@@ -284,7 +303,7 @@ class L1B_object(L1A_object):
plusminus_days = 30
AcqDate = self.im2shift_objDict['acquisition_date']
date_minmax = [AcqDate - timedelta(days=plusminus_days), AcqDate + timedelta(days=plusminus_days)]
dataset_cond = 'datasetid=%s' % CFG.usecase.datasetid_spatial_ref
dataset_cond = 'datasetid=%s' % CFG.datasetid_spatial_ref
cloudcov_cond = 'cloudcover < %s' % max_cloudcov
# FIXME cloudcover noch nicht für alle scenes im proc_level METADATA verfügbar
dayrange_cond = "(EXTRACT(MONTH FROM scenes.acquisitiondate), EXTRACT(DAY FROM scenes.acquisitiondate)) " \
......@@ -294,7 +313,7 @@ class L1B_object(L1A_object):
def query_scenes(condlist):
return DB_T.get_overlapping_scenes_from_postgreSQLdb(
CFG.job.conn_database,
CFG.conn_database,
table='scenes',
tgt_corners_lonlat=self.trueDataCornerLonLat,
conditions=condlist,
......@@ -311,9 +330,9 @@ class L1B_object(L1A_object):
# das ist nur Ergebnis aus scenes_proc
# -> dort liegt nur eine referenz, wenn die szene schon bei CFG.job-Beginn in Datensatzliste drin war
res = DB_T.get_overlapping_scenes_from_postgreSQLdb(
CFG.job.conn_database,
CFG.conn_database,
tgt_corners_lonlat=self.trueDataCornerLonLat,
conditions=['datasetid=%s' % CFG.usecase.datasetid_spatial_ref],
conditions=['datasetid=%s' % CFG.datasetid_spatial_ref],
add_cmds='ORDER BY scenes.cloudcover ASC',
timeout=25000)
filt_overlap_scenes = self._sceneIDList_to_filt_overlap_scenes([i[0] for i in res[:50]], 20.)
......@@ -354,8 +373,8 @@ class L1B_object(L1A_object):
# start download of scene data not available and start L1A processing
def dl_cmd(scene_ID): print('%s %s %s' % (
CFG.job.java_commands['keyword'].strip(), # FIXME CFG.job.java_commands is deprecated
CFG.job.java_commands["value_download"].strip(), scene_ID))
CFG.java_commands['keyword'].strip(), # FIXME CFG.java_commands is deprecated
CFG.java_commands["value_download"].strip(), scene_ID))
path = PG.path_generator(scene_ID=sc['scene_ID']).get_path_imagedata()