Skip to content
GitLab
Commit a16e19c1 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Config revision - intermediate state: JSON file completely parsed. 


Former-commit-id: cc4f6a67
parent 068e26cb
Expand all Hide whitespace changes
Inline Side-by-side
bin/run_gms.py
View file @ a16e19c1
......@@ -207,7 +207,6 @@ def get_gms_argparser():
'-blocksize': dict(),
'-profiling': dict(),
'-bench_all': dict(),
'-bench_cloudMask': dict(),
}
......
gms_preprocessing/algorithms/L1C_P.py
View file @ a16e19c1
......@@ -148,8 +148,8 @@ class L1C_object(L1B_object):
meshwidth=10,
nodata_mask=None, # dont overwrite areas outside the image with nodata
outFill=get_outFillZeroSaturated(np.float32)[0],
accurracy=CFG.job.SZA_SAA_calculation_accurracy,
lonlat_arr=self.lonlat_arr if CFG.job.SZA_SAA_calculation_accurracy == 'fine' else None)
accurracy=CFG.usecase.SZA_SAA_calculation_accurracy,
lonlat_arr=self.lonlat_arr if CFG.usecase.SZA_SAA_calculation_accurracy == 'fine' else None)
return self._SZA_arr
@SZA_arr.setter
......@@ -662,7 +662,7 @@ class AtmCorr(object):
# compute cloud mask if not already provided
if no_avail_CMs:
algorithm = CFG.job.cloud_masking_algorithm[self.inObjs[0].satellite]
algorithm = CFG.usecase.cloud_masking_algorithm[self.inObjs[0].satellite]
if algorithm == 'SICOR':
return None
......@@ -792,7 +792,7 @@ class AtmCorr(object):
script = False
# check if ECMWF data are available - if not, start the download
if CFG.job.auto_download_ecmwf:
if CFG.usecase.auto_download_ecmwf:
self._check_or_download_ECMWF_data()
# validate SNR
......
gms_preprocessing/config.py
View file @ a16e19c1
This diff is collapsed.
gms_preprocessing/io/input_reader.py
View file @ a16e19c1
......@@ -337,10 +337,10 @@ class SRF(object):
plt.legend(loc='upper right')
def pickle_SRF_DB(L1A_Instances):
def pickle_SRF_DB(L1A_Instances, dir_out):
list_GMS_identifiers = [i.GMS_identifier for i in L1A_Instances]
out_dict = collections.OrderedDict()
logger = GMS_logger('log__SRF2PKL', path_logfile=os.path.join(CFG.job.path_testing, 'out/log__SRF2PKL.log'),
logger = GMS_logger('log__SRF2PKL', path_logfile=os.path.join(dir_out, 'log__SRF2PKL.log'),
log_level=CFG.job.log_level, append=False)
for Id, Inst in zip(list_GMS_identifiers, L1A_Instances):
Id['logger'] = logger
......@@ -348,7 +348,7 @@ def pickle_SRF_DB(L1A_Instances):
Inst.satellite + '_' + Inst.sensor + (('_' + Inst.subsystem) if Inst.subsystem not in ['', None] else '')] \
= SRF_reader(Id)
print(list(out_dict.keys()))
outFilename = os.path.join(CFG.job.path_testing, 'out/SRF_DB.pkl')
outFilename = os.path.join(dir_out, 'SRF_DB.pkl')
with open(outFilename, 'wb') as outFile:
dill.dump(out_dict, outFile)
print('Saved SRF dictionary to %s' % outFilename)
......
gms_preprocessing/misc/database_tools.py
View file @ a16e19c1
......@@ -610,19 +610,22 @@ class GMS_JOB(object):
"""
:param conn_db: <str> the database connection parameters as given by CFG.job.conn_params
"""
# privates
self._virtualsensorid = None
# defaults
self.conn = conn_db
self.dataframe = GeoDataFrame()
self.scene_counts = {} # set by self.create()
self.exists_in_db = False
self.id = None #: int
self.creationtime = datetime.now()
self.creationtime = datetime.now() # default, needed to create new job
self.finishtime = None
self.sceneids = []
self.timerange_start = datetime.min
self.timerange_end = datetime.max
self.bounds = box(-180, -90, 180, 90)
self.bounds = box(-180, -90, 180, 90) # default, needed to create new job
self.distribution_index = None
self.progress = None
self.feedback = None
......@@ -633,20 +636,31 @@ class GMS_JOB(object):
'timerange_end', 'bounds', 'distribution_index', 'progress', 'feedback',
'failed_sceneids', 'datasetid_spatial_ref',
'virtualsensorid', 'ref_job_id', 'datacube_mgrs_tiles_proc', 'comment',
'status', 'statistics']
self.virtualsensorid = None # set by self._set_target_sensor_specs()
# FIXME notnull but not getable via virtual sensor id and not needed anymore in DB (gsd is given):
self.datasetid_spatial_ref = 249
self.datasetname_spatial_ref = 'SENTINEL-2A'
'non_ref_datasetids', 'max_cloudcover', 'season_code', 'status',
'path_analysis_script', 'analysis_parameter', 'statistics', 'job_mode']
self.datasetid_spatial_ref = 249 # this is overwritten if existing job is read from DB but needed to create new
self.datasetname_spatial_ref = 'SENTINEL-2A' # same here
self.status = None
self.statistics = []
self.comment = None
self.epsg = None # set by self._set_target_sensor_specs()
self.ground_spatial_sampling = None # set by self._set_target_sensor_specs()
self.analysis_parameter = None
def __repr__(self):
return 'GMS job:\n\n' + GeoSeries(self.db_entry).to_string()
@property
def virtualsensorid(self):
return self._virtualsensorid
@virtualsensorid.setter
def virtualsensorid(self, value):
"""Set virtual sensor ID but continue if no data value is received
NOTE: set by self._set_target_sensor_specs() and self.from_ID()"""
if value != -1: # no data value
self._virtualsensorid = value
def _set_target_sensor_specs(self, virtual_sensor_id, datasetid_spatial_ref):
self.virtualsensorid = virtual_sensor_id
res = get_info_from_postgreSQLdb(self.conn, 'virtual_sensors', ['spatial_resolution',
......@@ -671,11 +685,17 @@ class GMS_JOB(object):
db_entry = collections.OrderedDict()
for i in self.jobs_table_columns:
db_entry[i] = getattr(self, i)
val = getattr(self, i)
if i == 'virtualsensorid' and val is None:
val = -1 # nodata value
db_entry[i] = val
return db_entry
def from_dictlist(self, dictlist_data2process, virtual_sensor_id, datasetid_spatial_ref=249, comment=None):
# type: (list, int, int, str) -> object
# type: (list, int, int, str) -> GMS_JOB
"""
:param dictlist_data2process: <list> a list of dictionaries containing the keys "satellite", "sensor" and
"filenames",
......@@ -910,7 +930,7 @@ class GMS_JOB(object):
self.timerange_end = self.dataframe.acquisitiondate.max().to_pydatetime()
def from_job_ID(self, job_ID):
# type: (int) -> object
# type: (int) -> GMS_JOB
"""
Create a GMS_JOB instance by querying the database for a specific job ID.
:param job_ID: <int> a valid id from the database table 'jobs'
......
gms_preprocessing/misc/definition_dicts.py
View file @ a16e19c1
......@@ -104,7 +104,7 @@ def get_mask_classdefinition(maskname, satellite):
'Snow': 60} # SICOR
}
return legends[CFG.job.cloud_masking_algorithm[satellite]]
return legends[CFG.usecase.cloud_masking_algorithm[satellite]]
else:
raise ValueError("'%s' is not a supported mask name." % maskname)
......
gms_preprocessing/misc/environment.py
View file @ a16e19c1
......@@ -47,7 +47,7 @@ class GMSEnvironment(object):
# noinspection PyUnresolvedReferences
import fmask # noqa F401 unused
except ImportError:
if 'FMASK' in list(CFG.job.cloud_masking_algorithm.values()):
if 'FMASK' in list(CFG.usecase.cloud_masking_algorithm.values()):
msg = "FMASK library is not installed because it is not pip-installable and must be installed " \
"manually, e.g. for Anaconda by running 'conda install -c conda-forge python-fmask'!"
self.logger.warning(MissingNonPipLibraryWarning(msg))
......
gms_preprocessing/model/gms_object.py
View file @ a16e19c1
......@@ -302,7 +302,7 @@ class GMS_object(Dataset):
@property
def cloud_masking_algorithm(self):
if not self._cloud_masking_algorithm:
self._cloud_masking_algorithm = CFG.job.cloud_masking_algorithm[self.satellite]
self._cloud_masking_algorithm = CFG.usecase.cloud_masking_algorithm[self.satellite]
return self._cloud_masking_algorithm
@property
......
gms_preprocessing/options_default.json
View file @ a16e19c1
{
"job": {
"exec_mode": "Python", /*"Python" or "Flink"*/
"db_host": "localhost",
"CPUs": "None", /*number of CPU cores to be used for processing (default: "None" -> use all available)*/
"allow_subMultiprocessing": true, /*allow multiprocessing within workers*/
"disable_exception_handler": false, /*enable/disable automatic handling of unexpected exceptions*/
"log_level": "INFO", /*the logging level to be used (choices: 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL';*/
"tiling_block_size_XY": [
2048,
2048
], /*X/Y block size to be used for any tiling process*/
"is_test": false, /*whether the current job represents a software test job (run by a test runner) or not*/
"profiling": false, /*enable/disable code profiling*/
"benchmark_global": false, /*enable/disable benchmark of the whole processing pipeline*/
"bench_cloudMask": false, /*enable/disable benchmark of the of the cloud mask generator module*/
"paths": {
"path_fileserver": "",
"path_archive": "", /*input path where downloaded data are stored*/
"path_procdata_scenes": "", /*output path to store processed scenes*/
"path_procdata_MGRS": "", /*output path to store processed MGRS tiles*/
"path_tempdir": "",
"path_testing": "",
"path_benchmarks": "",
"path_job_logs": "", /*output path to store job log files*/
"path_spatIdxSrv": "",
"path_ac_tables": "",
"path_SNR_models": "",
"path_SRFs": "",
"path_dem_proc_srtm_90m": "",
"path_earthSunDist": "",
"path_solar_irr": "",
"path_cloud_classif": "",
"path_ECMWF_db": ""
"exec_mode": "Python", /*"Python" or "Flink"*/
"db_host": "localhost",
"CPUs": "None", /*number of CPU cores to be used for processing (default: "None" -> use all available)*/
"allow_subMultiprocessing": true, /*allow multiprocessing within workers*/
"disable_exception_handler": false, /*enable/disable automatic handling of unexpected exceptions*/
"log_level": "INFO", /*the logging level to be used (choices: 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL';*/
"tiling_block_size_XY": [
2048,
2048
], /*X/Y block size to be used for any tiling process*/
"is_test": false, /*whether the current job represents a software test job (run by a test runner) or not*/
"profiling": false, /*enable/disable code profiling*/
"benchmark_global": false, /*enable/disable benchmark of the whole processing pipeline*/
"paths": {
"path_fileserver": "",
"path_archive": "", /*input path where downloaded data are stored*/
"path_procdata_scenes": "", /*output path to store processed scenes*/
"path_procdata_MGRS": "", /*output path to store processed MGRS tiles*/
"path_tempdir": "",
"path_benchmarks": "",
"path_job_logs": "", /*output path to store job log files*/
"path_spatIdxSrv": "",
"path_ac_tables": "",
"path_SNR_models": "",
"path_SRFs": "",
"path_dem_proc_srtm_90m": "",
"path_earthSunDist": "",
"path_solar_irr": "",
"path_cloud_classif": "",
"path_ECMWF_db": ""
},
"processors": {
"general_opts": { /*configuration parameters needed in all processors*/
"skip_thermal": true,
"skip_pan": true,
"sort_bands_by_cwl": true,
"conversion_type_optical": "BOA_Ref", /*'Rad' / 'TOA_Ref' / 'BOA_Ref'*/
"conversion_type_thermal": "Rad", /*'Rad' / 'Temp'*/
"scale_factor_TOARef": 10000,
"scale_factor_BOARef": 10000
},
"processors": {
"L1A_P": { /*Level 1A processing: Data import and metadata homogenization*/
"run_processor": true,
"write_output": true,
"delete_output": true,
"SZA_SAA_calculation_accurracy": "coarse", /*"coarse" or "fine"*/
"export_VZA_SZA_SAA_RAA_stats": true
"L1A_P": { /*Level 1A processing: Data import and metadata homogenization*/
"run_processor": true,
"write_output": true,
"delete_output": true,
"SZA_SAA_calculation_accurracy": "coarse", /*"coarse" or "fine"*/
"export_VZA_SZA_SAA_RAA_stats": true
},
"L1B_P": { /*Level 1B processing: calculation of geometric shift*/
"run_processor": true,
"write_output": true,
"delete_output": true,
"skip_coreg": false
},
"L1C_P": { /*Level 1C processing: atmospheric correction*/
"run_processor": true,
"write_output": true,
"delete_output": true,
"cloud_masking_algorithm": {
"Landsat-4": "FMASK",
"Landsat-5": "FMASK",
"Landsat-7": "FMASK",
"Landsat-8": "FMASK",
"Sentinel-2A": "SICOR",
"Sentinel-2B": "SICOR"
}, /*'FMASK', 'Classical Bayesian', 'SICOR'*/
"export_L1C_obj_dumps": false,
"auto_download_ecmwf": false
},
"L2A_P": { /*Level 2A processing: geometric homogenization*/
"run_processor": true,
"write_output": true,
"delete_output": true
},
"L2B_P": { /*Level 2B processing: spectral homogenization*/
"run_processor": true,
"write_output": true,
"delete_output": false
},
"L2C_P": { /*Level 2C processing: accurracy assessment and MGRS tiling*/
"run_processor": true,
"write_output": true,
"delete_output": false
}
},
"L1B_P": { /*Level 1B processing: calculation of geometric shift*/
"run_processor": true,
"write_output": true,
"delete_output": true,
"skip_coreg": false
},
"L1C_P": { /*Level 1C processing: atmospheric correction*/
"run_processor": true,
"write_output": true,
"delete_output": true,
"cloud_masking_algorithm": {
"Landsat-4": "FMASK",
"Landsat-5": "FMASK",
"Landsat-7": "FMASK",
"Landsat-8": "FMASK",
"Sentinel-2A": "SICOR",
"Sentinel-2B": "SICOR"
}, /*'FMASK', 'Classical Bayesian', 'SICOR'*/
"export_L1C_obj_dumps": false,
"scale_factor_errors_ac": 255,
"auto_download_ecmwf": false
},
"L2A_P": { /*Level 2A processing: geometric homogenization*/
"run_processor": true,
"write_output": true,
"delete_output": true,
"align_coord_grids": true, /*allows to force deactivation of image resampling*/
"match_gsd": true
},
"L2B_P": { /*Level 2B processing: spectral homogenization*/
"run_processor": true,
"write_output": true,
"delete_output": false
},
"L2C_P": {
/*Level 2C processing: accurracy assessment and MGRS tiling*/
"run_processor": true,
"write_output": true,
"delete_output": false
}
},
"usecase": {
"skip_thermal": true,
"skip_pan": true,
"sort_bands_by_cwl": true,
"conversion_type_optical": "BOA_Ref", /*'Rad' / 'TOA_Ref' / 'BOA_Ref'*/
"conversion_type_thermal": "Rad", /*'Rad' / 'Temp'*/
"scale_factor_TOARef": 10000,
"scale_factor_BOARef": 10000,
"scale_factor_errors_ac": 255,
"virtual_sensor_id": "None", /*"None": use WebApp input; 1: Landsat-8, 10: Sentinel-2A 10m*/
"virtual_sensor_name": "None", /*"None": use WebApp input*/
"virtual_sensor_id": 10, /*"None": use WebApp input; 1: Landsat-8, 10: Sentinel-2A 10m*/
"datasetid_spatial_ref": "None", /*"None": use WebApp input*/
"datasetid_spectral_ref": 249, /*249=Sentinel-2A*/
"target_CWL": [
......@@ -108,8 +109,6 @@
], /*target sensor x-coordinate-grid. e.g. [15, 45]*/
"spatial_ref_gridy": [
], /*target sensor y-coordinate-grid. e.g. [15, 45]*/
"align_coord_grids": true, /*allows to force deactivation of image resampling*/
"match_gsd": true
] /*target sensor y-coordinate-grid. e.g. [15, 45]*/
}
}
gms_preprocessing/options_schema.py 0 → 100644
View file @ a16e19c1
"""Definition of gms options schema (as used by cerberus library)."""
gms_schema = {
}
tests/test_config.py
View file @ a16e19c1
......@@ -11,8 +11,9 @@ Tests for gms_preprocessing.config
import os
from unittest import TestCase
from gms_preprocessing.config import get_options
from gms_preprocessing import __path__
from gms_preprocessing.config import get_options
from gms_preprocessing.config import JobConfig
class Test_get_options(TestCase):
......@@ -23,3 +24,13 @@ class Test_get_options(TestCase):
def test_target_is_file_validation(self):
opts_dict = get_options(os.path.join(__path__[0], 'options_default.json'))
self.assertIsInstance(opts_dict, dict)
class Test_JobConfig(TestCase):
def setUp(self):
self.jobID = 26186662
self.db_host = 'localhost'
def test(self):
cfg = JobConfig(self.jobID, self.db_host)
print(cfg)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment