Commit 3e1d28bc authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Added function to get jsonable dict from config. Revised options schema. Moved...

Added function to get jsonable dict from config. Revised options schema. Moved config and options to new submodule 'options'.


Former-commit-id: 81770264
parent 81d2763b
...@@ -8,7 +8,7 @@ from . import algorithms # noqa: E402 ...@@ -8,7 +8,7 @@ from . import algorithms # noqa: E402
from . import io # noqa: E402 from . import io # noqa: E402
from . import misc # noqa: E402 from . import misc # noqa: E402
from . import processing # noqa: E402 from . import processing # noqa: E402
from . import config # noqa: E402 from .options.config import set_config # noqa: E402
from .processing.process_controller import process_controller # noqa: E402 from .processing.process_controller import process_controller # noqa: E402
__author__ = """Daniel Scheffler""" __author__ = """Daniel Scheffler"""
......
{ {
"exec_mode": "Python", /*"Python" or "Flink"*/ "global_opts": {
"db_host": "localhost", "exec_mode": "Python", /*"Python" or "Flink"*/
"CPUs": "None", /*number of CPU cores to be used for processing (default: "None" -> use all available)*/ "db_host": "localhost",
"allow_subMultiprocessing": true, /*allow multiprocessing within workers*/ "CPUs": "None", /*number of CPU cores to be used for processing (default: "None" -> use all available)*/
"disable_exception_handler": false, /*enable/disable automatic handling of unexpected exceptions*/ "allow_subMultiprocessing": true, /*allow multiprocessing within workers*/
"log_level": "INFO", /*the logging level to be used (choices: 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL';*/ "disable_exception_handler": false, /*enable/disable automatic handling of unexpected exceptions*/
"tiling_block_size_XY": [ "log_level": "INFO", /*the logging level to be used (choices: 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL';*/
2048, "tiling_block_size_XY": [
2048 2048,
], /*X/Y block size to be used for any tiling process*/ 2048
"is_test": false, /*whether the current job represents a software test job (run by a test runner) or not*/ ], /*X/Y block size to be used for any tiling process*/
"profiling": false, /*enable/disable code profiling*/ "is_test": false, /*whether the current job represents a software test job (run by a test runner) or not*/
"benchmark_global": false, /*enable/disable benchmark of the whole processing pipeline*/ "profiling": false, /*enable/disable code profiling*/
"benchmark_global": false /*enable/disable benchmark of the whole processing pipeline*/
},
"paths": { "paths": {
"path_fileserver": "", "path_fileserver": "",
"path_archive": "", /*input path where downloaded data are stored*/ "path_archive": "", /*input path where downloaded data are stored*/
...@@ -32,6 +35,7 @@ ...@@ -32,6 +35,7 @@
}, },
"processors": { "processors": {
"general_opts": { /*configuration parameters needed in all processors*/ "general_opts": { /*configuration parameters needed in all processors*/
"skip_thermal": true, "skip_thermal": true,
"skip_pan": true, "skip_pan": true,
...@@ -41,7 +45,8 @@ ...@@ -41,7 +45,8 @@
"scale_factor_TOARef": 10000, "scale_factor_TOARef": 10000,
"scale_factor_BOARef": 10000 "scale_factor_BOARef": 10000
}, },
"L1A_P": { /*Level 1A processing: Data import and metadata homogenization*/
"L1A": { /*Level 1A processing: Data import and metadata homogenization*/
"run_processor": true, "run_processor": true,
"write_output": true, "write_output": true,
"delete_output": true, "delete_output": true,
...@@ -49,13 +54,15 @@ ...@@ -49,13 +54,15 @@
"export_VZA_SZA_SAA_RAA_stats": true "export_VZA_SZA_SAA_RAA_stats": true
}, },
"L1B_P": { /*Level 1B processing: calculation of geometric shift*/
"L1B": { /*Level 1B processing: calculation of geometric shift*/
"run_processor": true, "run_processor": true,
"write_output": true, "write_output": true,
"delete_output": true, "delete_output": true,
"skip_coreg": false "skip_coreg": false
}, },
"L1C_P": { /*Level 1C processing: atmospheric correction*/
"L1C": { /*Level 1C processing: atmospheric correction*/
"run_processor": true, "run_processor": true,
"write_output": true, "write_output": true,
"delete_output": true, "delete_output": true,
...@@ -71,19 +78,22 @@ ...@@ -71,19 +78,22 @@
"scale_factor_errors_ac": 255, "scale_factor_errors_ac": 255,
"auto_download_ecmwf": false "auto_download_ecmwf": false
}, },
"L2A_P": { /*Level 2A processing: geometric homogenization*/
"L2A": { /*Level 2A processing: geometric homogenization*/
"run_processor": true, "run_processor": true,
"write_output": true, "write_output": true,
"delete_output": true, "delete_output": true,
"align_coord_grids": true, /*allows to force deactivation of image resampling*/ "align_coord_grids": true, /*allows to force deactivation of image resampling*/
"match_gsd": true "match_gsd": true
}, },
"L2B_P": { /*Level 2B processing: spectral homogenization*/
"L2B": { /*Level 2B processing: spectral homogenization*/
"run_processor": true, "run_processor": true,
"write_output": true, "write_output": true,
"delete_output": false "delete_output": false
}, },
"L2C_P": {
"L2C": {
/*Level 2C processing: accurracy assessment and MGRS tiling*/ /*Level 2C processing: accurracy assessment and MGRS tiling*/
"run_processor": true, "run_processor": true,
"write_output": true, "write_output": true,
......
"""Definition of gms options schema (as used by cerberus library).""" """Definition of gms options schema (as used by cerberus library)."""
gms_schema = dict( gms_schema = dict(
exec_mode=dict(type='string', required=True, allowed=['Python', 'Flink']), global_opts=dict(
db_host=dict(type='string', required=True), type='dict', required=True,
CPUs=dict(type='integer', required=True, nullable=True), schema=dict(
allow_subMultiprocessing=dict(type='boolean', required=True), exec_mode=dict(type='string', required=True, allowed=['Python', 'Flink']),
disable_exception_handler=dict(type='boolean', required=True), db_host=dict(type='string', required=True),
log_level=dict(type='string', required=True, allowed=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']), CPUs=dict(type='integer', required=True, nullable=True),
tiling_block_size_XY=dict(type='list', required=True, schema=dict(type="integer"), minlength=2, maxlength=2), allow_subMultiprocessing=dict(type='boolean', required=True),
is_test=dict(type='boolean', required=True), disable_exception_handler=dict(type='boolean', required=True),
profiling=dict(type='boolean', required=True), log_level=dict(type='string', required=True, allowed=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']),
benchmark_global=dict(type='boolean', required=True), tiling_block_size_XY=dict(type='list', required=True, schema=dict(type="integer"), minlength=2,
maxlength=2),
is_test=dict(type='boolean', required=True),
profiling=dict(type='boolean', required=True),
benchmark_global=dict(type='boolean', required=True),
)),
paths=dict( paths=dict(
type='dict', required=True, type='dict', required=True,
schema=dict( schema=dict(
...@@ -42,20 +47,20 @@ gms_schema = dict( ...@@ -42,20 +47,20 @@ gms_schema = dict(
scale_factor_TOARef=dict(type='integer', required=True), scale_factor_TOARef=dict(type='integer', required=True),
scale_factor_BOARef=dict(type='integer', required=True), scale_factor_BOARef=dict(type='integer', required=True),
)), )),
L1A_P=dict(type='dict', required=True, schema=dict( L1A=dict(type='dict', required=True, schema=dict(
run_processor=dict(type='boolean', required=True), run_processor=dict(type='boolean', required=True),
write_output=dict(type='boolean', required=True), write_output=dict(type='boolean', required=True),
delete_output=dict(type='boolean', required=True), delete_output=dict(type='boolean', required=True),
SZA_SAA_calculation_accurracy=dict(type='string', required=False, allowed=['coarse', 'fine']), SZA_SAA_calculation_accurracy=dict(type='string', required=False, allowed=['coarse', 'fine']),
export_VZA_SZA_SAA_RAA_stats=dict(type='boolean', required=True), export_VZA_SZA_SAA_RAA_stats=dict(type='boolean', required=True),
)), )),
L1B_P=dict(type='dict', required=True, schema=dict( L1B=dict(type='dict', required=True, schema=dict(
run_processor=dict(type='boolean', required=True), run_processor=dict(type='boolean', required=True),
write_output=dict(type='boolean', required=True), write_output=dict(type='boolean', required=True),
delete_output=dict(type='boolean', required=True), delete_output=dict(type='boolean', required=True),
skip_coreg=dict(type='boolean', required=True), skip_coreg=dict(type='boolean', required=True),
)), )),
L1C_P=dict(type='dict', required=True, schema=dict( L1C=dict(type='dict', required=True, schema=dict(
run_processor=dict(type='boolean', required=True), run_processor=dict(type='boolean', required=True),
write_output=dict(type='boolean', required=True), write_output=dict(type='boolean', required=True),
delete_output=dict(type='boolean', required=True), delete_output=dict(type='boolean', required=True),
...@@ -71,19 +76,19 @@ gms_schema = dict( ...@@ -71,19 +76,19 @@ gms_schema = dict(
scale_factor_errors_ac=dict(type='integer', required=True), scale_factor_errors_ac=dict(type='integer', required=True),
auto_download_ecmwf=dict(type='boolean', required=True), auto_download_ecmwf=dict(type='boolean', required=True),
)), )),
L2A_P=dict(type='dict', required=True, schema=dict( L2A=dict(type='dict', required=True, schema=dict(
run_processor=dict(type='boolean', required=True), run_processor=dict(type='boolean', required=True),
write_output=dict(type='boolean', required=True), write_output=dict(type='boolean', required=True),
delete_output=dict(type='boolean', required=True), delete_output=dict(type='boolean', required=True),
align_coord_grids=dict(type='boolean', required=True), align_coord_grids=dict(type='boolean', required=True),
match_gsd=dict(type='boolean', required=True), match_gsd=dict(type='boolean', required=True),
)), )),
L2B_P=dict(type='dict', required=True, schema=dict( L2B=dict(type='dict', required=True, schema=dict(
run_processor=dict(type='boolean', required=True), run_processor=dict(type='boolean', required=True),
write_output=dict(type='boolean', required=True), write_output=dict(type='boolean', required=True),
delete_output=dict(type='boolean', required=True), delete_output=dict(type='boolean', required=True),
)), )),
L2C_P=dict(type='dict', required=True, schema=dict( L2C=dict(type='dict', required=True, schema=dict(
run_processor=dict(type='boolean', required=True), run_processor=dict(type='boolean', required=True),
write_output=dict(type='boolean', required=True), write_output=dict(type='boolean', required=True),
delete_output=dict(type='boolean', required=True), delete_output=dict(type='boolean', required=True),
......
...@@ -10,11 +10,11 @@ Tests for gms_preprocessing.config ...@@ -10,11 +10,11 @@ Tests for gms_preprocessing.config
import os import os
from unittest import TestCase from unittest import TestCase
from json import JSONDecodeError from json import JSONDecodeError, dumps
from gms_preprocessing import __path__ from gms_preprocessing import __path__
from gms_preprocessing.config import get_options from gms_preprocessing.options.config import get_options
from gms_preprocessing.config import JobConfig from gms_preprocessing.options.config import JobConfig
class Test_get_options(TestCase): class Test_get_options(TestCase):
...@@ -57,7 +57,7 @@ class Test_JobConfig(TestCase): ...@@ -57,7 +57,7 @@ class Test_JobConfig(TestCase):
JobConfig(self.jobID, self.db_host, json_config=cfg) JobConfig(self.jobID, self.db_host, json_config=cfg)
def test_jsonconfig_str_schema_violation(self): def test_jsonconfig_str_schema_violation(self):
cfg = '{"exec_mode": "badvalue"}' cfg = '{"global_opts": {"exec_mode": "badvalue"}}'
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
JobConfig(self.jobID, self.db_host, json_config=cfg) JobConfig(self.jobID, self.db_host, json_config=cfg)
...@@ -65,3 +65,11 @@ class Test_JobConfig(TestCase): ...@@ -65,3 +65,11 @@ class Test_JobConfig(TestCase):
cfg = os.path.join(__path__[0], 'options_default.json') cfg = os.path.join(__path__[0], 'options_default.json')
cfg = JobConfig(self.jobID, self.db_host, json_config=cfg) cfg = JobConfig(self.jobID, self.db_host, json_config=cfg)
self.assertIsInstance(cfg, JobConfig) self.assertIsInstance(cfg, JobConfig)
def test_to_jsonable_dict(self):
cfg = JobConfig(self.jobID, self.db_host)
jsonable_dict = cfg.to_jsonable_dict()
self.assertIsInstance(cfg.to_jsonable_dict(), dict)
# test if dict is jsonable
dumps(jsonable_dict)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment