Skip to content
GitLab
Commit 81770264 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Added function to get jsonable dict from config. Revised options schema. Moved... 

Added function to get jsonable dict from config. Revised options schema. Moved config and options to new submodule 'options'.
parent cd5315eb
Expand all Hide whitespace changes
Inline Side-by-side
gms_preprocessing/__init__.py
View file @ 81770264
......@@ -8,7 +8,7 @@ from . import algorithms # noqa: E402
from . import io # noqa: E402
from . import misc # noqa: E402
from . import processing # noqa: E402
from . import config # noqa: E402
from .options.config import set_config # noqa: E402
from .processing.process_controller import process_controller # noqa: E402
__author__ = """Daniel Scheffler"""
......
gms_preprocessing/options_default.json → gms_preprocessing/options/options_default.json
View file @ 81770264
{
"exec_mode": "Python", /*"Python" or "Flink"*/
"db_host": "localhost",
"CPUs": "None", /*number of CPU cores to be used for processing (default: "None" -> use all available)*/
"allow_subMultiprocessing": true, /*allow multiprocessing within workers*/
"disable_exception_handler": false, /*enable/disable automatic handling of unexpected exceptions*/
"log_level": "INFO", /*the logging level to be used (choices: 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL';*/
"tiling_block_size_XY": [
2048,
2048
], /*X/Y block size to be used for any tiling process*/
"is_test": false, /*whether the current job represents a software test job (run by a test runner) or not*/
"profiling": false, /*enable/disable code profiling*/
"benchmark_global": false, /*enable/disable benchmark of the whole processing pipeline*/
"global_opts": {
"exec_mode": "Python", /*"Python" or "Flink"*/
"db_host": "localhost",
"CPUs": "None", /*number of CPU cores to be used for processing (default: "None" -> use all available)*/
"allow_subMultiprocessing": true, /*allow multiprocessing within workers*/
"disable_exception_handler": false, /*enable/disable automatic handling of unexpected exceptions*/
"log_level": "INFO", /*the logging level to be used (choices: 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL';*/
"tiling_block_size_XY": [
2048,
2048
], /*X/Y block size to be used for any tiling process*/
"is_test": false, /*whether the current job represents a software test job (run by a test runner) or not*/
"profiling": false, /*enable/disable code profiling*/
"benchmark_global": false /*enable/disable benchmark of the whole processing pipeline*/
},
"paths": {
"path_fileserver": "",
"path_archive": "", /*input path where downloaded data are stored*/
......@@ -32,6 +35,7 @@
},
"processors": {
"general_opts": { /*configuration parameters needed in all processors*/
"skip_thermal": true,
"skip_pan": true,
......@@ -41,7 +45,8 @@
"scale_factor_TOARef": 10000,
"scale_factor_BOARef": 10000
},
"L1A_P": { /*Level 1A processing: Data import and metadata homogenization*/
"L1A": { /*Level 1A processing: Data import and metadata homogenization*/
"run_processor": true,
"write_output": true,
"delete_output": true,
......@@ -49,13 +54,15 @@
"export_VZA_SZA_SAA_RAA_stats": true
},
"L1B_P": { /*Level 1B processing: calculation of geometric shift*/
"L1B": { /*Level 1B processing: calculation of geometric shift*/
"run_processor": true,
"write_output": true,
"delete_output": true,
"skip_coreg": false
},
"L1C_P": { /*Level 1C processing: atmospheric correction*/
"L1C": { /*Level 1C processing: atmospheric correction*/
"run_processor": true,
"write_output": true,
"delete_output": true,
......@@ -71,19 +78,22 @@
"scale_factor_errors_ac": 255,
"auto_download_ecmwf": false
},
"L2A_P": { /*Level 2A processing: geometric homogenization*/
"L2A": { /*Level 2A processing: geometric homogenization*/
"run_processor": true,
"write_output": true,
"delete_output": true,
"align_coord_grids": true, /*allows to force deactivation of image resampling*/
"match_gsd": true
},
"L2B_P": { /*Level 2B processing: spectral homogenization*/
"L2B": { /*Level 2B processing: spectral homogenization*/
"run_processor": true,
"write_output": true,
"delete_output": false
},
"L2C_P": {
"L2C": {
/*Level 2C processing: accurracy assessment and MGRS tiling*/
"run_processor": true,
"write_output": true,
......
gms_preprocessing/options_schema.py → gms_preprocessing/options/options_schema.py
View file @ 81770264
"""Definition of gms options schema (as used by cerberus library)."""
gms_schema = dict(
exec_mode=dict(type='string', required=True, allowed=['Python', 'Flink']),
db_host=dict(type='string', required=True),
CPUs=dict(type='integer', required=True, nullable=True),
allow_subMultiprocessing=dict(type='boolean', required=True),
disable_exception_handler=dict(type='boolean', required=True),
log_level=dict(type='string', required=True, allowed=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']),
tiling_block_size_XY=dict(type='list', required=True, schema=dict(type="integer"), minlength=2, maxlength=2),
is_test=dict(type='boolean', required=True),
profiling=dict(type='boolean', required=True),
benchmark_global=dict(type='boolean', required=True),
global_opts=dict(
type='dict', required=True,
schema=dict(
exec_mode=dict(type='string', required=True, allowed=['Python', 'Flink']),
db_host=dict(type='string', required=True),
CPUs=dict(type='integer', required=True, nullable=True),
allow_subMultiprocessing=dict(type='boolean', required=True),
disable_exception_handler=dict(type='boolean', required=True),
log_level=dict(type='string', required=True, allowed=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']),
tiling_block_size_XY=dict(type='list', required=True, schema=dict(type="integer"), minlength=2,
maxlength=2),
is_test=dict(type='boolean', required=True),
profiling=dict(type='boolean', required=True),
benchmark_global=dict(type='boolean', required=True),
)),
paths=dict(
type='dict', required=True,
schema=dict(
......@@ -42,20 +47,20 @@ gms_schema = dict(
scale_factor_TOARef=dict(type='integer', required=True),
scale_factor_BOARef=dict(type='integer', required=True),
)),
L1A_P=dict(type='dict', required=True, schema=dict(
L1A=dict(type='dict', required=True, schema=dict(
run_processor=dict(type='boolean', required=True),
write_output=dict(type='boolean', required=True),
delete_output=dict(type='boolean', required=True),
SZA_SAA_calculation_accurracy=dict(type='string', required=False, allowed=['coarse', 'fine']),
export_VZA_SZA_SAA_RAA_stats=dict(type='boolean', required=True),
)),
L1B_P=dict(type='dict', required=True, schema=dict(
L1B=dict(type='dict', required=True, schema=dict(
run_processor=dict(type='boolean', required=True),
write_output=dict(type='boolean', required=True),
delete_output=dict(type='boolean', required=True),
skip_coreg=dict(type='boolean', required=True),
)),
L1C_P=dict(type='dict', required=True, schema=dict(
L1C=dict(type='dict', required=True, schema=dict(
run_processor=dict(type='boolean', required=True),
write_output=dict(type='boolean', required=True),
delete_output=dict(type='boolean', required=True),
......@@ -71,19 +76,19 @@ gms_schema = dict(
scale_factor_errors_ac=dict(type='integer', required=True),
auto_download_ecmwf=dict(type='boolean', required=True),
)),
L2A_P=dict(type='dict', required=True, schema=dict(
L2A=dict(type='dict', required=True, schema=dict(
run_processor=dict(type='boolean', required=True),
write_output=dict(type='boolean', required=True),
delete_output=dict(type='boolean', required=True),
align_coord_grids=dict(type='boolean', required=True),
match_gsd=dict(type='boolean', required=True),
)),
L2B_P=dict(type='dict', required=True, schema=dict(
L2B=dict(type='dict', required=True, schema=dict(
run_processor=dict(type='boolean', required=True),
write_output=dict(type='boolean', required=True),
delete_output=dict(type='boolean', required=True),
)),
L2C_P=dict(type='dict', required=True, schema=dict(
L2C=dict(type='dict', required=True, schema=dict(
run_processor=dict(type='boolean', required=True),
write_output=dict(type='boolean', required=True),
delete_output=dict(type='boolean', required=True),
......
tests/test_config.py
View file @ 81770264
......@@ -10,11 +10,11 @@ Tests for gms_preprocessing.config
import os
from unittest import TestCase
from json import JSONDecodeError
from json import JSONDecodeError, dumps
from gms_preprocessing import __path__
from gms_preprocessing.config import get_options
from gms_preprocessing.config import JobConfig
from gms_preprocessing.options.config import get_options
from gms_preprocessing.options.config import JobConfig
class Test_get_options(TestCase):
......@@ -57,7 +57,7 @@ class Test_JobConfig(TestCase):
JobConfig(self.jobID, self.db_host, json_config=cfg)
def test_jsonconfig_str_schema_violation(self):
cfg = '{"exec_mode": "badvalue"}'
cfg = '{"global_opts": {"exec_mode": "badvalue"}}'
with self.assertRaises(ValueError):
JobConfig(self.jobID, self.db_host, json_config=cfg)
......@@ -65,3 +65,11 @@ class Test_JobConfig(TestCase):
cfg = os.path.join(__path__[0], 'options_default.json')
cfg = JobConfig(self.jobID, self.db_host, json_config=cfg)
self.assertIsInstance(cfg, JobConfig)
def test_to_jsonable_dict(self):
cfg = JobConfig(self.jobID, self.db_host)
jsonable_dict = cfg.to_jsonable_dict()
self.assertIsInstance(cfg.to_jsonable_dict(), dict)
# test if dict is jsonable
dumps(jsonable_dict)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment