Added parser functions for options_default.json. Revised options_default.json....

Added parser functions for options_default.json. Revised options_default.json. Added test_config.py.


Former-commit-id: 47007049

gms_preprocessing/config.py

@@ -14,6 +14,9 @@ import psycopg2.extras
 from collections import OrderedDict
 import multiprocessing
 from inspect import getargvalues, stack, getfullargspec, signature, _empty
+import json
+from jsmin import jsmin
+from cerberus import Validator
 
 __author__ = 'Daniel Scheffler'
 
@@ -559,3 +562,63 @@ def is_GMSConfig_available():
             return True
     except (EnvironmentError, OSError):
         return False
+
+
+gms_schema = None
+
+
+def json_to_python(dd):
+    def is_number(s):
+        try:
+            float(s)
+            return True
+        except ValueError:
+            return False
+
+    if type(dd) is dict:
+        return {json_to_python(k): json_to_python(v) for k, v in dd.items()}
+    elif type(dd) is list:
+        return [json_to_python(v) for v in dd]
+    else:
+        if dd == "None":
+            return None
+        if dd == "slice(None, None, None)":
+            return slice(None)
+        if dd in [True, "true"]:
+            return True
+        if dd in [False, "false"]:
+            return False
+        if is_number(dd):
+            try:
+                if str(int(dd)) != str(float(dd)):
+                    return int(dd)
+                else:
+                    return float(dd)
+            except ValueError:
+                return float(dd)
+        else:
+            return dd
+
+
+def get_options(target, validation=True):
+    """
+    return dictionary will all options
+    :param validation: True / False, whether to validate options read from files ot not
+    :param target: if path to file, then json is used to load, otherwise the default template
+    is used
+    :return: dictionary with options
+    """
+
+    if os.path.isfile(target):
+        with open(target, "r") as fl:
+            options = json_to_python(json.loads(jsmin(fl.read())))
+
+        if validation is True:
+            raise NotImplementedError("Currently options cannot be validated yet.")
+            vv = Validator(allow_unknown=True, schema=gms_schema)
+            if vv.validate(document=options) is False:
+                raise ValueError("Options is malformed: %s" % str(vv.errors))
+
+        return options
+    else:
+        raise FileNotFoundError("target: %s is not a valid file path" % target)

gms_preprocessing/options_default.json

@@ -2,7 +2,7 @@
     "job": {
         "exec_mode": "Python",  /*"Python" or "Flink"*/
         "db_host": "localhost",
-        "CPUs": -1,  /*number of CPU cores to be used for processing (default: -1 -> use all available)*/
+        "CPUs": "None",  /*number of CPU cores to be used for processing (default: "None" -> use all available)*/
         "allow_subMultiprocessing": true,  /*allow multiprocessing within workers*/
         "disable_exception_handler": false,  /*enable/disable automatic handling of unexpected exceptions*/
         "log_level": "INFO",  /*the logging level to be used (choices: 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL';*/
@@ -89,9 +89,9 @@
         "scale_factor_TOARef": 10000,
         "scale_factor_BOARef": 10000,
         "scale_factor_errors_ac": 255,
-        "virtual_sensor_id": -1,   /*-1: use WebApp input; 1:  Landsat-8,  10: Sentinel-2A 10m*/
-        "virtual_sensor_name": -1, /*-1: use WebApp input*/
-        "datasetid_spatial_ref": -1, /*-1: use WebApp input*/
+        "virtual_sensor_id": "None",   /*"None": use WebApp input; 1:  Landsat-8,  10: Sentinel-2A 10m*/
+        "virtual_sensor_name": "None", /*"None": use WebApp input*/
+        "datasetid_spatial_ref": "None", /*"None": use WebApp input*/
         "datasetid_spectral_ref": 249,   /*249=Sentinel-2A*/
         "target_CWL": [
 
@@ -102,7 +102,7 @@
         "target_gsd": [
 
         ], /*X/Y pixel size of target sensor as list with two float/integer values*/
-        "target_epsg_code": -1,  /*target projection as EPSG code. -1: use projection of input data.*/
+        "target_epsg_code": "None",  /*target projection as EPSG code. "None": use projection of input data.*/
         "spatial_ref_gridx": [
 
         ],  /*target sensor x-coordinate-grid. e.g. [15, 45]*/

tests/test_config.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+test_config
+-----------
+
+Tests for gms_preprocessing.config
+"""
+
+import os
+from unittest import TestCase
+
+from gms_preprocessing.config import get_options
+from gms_preprocessing import __path__
+
+
+class Test_get_options(TestCase):
+    def test_target_is_file_no_validation(self):
+        opts_dict = get_options(os.path.join(__path__[0], 'options_default.json'), validation=False)
+        self.assertIsInstance(opts_dict, dict)
+
+    def test_target_is_file_validation(self):
+        opts_dict = get_options(os.path.join(__path__[0], 'options_default.json'))
+        self.assertIsInstance(opts_dict, dict)