Skip to content
GitLab
Commit ce422a51 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Fixed file path issues. Bugfixes. 


Former-commit-id: 81889e71
Former-commit-id: 4a55a1b8
parent e2b11a34
Hide whitespace changes
Inline Side-by-side
gms_preprocessing/__init__.py
View file @ ce422a51
......@@ -9,6 +9,7 @@ from . import io # noqa: E402
from . import misc # noqa: E402
from . import processing # noqa: E402
from . import options # noqa: E402
from .options import config # noqa: E402
from .options.config import set_config # noqa: E402
from .processing.process_controller import process_controller # noqa: E402
......@@ -20,6 +21,7 @@ __all__ = ['algorithms',
'io',
'misc',
'processing',
'config' # only to keep compatibility with HU-INF codes
'options',
'set_config',
'process_controller',
......
gms_preprocessing/options/config.py
View file @ ce422a51
......@@ -43,8 +43,8 @@ class GMS_configuration(object):
GMS_config = GMS_configuration()
path_options_default = os.path.join(os.path.dirname(pkgutil.get_loader("gms_preprocessing").path), 'options',
'options_default.json')
path_gmslib = os.path.dirname(pkgutil.get_loader("gms_preprocessing").path)
path_options_default = os.path.join(path_gmslib, 'options', 'options_default.json')
def set_config(job_ID, exec_mode='Python', db_host='localhost', reset=False, json_config='', exec_L1AP=None,
......@@ -170,10 +170,10 @@ class JobConfig(object):
json_paths = json_opts['paths'] # type: dict
self.path_spatIdxSrv = self.DB_config_table['path_spatial_index_mediator_server']
self.path_tempdir = self.DB_config_table['path_tempdir']
self.path_ac_tables = self.DB_config_table['path_ac_tables']
self.path_SNR_models = self.DB_config_table['path_SNR_models']
self.path_dem_proc_srtm_90m = self.DB_config_table['path_dem_proc_srtm_90m']
self.path_tempdir = self.absP(self.DB_config_table['path_tempdir'])
self.path_ac_tables = self.absP(self.DB_config_table['path_ac_tables'])
self.path_SNR_models = self.absP(self.DB_config_table['path_SNR_models'])
self.path_dem_proc_srtm_90m = self.absP(self.DB_config_table['path_dem_proc_srtm_90m'])
if not self.is_test:
# normal mode
......@@ -191,33 +191,35 @@ class JobConfig(object):
gp('path_procdata_MGRS', json_paths['path_procdata_MGRS'],
fallback=self.joinP(self.path_fileserver, self.DB_config_table['foldername_procdata_MGRS']))
self.path_earthSunDist = self.DB_config_table['path_earthSunDist']
self.path_SRFs = self.DB_config_table['path_SRFs']
self.path_cloud_classif = self.DB_config_table['path_cloud_classif']
self.path_solar_irr = self.DB_config_table['path_solar_irr']
self.path_ECMWF_db = self.DB_config_table['path_ECMWF_db']
self.path_earthSunDist = self.absP(self.DB_config_table['path_earthSunDist'])
self.path_SRFs = self.absP(self.DB_config_table['path_SRFs'])
self.path_cloud_classif = self.absP(self.DB_config_table['path_cloud_classif'])
self.path_solar_irr = self.absP(self.DB_config_table['path_solar_irr'])
self.path_ECMWF_db = self.absP(self.DB_config_table['path_ECMWF_db'])
self.path_benchmarks = \
gp('path_benchmarks', json_paths['path_benchmarks'],
fallback=self.DB_config_table['path_benchmarks'])
fallback=self.absP(self.DB_config_table['path_benchmarks']))
self.path_job_logs = \
gp('path_job_logs', json_paths['path_job_logs'], fallback=self.DB_config_table['path_job_logs'])
gp('path_job_logs', json_paths['path_job_logs'],
fallback=self.absP(self.DB_config_table['path_job_logs']))
else:
# software test mode, the repository should be self-contained -> use only relative paths
self.path_fileserver = self.absP('../../tests/data/')
self.path_archive = self.absP('../../tests/data/archive_data/')
self.path_procdata_scenes = self.absP('../../tests/data/output_scenes/')
self.path_procdata_MGRS = self.absP('../../tests/data/output_mgrs_tiles/')
self.path_earthSunDist = self.absP('../database/earth_sun_distance/Earth_Sun_distances_per_day_edited.csv')
self.path_SRFs = self.absP('../database/srf/')
self.path_cloud_classif = self.absP('../database/cloud_classifier/')
self.path_solar_irr = self.absP(
'../database/solar_irradiance/SUNp1fontenla__350-2500nm_@0.1nm_converted.txt')
self.path_ECMWF_db = self.absP('../../tests/data/processed_ECMWF/')
self.path_benchmarks = self.absP('../benchmarks/')
self.path_job_logs = self.absP('../logs/job_logs/')
self.path_fileserver = self.joinP(path_gmslib, '..', 'tests', 'data')
self.path_archive = self.joinP(path_gmslib, '..', 'tests', 'data', 'archive_data')
self.path_procdata_scenes = self.joinP(path_gmslib, '..', 'tests', 'data', 'output_scenes')
self.path_procdata_MGRS = self.joinP(path_gmslib, '..', 'tests', 'data', 'output_mgrs_tiles')
self.path_earthSunDist = self.joinP(path_gmslib, 'database', 'earth_sun_distance',
'Earth_Sun_distances_per_day_edited.csv')
self.path_SRFs = self.joinP(path_gmslib, 'database', 'srf')
self.path_cloud_classif = self.joinP(path_gmslib, 'database', 'cloud_classifier')
self.path_solar_irr = self.joinP(path_gmslib, 'database', 'solar_irradiance',
'SUNp1fontenla__350-2500nm_@0.1nm_converted.txt')
self.path_ECMWF_db = self.joinP(path_gmslib, '..', 'tests', 'data', 'processed_ECMWF')
self.path_benchmarks = self.joinP(path_gmslib, '..', 'tests', 'data', 'benchmarks')
self.path_job_logs = self.joinP(path_gmslib, 'logs', 'job_logs')
###########################
# processor configuration #
......@@ -251,7 +253,7 @@ class JobConfig(object):
self.SZA_SAA_calculation_accurracy = \
gp('SZA_SAA_calculation_accurracy', json_processors['L1A']['SZA_SAA_calculation_accurracy'])
self.export_VZA_SZA_SAA_RAA_stats = \
gp('export_VZA_SZA_SAA_RAA_stats', json_processors['L1A']['SZA_SAA_calculation_accurracy'])
gp('export_VZA_SZA_SAA_RAA_stats', json_processors['L1A']['export_VZA_SZA_SAA_RAA_stats'])
# L1B
self.exec_L1BP = gp('exec_L1BP', [
......
gms_preprocessing/options/options_default.json
View file @ ce422a51
......@@ -3,7 +3,7 @@
"exec_mode": "Python", /*"Python" or "Flink"*/
"db_host": "localhost",
"CPUs": "None", /*number of CPU cores to be used for processing (default: "None" -> use all available)*/
"allow_subMultiprocessing": true, /*allow multiprocessing within workers*/
"allow_subMultiprocessing": true, /*allow multiprocessing within multiprocessing workers*/
"disable_exception_handler": false, /*enable/disable automatic handling of unexpected exceptions*/
"log_level": "INFO", /*the logging level to be used (choices: 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL';*/
"tiling_block_size_XY": [
......@@ -100,7 +100,9 @@
"delete_output": false
}
},
"usecase": {
"usecase": { /*NOTE: These options will not respected in the WebApp! Use the WebApp GUI instead.*/
"virtual_sensor_id": 10, /*"None": use WebApp input; 1: Landsat-8, 10: Sentinel-2A 10m*/
"datasetid_spatial_ref": "None", /*"None": use WebApp input*/
"datasetid_spectral_ref": 249, /*249=Sentinel-2A*/
......
gms_preprocessing/processing/process_controller.py
View file @ ce422a51
......@@ -79,7 +79,8 @@ class process_controller(object):
self.summary_quick = None
# set GMS configuration
set_config(job_ID=job_ID, exec_mode=exec_mode, db_host=db_host, reset=True, **job_config_kwargs)
set_config(job_ID=job_ID, exec_mode=exec_mode, db_host=db_host, reset=True,
**job_config_kwargs if job_config_kwargs else {})
self.config = GMS_config
# check environment
......
tests/test_spechomo_classifier.py
View file @ ce422a51
......@@ -12,13 +12,12 @@ import unittest
import os
import numpy as np
from gms_preprocessing import __file__ # noqa E402 module level import not at top of file
from gms_preprocessing.options.config import set_config # noqa E402 module level import not at top of file
from gms_preprocessing import __path__ # noqa E402 module level import not at top of file
from gms_preprocessing import set_config # noqa E402 module level import not at top of file
from gms_preprocessing.algorithms.L2B_P import SpecHomo_Classifier # noqa E402 module level import not at top of file
testdata = os.path.join(os.path.dirname(__file__),
'../tests/data/hy_spec_data/Bavaria_farmland_LMU_Hyspex_subset.bsq')
testdata = os.path.join(__path__[0], '../tests/data/hy_spec_data/Bavaria_farmland_LMU_Hyspex_subset.bsq')
class Test_SpecHomo_Classifier(unittest.TestCase):
......@@ -27,7 +26,7 @@ class Test_SpecHomo_Classifier(unittest.TestCase):
@classmethod
def setUpClass(cls):
# Testjob Landsat-8
set_config(exec_mode='Python', job_ID=26186196, db_host='geoms', reset=True)
set_config(exec_mode='Python', job_ID=26186196, db_host='geoms', reset=True, is_test=True)
cls.SHC = SpecHomo_Classifier([testdata, testdata, ], v=False)
def test_generate_reference_cube_L8(self):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment