Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
geomultisens
gms_preprocessing
Commits
ce422a51
Commit
ce422a51
authored
Nov 28, 2017
by
Daniel Scheffler
Browse files
Fixed file path issues. Bugfixes.
Former-commit-id:
81889e71
Former-commit-id:
4a55a1b8
parent
e2b11a34
Changes
5
Hide whitespace changes
Inline
Side-by-side
gms_preprocessing/__init__.py
View file @
ce422a51
...
...
@@ -9,6 +9,7 @@ from . import io # noqa: E402
from
.
import
misc
# noqa: E402
from
.
import
processing
# noqa: E402
from
.
import
options
# noqa: E402
from
.options
import
config
# noqa: E402
from
.options.config
import
set_config
# noqa: E402
from
.processing.process_controller
import
process_controller
# noqa: E402
...
...
@@ -20,6 +21,7 @@ __all__ = ['algorithms',
'io'
,
'misc'
,
'processing'
,
'config'
# only to keep compatibility with HU-INF codes
'options'
,
'set_config'
,
'process_controller'
,
...
...
gms_preprocessing/options/config.py
View file @
ce422a51
...
...
@@ -43,8 +43,8 @@ class GMS_configuration(object):
GMS_config
=
GMS_configuration
()
path_
options_default
=
os
.
path
.
join
(
os
.
path
.
dirname
(
pkgutil
.
get_loader
(
"gms_preprocessing"
).
path
)
,
'options'
,
'options_default.json'
)
path_
gmslib
=
os
.
path
.
dirname
(
pkgutil
.
get_loader
(
"gms_preprocessing"
).
path
)
path_options_default
=
os
.
path
.
join
(
path_gmslib
,
'options'
,
'options_default.json'
)
def
set_config
(
job_ID
,
exec_mode
=
'Python'
,
db_host
=
'localhost'
,
reset
=
False
,
json_config
=
''
,
exec_L1AP
=
None
,
...
...
@@ -170,10 +170,10 @@ class JobConfig(object):
json_paths
=
json_opts
[
'paths'
]
# type: dict
self
.
path_spatIdxSrv
=
self
.
DB_config_table
[
'path_spatial_index_mediator_server'
]
self
.
path_tempdir
=
self
.
DB_config_table
[
'path_tempdir'
]
self
.
path_ac_tables
=
self
.
DB_config_table
[
'path_ac_tables'
]
self
.
path_SNR_models
=
self
.
DB_config_table
[
'path_SNR_models'
]
self
.
path_dem_proc_srtm_90m
=
self
.
DB_config_table
[
'path_dem_proc_srtm_90m'
]
self
.
path_tempdir
=
self
.
absP
(
self
.
DB_config_table
[
'path_tempdir'
]
)
self
.
path_ac_tables
=
self
.
absP
(
self
.
DB_config_table
[
'path_ac_tables'
]
)
self
.
path_SNR_models
=
self
.
absP
(
self
.
DB_config_table
[
'path_SNR_models'
]
)
self
.
path_dem_proc_srtm_90m
=
self
.
absP
(
self
.
DB_config_table
[
'path_dem_proc_srtm_90m'
]
)
if
not
self
.
is_test
:
# normal mode
...
...
@@ -191,33 +191,35 @@ class JobConfig(object):
gp
(
'path_procdata_MGRS'
,
json_paths
[
'path_procdata_MGRS'
],
fallback
=
self
.
joinP
(
self
.
path_fileserver
,
self
.
DB_config_table
[
'foldername_procdata_MGRS'
]))
self
.
path_earthSunDist
=
self
.
DB_config_table
[
'path_earthSunDist'
]
self
.
path_SRFs
=
self
.
DB_config_table
[
'path_SRFs'
]
self
.
path_cloud_classif
=
self
.
DB_config_table
[
'path_cloud_classif'
]
self
.
path_solar_irr
=
self
.
DB_config_table
[
'path_solar_irr'
]
self
.
path_ECMWF_db
=
self
.
DB_config_table
[
'path_ECMWF_db'
]
self
.
path_earthSunDist
=
self
.
absP
(
self
.
DB_config_table
[
'path_earthSunDist'
]
)
self
.
path_SRFs
=
self
.
absP
(
self
.
DB_config_table
[
'path_SRFs'
]
)
self
.
path_cloud_classif
=
self
.
absP
(
self
.
DB_config_table
[
'path_cloud_classif'
]
)
self
.
path_solar_irr
=
self
.
absP
(
self
.
DB_config_table
[
'path_solar_irr'
]
)
self
.
path_ECMWF_db
=
self
.
absP
(
self
.
DB_config_table
[
'path_ECMWF_db'
]
)
self
.
path_benchmarks
=
\
gp
(
'path_benchmarks'
,
json_paths
[
'path_benchmarks'
],
fallback
=
self
.
DB_config_table
[
'path_benchmarks'
])
fallback
=
self
.
absP
(
self
.
DB_config_table
[
'path_benchmarks'
])
)
self
.
path_job_logs
=
\
gp
(
'path_job_logs'
,
json_paths
[
'path_job_logs'
],
fallback
=
self
.
DB_config_table
[
'path_job_logs'
])
gp
(
'path_job_logs'
,
json_paths
[
'path_job_logs'
],
fallback
=
self
.
absP
(
self
.
DB_config_table
[
'path_job_logs'
]))
else
:
# software test mode, the repository should be self-contained -> use only relative paths
self
.
path_fileserver
=
self
.
absP
(
'../../tests/data/'
)
self
.
path_archive
=
self
.
absP
(
'../../tests/data/archive_data/'
)
self
.
path_procdata_scenes
=
self
.
absP
(
'../../tests/data/output_scenes/'
)
self
.
path_procdata_MGRS
=
self
.
absP
(
'../../tests/data/output_mgrs_tiles/'
)
self
.
path_earthSunDist
=
self
.
absP
(
'../database/earth_sun_distance/Earth_Sun_distances_per_day_edited.csv'
)
self
.
path_SRFs
=
self
.
absP
(
'../database/srf/'
)
self
.
path_cloud_classif
=
self
.
absP
(
'../database/cloud_classifier/'
)
self
.
path_solar_irr
=
self
.
absP
(
'../database/solar_irradiance/SUNp1fontenla__350-2500nm_@0.1nm_converted.txt'
)
self
.
path_ECMWF_db
=
self
.
absP
(
'../../tests/data/processed_ECMWF/'
)
self
.
path_benchmarks
=
self
.
absP
(
'../benchmarks/'
)
self
.
path_job_logs
=
self
.
absP
(
'../logs/job_logs/'
)
self
.
path_fileserver
=
self
.
joinP
(
path_gmslib
,
'..'
,
'tests'
,
'data'
)
self
.
path_archive
=
self
.
joinP
(
path_gmslib
,
'..'
,
'tests'
,
'data'
,
'archive_data'
)
self
.
path_procdata_scenes
=
self
.
joinP
(
path_gmslib
,
'..'
,
'tests'
,
'data'
,
'output_scenes'
)
self
.
path_procdata_MGRS
=
self
.
joinP
(
path_gmslib
,
'..'
,
'tests'
,
'data'
,
'output_mgrs_tiles'
)
self
.
path_earthSunDist
=
self
.
joinP
(
path_gmslib
,
'database'
,
'earth_sun_distance'
,
'Earth_Sun_distances_per_day_edited.csv'
)
self
.
path_SRFs
=
self
.
joinP
(
path_gmslib
,
'database'
,
'srf'
)
self
.
path_cloud_classif
=
self
.
joinP
(
path_gmslib
,
'database'
,
'cloud_classifier'
)
self
.
path_solar_irr
=
self
.
joinP
(
path_gmslib
,
'database'
,
'solar_irradiance'
,
'SUNp1fontenla__350-2500nm_@0.1nm_converted.txt'
)
self
.
path_ECMWF_db
=
self
.
joinP
(
path_gmslib
,
'..'
,
'tests'
,
'data'
,
'processed_ECMWF'
)
self
.
path_benchmarks
=
self
.
joinP
(
path_gmslib
,
'..'
,
'tests'
,
'data'
,
'benchmarks'
)
self
.
path_job_logs
=
self
.
joinP
(
path_gmslib
,
'logs'
,
'job_logs'
)
###########################
# processor configuration #
...
...
@@ -251,7 +253,7 @@ class JobConfig(object):
self
.
SZA_SAA_calculation_accurracy
=
\
gp
(
'SZA_SAA_calculation_accurracy'
,
json_processors
[
'L1A'
][
'SZA_SAA_calculation_accurracy'
])
self
.
export_VZA_SZA_SAA_RAA_stats
=
\
gp
(
'export_VZA_SZA_SAA_RAA_stats'
,
json_processors
[
'L1A'
][
'
SZA_SAA_calculation_accurracy
'
])
gp
(
'export_VZA_SZA_SAA_RAA_stats'
,
json_processors
[
'L1A'
][
'
export_VZA_SZA_SAA_RAA_stats
'
])
# L1B
self
.
exec_L1BP
=
gp
(
'exec_L1BP'
,
[
...
...
gms_preprocessing/options/options_default.json
View file @
ce422a51
...
...
@@ -3,7 +3,7 @@
"exec_mode"
:
"Python"
,
/*
"Python"
or
"Flink"
*/
"db_host"
:
"localhost"
,
"CPUs"
:
"None"
,
/*number
of
CPU
cores
to
be
used
for
processing
(default:
"None"
->
use
all
available)*/
"allow_subMultiprocessing"
:
true
,
/*allow
multiprocessing
within
workers*/
"allow_subMultiprocessing"
:
true
,
/*allow
multiprocessing
within
multiprocessing
workers*/
"disable_exception_handler"
:
false
,
/*enable/disable
automatic
handling
of
unexpected
exceptions*/
"log_level"
:
"INFO"
,
/*the
logging
level
to
be
used
(choices:
'DEBUG'
,
'INFO'
,
'WARNING'
,
'ERROR'
,
'CRITICAL';*/
"tiling_block_size_XY"
:
[
...
...
@@ -100,7 +100,9 @@
"delete_output"
:
false
}
},
"usecase"
:
{
"usecase"
:
{
/*NOTE:
These
options
will
not
respected
in
the
WebApp!
Use
the
WebApp
GUI
instead.*/
"virtual_sensor_id"
:
10
,
/*
"None"
:
use
WebApp
input;
1
:
Landsat
-8
,
10
:
Sentinel
-2
A
10
m*/
"datasetid_spatial_ref"
:
"None"
,
/*
"None"
:
use
WebApp
input*/
"datasetid_spectral_ref"
:
249
,
/*
249
=Sentinel
-2
A*/
...
...
gms_preprocessing/processing/process_controller.py
View file @
ce422a51
...
...
@@ -79,7 +79,8 @@ class process_controller(object):
self
.
summary_quick
=
None
# set GMS configuration
set_config
(
job_ID
=
job_ID
,
exec_mode
=
exec_mode
,
db_host
=
db_host
,
reset
=
True
,
**
job_config_kwargs
)
set_config
(
job_ID
=
job_ID
,
exec_mode
=
exec_mode
,
db_host
=
db_host
,
reset
=
True
,
**
job_config_kwargs
if
job_config_kwargs
else
{})
self
.
config
=
GMS_config
# check environment
...
...
tests/test_spechomo_classifier.py
View file @
ce422a51
...
...
@@ -12,13 +12,12 @@ import unittest
import
os
import
numpy
as
np
from
gms_preprocessing
import
__
file
__
# noqa E402 module level import not at top of file
from
gms_preprocessing
.options.config
import
set_config
# noqa E402 module level import not at top of file
from
gms_preprocessing
import
__
path
__
# noqa E402 module level import not at top of file
from
gms_preprocessing
import
set_config
# noqa E402 module level import not at top of file
from
gms_preprocessing.algorithms.L2B_P
import
SpecHomo_Classifier
# noqa E402 module level import not at top of file
testdata
=
os
.
path
.
join
(
os
.
path
.
dirname
(
__file__
),
'../tests/data/hy_spec_data/Bavaria_farmland_LMU_Hyspex_subset.bsq'
)
testdata
=
os
.
path
.
join
(
__path__
[
0
],
'../tests/data/hy_spec_data/Bavaria_farmland_LMU_Hyspex_subset.bsq'
)
class
Test_SpecHomo_Classifier
(
unittest
.
TestCase
):
...
...
@@ -27,7 +26,7 @@ class Test_SpecHomo_Classifier(unittest.TestCase):
@
classmethod
def
setUpClass
(
cls
):
# Testjob Landsat-8
set_config
(
exec_mode
=
'Python'
,
job_ID
=
26186196
,
db_host
=
'geoms'
,
reset
=
True
)
set_config
(
exec_mode
=
'Python'
,
job_ID
=
26186196
,
db_host
=
'geoms'
,
reset
=
True
,
is_test
=
True
)
cls
.
SHC
=
SpecHomo_Classifier
([
testdata
,
testdata
,
],
v
=
False
)
def
test_generate_reference_cube_L8
(
self
):
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment