Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
geomultisens
gms_preprocessing
Commits
a16e19c1
Commit
a16e19c1
authored
Nov 24, 2017
by
Daniel Scheffler
Browse files
Config revision - intermediate state: JSON file completely parsed.
Former-commit-id:
cc4f6a67
parent
068e26cb
Changes
11
Expand all
Hide whitespace changes
Inline
Side-by-side
bin/run_gms.py
View file @
a16e19c1
...
...
@@ -207,7 +207,6 @@ def get_gms_argparser():
'-blocksize'
:
dict
(),
'-profiling'
:
dict
(),
'-bench_all'
:
dict
(),
'-bench_cloudMask'
:
dict
(),
}
...
...
gms_preprocessing/algorithms/L1C_P.py
View file @
a16e19c1
...
...
@@ -148,8 +148,8 @@ class L1C_object(L1B_object):
meshwidth
=
10
,
nodata_mask
=
None
,
# dont overwrite areas outside the image with nodata
outFill
=
get_outFillZeroSaturated
(
np
.
float32
)[
0
],
accurracy
=
CFG
.
job
.
SZA_SAA_calculation_accurracy
,
lonlat_arr
=
self
.
lonlat_arr
if
CFG
.
job
.
SZA_SAA_calculation_accurracy
==
'fine'
else
None
)
accurracy
=
CFG
.
usecase
.
SZA_SAA_calculation_accurracy
,
lonlat_arr
=
self
.
lonlat_arr
if
CFG
.
usecase
.
SZA_SAA_calculation_accurracy
==
'fine'
else
None
)
return
self
.
_SZA_arr
@
SZA_arr
.
setter
...
...
@@ -662,7 +662,7 @@ class AtmCorr(object):
# compute cloud mask if not already provided
if
no_avail_CMs
:
algorithm
=
CFG
.
job
.
cloud_masking_algorithm
[
self
.
inObjs
[
0
].
satellite
]
algorithm
=
CFG
.
usecase
.
cloud_masking_algorithm
[
self
.
inObjs
[
0
].
satellite
]
if
algorithm
==
'SICOR'
:
return
None
...
...
@@ -792,7 +792,7 @@ class AtmCorr(object):
script
=
False
# check if ECMWF data are available - if not, start the download
if
CFG
.
job
.
auto_download_ecmwf
:
if
CFG
.
usecase
.
auto_download_ecmwf
:
self
.
_check_or_download_ECMWF_data
()
# validate SNR
...
...
gms_preprocessing/config.py
View file @
a16e19c1
This diff is collapsed.
Click to expand it.
gms_preprocessing/io/input_reader.py
View file @
a16e19c1
...
...
@@ -337,10 +337,10 @@ class SRF(object):
plt
.
legend
(
loc
=
'upper right'
)
def
pickle_SRF_DB
(
L1A_Instances
):
def
pickle_SRF_DB
(
L1A_Instances
,
dir_out
):
list_GMS_identifiers
=
[
i
.
GMS_identifier
for
i
in
L1A_Instances
]
out_dict
=
collections
.
OrderedDict
()
logger
=
GMS_logger
(
'log__SRF2PKL'
,
path_logfile
=
os
.
path
.
join
(
CFG
.
job
.
path_testing
,
'out/
log__SRF2PKL.log'
),
logger
=
GMS_logger
(
'log__SRF2PKL'
,
path_logfile
=
os
.
path
.
join
(
dir_out
,
'
log__SRF2PKL.log'
),
log_level
=
CFG
.
job
.
log_level
,
append
=
False
)
for
Id
,
Inst
in
zip
(
list_GMS_identifiers
,
L1A_Instances
):
Id
[
'logger'
]
=
logger
...
...
@@ -348,7 +348,7 @@ def pickle_SRF_DB(L1A_Instances):
Inst
.
satellite
+
'_'
+
Inst
.
sensor
+
((
'_'
+
Inst
.
subsystem
)
if
Inst
.
subsystem
not
in
[
''
,
None
]
else
''
)]
\
=
SRF_reader
(
Id
)
print
(
list
(
out_dict
.
keys
()))
outFilename
=
os
.
path
.
join
(
CFG
.
job
.
path_testing
,
'out/
SRF_DB.pkl'
)
outFilename
=
os
.
path
.
join
(
dir_out
,
'
SRF_DB.pkl'
)
with
open
(
outFilename
,
'wb'
)
as
outFile
:
dill
.
dump
(
out_dict
,
outFile
)
print
(
'Saved SRF dictionary to %s'
%
outFilename
)
...
...
gms_preprocessing/misc/database_tools.py
View file @
a16e19c1
...
...
@@ -610,19 +610,22 @@ class GMS_JOB(object):
"""
:param conn_db: <str> the database connection parameters as given by CFG.job.conn_params
"""
# privates
self
.
_virtualsensorid
=
None
# defaults
self
.
conn
=
conn_db
self
.
dataframe
=
GeoDataFrame
()
self
.
scene_counts
=
{}
# set by self.create()
self
.
exists_in_db
=
False
self
.
id
=
None
#: int
self
.
creationtime
=
datetime
.
now
()
self
.
creationtime
=
datetime
.
now
()
# default, needed to create new job
self
.
finishtime
=
None
self
.
sceneids
=
[]
self
.
timerange_start
=
datetime
.
min
self
.
timerange_end
=
datetime
.
max
self
.
bounds
=
box
(
-
180
,
-
90
,
180
,
90
)
self
.
bounds
=
box
(
-
180
,
-
90
,
180
,
90
)
# default, needed to create new job
self
.
distribution_index
=
None
self
.
progress
=
None
self
.
feedback
=
None
...
...
@@ -633,20 +636,31 @@ class GMS_JOB(object):
'timerange_end'
,
'bounds'
,
'distribution_index'
,
'progress'
,
'feedback'
,
'failed_sceneids'
,
'datasetid_spatial_ref'
,
'virtualsensorid'
,
'ref_job_id'
,
'datacube_mgrs_tiles_proc'
,
'comment'
,
'status'
,
'statistics'
]
self
.
virtualsensorid
=
None
# set by self._set_target_sensor_specs()
# FIXME notnull but not getable via virtual sensor id and not needed anymore in DB (gsd is given):
self
.
datasetid_spatial_ref
=
249
self
.
datasetname_spatial_ref
=
'SENTINEL-2A'
'non_ref_datasetids'
,
'max_cloudcover'
,
'season_code'
,
'status'
,
'path_analysis_script'
,
'analysis_parameter'
,
'statistics'
,
'job_mode'
]
self
.
datasetid_spatial_ref
=
249
# this is overwritten if existing job is read from DB but needed to create new
self
.
datasetname_spatial_ref
=
'SENTINEL-2A'
# same here
self
.
status
=
None
self
.
statistics
=
[]
self
.
comment
=
None
self
.
epsg
=
None
# set by self._set_target_sensor_specs()
self
.
ground_spatial_sampling
=
None
# set by self._set_target_sensor_specs()
self
.
analysis_parameter
=
None
def
__repr__
(
self
):
return
'GMS job:
\n\n
'
+
GeoSeries
(
self
.
db_entry
).
to_string
()
@
property
def
virtualsensorid
(
self
):
return
self
.
_virtualsensorid
@
virtualsensorid
.
setter
def
virtualsensorid
(
self
,
value
):
"""Set virtual sensor ID but continue if no data value is received
NOTE: set by self._set_target_sensor_specs() and self.from_ID()"""
if
value
!=
-
1
:
# no data value
self
.
_virtualsensorid
=
value
def
_set_target_sensor_specs
(
self
,
virtual_sensor_id
,
datasetid_spatial_ref
):
self
.
virtualsensorid
=
virtual_sensor_id
res
=
get_info_from_postgreSQLdb
(
self
.
conn
,
'virtual_sensors'
,
[
'spatial_resolution'
,
...
...
@@ -671,11 +685,17 @@ class GMS_JOB(object):
db_entry
=
collections
.
OrderedDict
()
for
i
in
self
.
jobs_table_columns
:
db_entry
[
i
]
=
getattr
(
self
,
i
)
val
=
getattr
(
self
,
i
)
if
i
==
'virtualsensorid'
and
val
is
None
:
val
=
-
1
# nodata value
db_entry
[
i
]
=
val
return
db_entry
def
from_dictlist
(
self
,
dictlist_data2process
,
virtual_sensor_id
,
datasetid_spatial_ref
=
249
,
comment
=
None
):
# type: (list, int, int, str) ->
object
# type: (list, int, int, str) ->
GMS_JOB
"""
:param dictlist_data2process: <list> a list of dictionaries containing the keys "satellite", "sensor" and
"filenames",
...
...
@@ -910,7 +930,7 @@ class GMS_JOB(object):
self
.
timerange_end
=
self
.
dataframe
.
acquisitiondate
.
max
().
to_pydatetime
()
def
from_job_ID
(
self
,
job_ID
):
# type: (int) ->
object
# type: (int) ->
GMS_JOB
"""
Create a GMS_JOB instance by querying the database for a specific job ID.
:param job_ID: <int> a valid id from the database table 'jobs'
...
...
gms_preprocessing/misc/definition_dicts.py
View file @
a16e19c1
...
...
@@ -104,7 +104,7 @@ def get_mask_classdefinition(maskname, satellite):
'Snow'
:
60
}
# SICOR
}
return
legends
[
CFG
.
job
.
cloud_masking_algorithm
[
satellite
]]
return
legends
[
CFG
.
usecase
.
cloud_masking_algorithm
[
satellite
]]
else
:
raise
ValueError
(
"'%s' is not a supported mask name."
%
maskname
)
...
...
gms_preprocessing/misc/environment.py
View file @
a16e19c1
...
...
@@ -47,7 +47,7 @@ class GMSEnvironment(object):
# noinspection PyUnresolvedReferences
import
fmask
# noqa F401 unused
except
ImportError
:
if
'FMASK'
in
list
(
CFG
.
job
.
cloud_masking_algorithm
.
values
()):
if
'FMASK'
in
list
(
CFG
.
usecase
.
cloud_masking_algorithm
.
values
()):
msg
=
"FMASK library is not installed because it is not pip-installable and must be installed "
\
"manually, e.g. for Anaconda by running 'conda install -c conda-forge python-fmask'!"
self
.
logger
.
warning
(
MissingNonPipLibraryWarning
(
msg
))
...
...
gms_preprocessing/model/gms_object.py
View file @
a16e19c1
...
...
@@ -302,7 +302,7 @@ class GMS_object(Dataset):
@
property
def
cloud_masking_algorithm
(
self
):
if
not
self
.
_cloud_masking_algorithm
:
self
.
_cloud_masking_algorithm
=
CFG
.
job
.
cloud_masking_algorithm
[
self
.
satellite
]
self
.
_cloud_masking_algorithm
=
CFG
.
usecase
.
cloud_masking_algorithm
[
self
.
satellite
]
return
self
.
_cloud_masking_algorithm
@
property
...
...
gms_preprocessing/options_default.json
View file @
a16e19c1
{
"job"
:
{
"exec_mode"
:
"Python"
,
/*
"Python"
or
"Flink"
*/
"db_host"
:
"localhost"
,
"CPUs"
:
"None"
,
/*number
of
CPU
cores
to
be
used
for
processing
(default:
"None"
->
use
all
available)*/
"allow_subMultiprocessing"
:
true
,
/*allow
multiprocessing
within
workers*/
"disable_exception_handler"
:
false
,
/*enable/disable
automatic
handling
of
unexpected
exceptions*/
"log_level"
:
"INFO"
,
/*the
logging
level
to
be
used
(choices:
'DEBUG'
,
'INFO'
,
'WARNING'
,
'ERROR'
,
'CRITICAL';*/
"tiling_block_size_XY"
:
[
2048
,
2048
],
/*X/Y
block
size
to
be
used
for
any
tiling
process*/
"is_test"
:
false
,
/*whether
the
current
job
represents
a
software
test
job
(run
by
a
test
runner)
or
not*/
"profiling"
:
false
,
/*enable/disable
code
profiling*/
"benchmark_global"
:
false
,
/*enable/disable
benchmark
of
the
whole
processing
pipeline*/
"bench_cloudMask"
:
false
,
/*enable/disable
benchmark
of
the
of
the
cloud
mask
generator
module*/
"paths"
:
{
"path_fileserver"
:
""
,
"path_archive"
:
""
,
/*input
path
where
downloaded
data
are
stored*/
"path_procdata_scenes"
:
""
,
/*output
path
to
store
processed
scenes*/
"path_procdata_MGRS"
:
""
,
/*output
path
to
store
processed
MGRS
tiles*/
"path_tempdir"
:
""
,
"path_testing"
:
""
,
"path_benchmarks"
:
""
,
"path_job_logs"
:
""
,
/*output
path
to
store
job
log
files*/
"path_spatIdxSrv"
:
""
,
"path_ac_tables"
:
""
,
"path_SNR_models"
:
""
,
"path_SRFs"
:
""
,
"path_dem_proc_srtm_90m"
:
""
,
"path_earthSunDist"
:
""
,
"path_solar_irr"
:
""
,
"path_cloud_classif"
:
""
,
"path_ECMWF_db"
:
""
"exec_mode"
:
"Python"
,
/*
"Python"
or
"Flink"
*/
"db_host"
:
"localhost"
,
"CPUs"
:
"None"
,
/*number
of
CPU
cores
to
be
used
for
processing
(default:
"None"
->
use
all
available)*/
"allow_subMultiprocessing"
:
true
,
/*allow
multiprocessing
within
workers*/
"disable_exception_handler"
:
false
,
/*enable/disable
automatic
handling
of
unexpected
exceptions*/
"log_level"
:
"INFO"
,
/*the
logging
level
to
be
used
(choices:
'DEBUG'
,
'INFO'
,
'WARNING'
,
'ERROR'
,
'CRITICAL';*/
"tiling_block_size_XY"
:
[
2048
,
2048
],
/*X/Y
block
size
to
be
used
for
any
tiling
process*/
"is_test"
:
false
,
/*whether
the
current
job
represents
a
software
test
job
(run
by
a
test
runner)
or
not*/
"profiling"
:
false
,
/*enable/disable
code
profiling*/
"benchmark_global"
:
false
,
/*enable/disable
benchmark
of
the
whole
processing
pipeline*/
"paths"
:
{
"path_fileserver"
:
""
,
"path_archive"
:
""
,
/*input
path
where
downloaded
data
are
stored*/
"path_procdata_scenes"
:
""
,
/*output
path
to
store
processed
scenes*/
"path_procdata_MGRS"
:
""
,
/*output
path
to
store
processed
MGRS
tiles*/
"path_tempdir"
:
""
,
"path_benchmarks"
:
""
,
"path_job_logs"
:
""
,
/*output
path
to
store
job
log
files*/
"path_spatIdxSrv"
:
""
,
"path_ac_tables"
:
""
,
"path_SNR_models"
:
""
,
"path_SRFs"
:
""
,
"path_dem_proc_srtm_90m"
:
""
,
"path_earthSunDist"
:
""
,
"path_solar_irr"
:
""
,
"path_cloud_classif"
:
""
,
"path_ECMWF_db"
:
""
},
"processors"
:
{
"general_opts"
:
{
/*configuration
parameters
needed
in
all
processors*/
"skip_thermal"
:
true
,
"skip_pan"
:
true
,
"sort_bands_by_cwl"
:
true
,
"conversion_type_optical"
:
"BOA_Ref"
,
/*'Rad'
/
'TOA_Ref'
/
'BOA_Ref'*/
"conversion_type_thermal"
:
"Rad"
,
/*'Rad'
/
'Temp'*/
"scale_factor_TOARef"
:
10000
,
"scale_factor_BOARef"
:
10000
},
"processors"
:
{
"L1A_P"
:
{
/*Level
1
A
processing:
Data
import
and
metadata
homogenization*/
"run_processor"
:
true
,
"write_output"
:
true
,
"delete_output"
:
true
,
"SZA_SAA_calculation_accurracy"
:
"coarse"
,
/*
"coarse"
or
"fine"
*/
"export_VZA_SZA_SAA_RAA_stats"
:
true
"L1A_P"
:
{
/*Level
1
A
processing:
Data
import
and
metadata
homogenization*/
"run_processor"
:
true
,
"write_output"
:
true
,
"delete_output"
:
true
,
"SZA_SAA_calculation_accurracy"
:
"coarse"
,
/*
"coarse"
or
"fine"
*/
"export_VZA_SZA_SAA_RAA_stats"
:
true
},
"L1B_P"
:
{
/*Level
1
B
processing:
calculation
of
geometric
shift*/
"run_processor"
:
true
,
"write_output"
:
true
,
"delete_output"
:
true
,
"skip_coreg"
:
false
},
"L1C_P"
:
{
/*Level
1
C
processing:
atmospheric
correction*/
"run_processor"
:
true
,
"write_output"
:
true
,
"delete_output"
:
true
,
"cloud_masking_algorithm"
:
{
"Landsat-4"
:
"FMASK"
,
"Landsat-5"
:
"FMASK"
,
"Landsat-7"
:
"FMASK"
,
"Landsat-8"
:
"FMASK"
,
"Sentinel-2A"
:
"SICOR"
,
"Sentinel-2B"
:
"SICOR"
},
/*'FMASK'
,
'Classical
Bayesian'
,
'SICOR'*/
"export_L1C_obj_dumps"
:
false
,
"auto_download_ecmwf"
:
false
},
"L2A_P"
:
{
/*Level
2
A
processing:
geometric
homogenization*/
"run_processor"
:
true
,
"write_output"
:
true
,
"delete_output"
:
true
},
"L2B_P"
:
{
/*Level
2
B
processing:
spectral
homogenization*/
"run_processor"
:
true
,
"write_output"
:
true
,
"delete_output"
:
false
},
"L2C_P"
:
{
/*Level
2
C
processing:
accurracy
assessment
and
MGRS
tiling*/
"run_processor"
:
true
,
"write_output"
:
true
,
"delete_output"
:
false
}
},
"L1B_P"
:
{
/*Level
1
B
processing:
calculation
of
geometric
shift*/
"run_processor"
:
true
,
"write_output"
:
true
,
"delete_output"
:
true
,
"skip_coreg"
:
false
},
"L1C_P"
:
{
/*Level
1
C
processing:
atmospheric
correction*/
"run_processor"
:
true
,
"write_output"
:
true
,
"delete_output"
:
true
,
"cloud_masking_algorithm"
:
{
"Landsat-4"
:
"FMASK"
,
"Landsat-5"
:
"FMASK"
,
"Landsat-7"
:
"FMASK"
,
"Landsat-8"
:
"FMASK"
,
"Sentinel-2A"
:
"SICOR"
,
"Sentinel-2B"
:
"SICOR"
},
/*'FMASK'
,
'Classical
Bayesian'
,
'SICOR'*/
"export_L1C_obj_dumps"
:
false
,
"scale_factor_errors_ac"
:
255
,
"auto_download_ecmwf"
:
false
},
"L2A_P"
:
{
/*Level
2
A
processing:
geometric
homogenization*/
"run_processor"
:
true
,
"write_output"
:
true
,
"delete_output"
:
true
,
"align_coord_grids"
:
true
,
/*allows
to
force
deactivation
of
image
resampling*/
"match_gsd"
:
true
},
"L2B_P"
:
{
/*Level
2
B
processing:
spectral
homogenization*/
"run_processor"
:
true
,
"write_output"
:
true
,
"delete_output"
:
false
},
"L2C_P"
:
{
/*Level
2
C
processing:
accurracy
assessment
and
MGRS
tiling*/
"run_processor"
:
true
,
"write_output"
:
true
,
"delete_output"
:
false
}
},
"usecase"
:
{
"skip_thermal"
:
true
,
"skip_pan"
:
true
,
"sort_bands_by_cwl"
:
true
,
"conversion_type_optical"
:
"BOA_Ref"
,
/*'Rad'
/
'TOA_Ref'
/
'BOA_Ref'*/
"conversion_type_thermal"
:
"Rad"
,
/*'Rad'
/
'Temp'*/
"scale_factor_TOARef"
:
10000
,
"scale_factor_BOARef"
:
10000
,
"scale_factor_errors_ac"
:
255
,
"virtual_sensor_id"
:
"None"
,
/*
"None"
:
use
WebApp
input;
1
:
Landsat
-8
,
10
:
Sentinel
-2
A
10
m*/
"virtual_sensor_name"
:
"None"
,
/*
"None"
:
use
WebApp
input*/
"virtual_sensor_id"
:
10
,
/*
"None"
:
use
WebApp
input;
1
:
Landsat
-8
,
10
:
Sentinel
-2
A
10
m*/
"datasetid_spatial_ref"
:
"None"
,
/*
"None"
:
use
WebApp
input*/
"datasetid_spectral_ref"
:
249
,
/*
249
=Sentinel
-2
A*/
"target_CWL"
:
[
...
...
@@ -108,8 +109,6 @@
],
/*target
sensor
x-coordinate-grid.
e.g.
[
15
,
45
]
*/
"spatial_ref_gridy"
:
[
],
/*target
sensor
y-coordinate-grid.
e.g.
[
15
,
45
]
*/
"align_coord_grids"
:
true
,
/*allows
to
force
deactivation
of
image
resampling*/
"match_gsd"
:
true
]
/*target
sensor
y-coordinate-grid.
e.g.
[
15
,
45
]
*/
}
}
gms_preprocessing/options_schema.py
0 → 100644
View file @
a16e19c1
"""Definition of gms options schema (as used by cerberus library)."""
gms_schema
=
{
}
tests/test_config.py
View file @
a16e19c1
...
...
@@ -11,8 +11,9 @@ Tests for gms_preprocessing.config
import
os
from
unittest
import
TestCase
from
gms_preprocessing.config
import
get_options
from
gms_preprocessing
import
__path__
from
gms_preprocessing.config
import
get_options
from
gms_preprocessing.config
import
JobConfig
class
Test_get_options
(
TestCase
):
...
...
@@ -23,3 +24,13 @@ class Test_get_options(TestCase):
def
test_target_is_file_validation
(
self
):
opts_dict
=
get_options
(
os
.
path
.
join
(
__path__
[
0
],
'options_default.json'
))
self
.
assertIsInstance
(
opts_dict
,
dict
)
class
Test_JobConfig
(
TestCase
):
def
setUp
(
self
):
self
.
jobID
=
26186662
self
.
db_host
=
'localhost'
def
test
(
self
):
cfg
=
JobConfig
(
self
.
jobID
,
self
.
db_host
)
print
(
cfg
)
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment