Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
geomultisens
gms_preprocessing
Commits
e2b11a34
Commit
e2b11a34
authored
Nov 27, 2017
by
Daniel Scheffler
Browse files
Completed move of config and options to new submodule.
Former-commit-id:
ea66f0bd
Former-commit-id:
a2792c94
parent
aef55daf
Changes
27
Hide whitespace changes
Inline
Side-by-side
gms_preprocessing/__init__.py
View file @
e2b11a34
...
...
@@ -8,6 +8,7 @@ from . import algorithms # noqa: E402
from
.
import
io
# noqa: E402
from
.
import
misc
# noqa: E402
from
.
import
processing
# noqa: E402
from
.
import
options
# noqa: E402
from
.options.config
import
set_config
# noqa: E402
from
.processing.process_controller
import
process_controller
# noqa: E402
...
...
@@ -19,6 +20,7 @@ __all__ = ['algorithms',
'io'
,
'misc'
,
'processing'
,
'config'
,
'options'
,
'set_config'
,
'process_controller'
,
]
gms_preprocessing/algorithms/L1A_P.py
View file @
e2b11a34
...
...
@@ -19,7 +19,7 @@ from py_tools_ds.geo.coord_trafo import pixelToLatLon
from
py_tools_ds.geo.map_info
import
mapinfo2geotransform
from
py_tools_ds.geo.projection
import
EPSG2WKT
from
..config
import
GMS_config
as
CFG
from
..
options.
config
import
GMS_config
as
CFG
from
.
import
geoprocessing
as
GEOP
from
..io
import
output_writer
as
OUT_W
from
..misc
import
helper_functions
as
HLP_F
...
...
gms_preprocessing/algorithms/L1B_P.py
View file @
e2b11a34
...
...
@@ -27,7 +27,7 @@ from py_tools_ds.geo.coord_trafo import reproject_shapelyGeometry, transform_any
from
py_tools_ds.geo.projection
import
prj_equal
,
EPSG2WKT
,
WKT2EPSG
from
py_tools_ds.geo.vector.topology
import
get_overlap_polygon
from
..config
import
GMS_config
as
CFG
from
..
options.
config
import
GMS_config
as
CFG
from
..model.gms_object
import
GMS_object
from
.L1A_P
import
L1A_object
from
..misc
import
database_tools
as
DB_T
...
...
gms_preprocessing/algorithms/L1C_P.py
View file @
e2b11a34
...
...
@@ -15,7 +15,7 @@ import numpy as np
from
geoarray
import
GeoArray
from
py_tools_ds.geo.map_info
import
mapinfo2geotransform
from
..config
import
GMS_config
as
CFG
from
..
options.
config
import
GMS_config
as
CFG
from
.
import
geoprocessing
as
GEOP
from
.L1B_P
import
L1B_object
from
..model.metadata
import
get_LayerBandsAssignment
...
...
gms_preprocessing/algorithms/L2B_P.py
View file @
e2b11a34
...
...
@@ -19,7 +19,7 @@ from sklearn.cluster import k_means_ # noqa F401 # flake8 issue
from
geoarray
import
GeoArray
# noqa F401 # flake8 issue
from
py_tools_ds.processing.progress_mon
import
ProgressBar
from
..config
import
GMS_config
as
CFG
from
..
options.
config
import
GMS_config
as
CFG
from
..io.input_reader
import
SRF
# noqa F401 # flake8 issue
from
..misc.logging
import
GMS_logger
from
.L2A_P
import
L2A_object
...
...
gms_preprocessing/algorithms/cloud_masking.py
View file @
e2b11a34
...
...
@@ -69,7 +69,7 @@ class _FMASK_Runner(object):
@
property
def
is_GMSConfig_available
(
self
):
from
..config
import
GMS_config
as
CFG
from
..
options.
config
import
GMS_config
as
CFG
try
:
if
CFG
is
not
None
:
return
True
...
...
@@ -349,7 +349,7 @@ class FMASK_Runner_Sentinel2(_FMASK_Runner):
given."""
if
not
self
.
_granule_ID
and
self
.
scene_ID
and
self
.
scene_ID
!=
-
9999
and
self
.
is_GMSConfig_available
:
from
..config
import
GMS_config
as
CFG
from
..
options.
config
import
GMS_config
as
CFG
res
=
get_info_from_postgreSQLdb
(
CFG
.
conn_database
,
'scenes'
,
[
'entityid'
],
{
'id'
:
self
.
scene_ID
})
assert
len
(
res
)
!=
0
,
\
"Invalid SceneID given - no corresponding scene with the ID=%s found in database.
\n
"
%
self
.
scene_ID
...
...
gms_preprocessing/algorithms/geoprocessing.py
View file @
e2b11a34
...
...
@@ -41,7 +41,7 @@ from geoarray import GeoArray
from
py_tools_ds.geo.coord_grid
import
snap_bounds_to_pixGrid
from
py_tools_ds.geo.coord_trafo
import
transform_utm_to_wgs84
,
transform_wgs84_to_utm
,
mapXY2imXY
,
imXY2mapXY
from
..config
import
GMS_config
as
CFG
from
..
options.
config
import
GMS_config
as
CFG
from
..misc.definition_dicts
import
get_outFillZeroSaturated
__author__
=
'Daniel Scheffler'
,
'Robert Behling'
...
...
gms_preprocessing/io/input_reader.py
View file @
e2b11a34
...
...
@@ -27,7 +27,7 @@ from py_tools_ds.geo.coord_calc import corner_coord_to_minmax
from
py_tools_ds.geo.coord_trafo
import
transform_any_prj
from
py_tools_ds.numeric.vector
import
find_nearest
from
..config
import
GMS_config
as
CFG
from
..
options.
config
import
GMS_config
as
CFG
from
..model
import
metadata
as
META
from
..misc
import
path_generator
as
PG
from
..misc
import
helper_functions
as
HLP_F
...
...
gms_preprocessing/io/output_writer.py
View file @
e2b11a34
...
...
@@ -32,7 +32,7 @@ import warnings
import
logging
from
itertools
import
chain
from
..config
import
GMS_config
as
CFG
from
..
options.
config
import
GMS_config
as
CFG
from
..misc
import
helper_functions
as
HLP_F
from
..misc.definition_dicts
import
\
get_mask_classdefinition
,
get_mask_colormap
,
get_outFillZeroSaturated
,
dtype_lib_Python_IDL
...
...
gms_preprocessing/misc/database_tools.py
View file @
e2b11a34
...
...
@@ -22,7 +22,7 @@ from shapely.geometry import Polygon, box, MultiPolygon
from
sqlalchemy
import
create_engine
from
sqlalchemy.types
import
to_instance
,
TypeEngine
from
..config
import
GMS_config
as
CFG
from
..
options.
config
import
GMS_config
as
CFG
from
.
import
path_generator
as
PG
from
.definition_dicts
import
proc_chain
...
...
gms_preprocessing/misc/definition_dicts.py
View file @
e2b11a34
...
...
@@ -5,7 +5,7 @@ import re
import
numpy
as
np
from
..config
import
GMS_config
as
CFG
from
..
options.
config
import
GMS_config
as
CFG
__author__
=
'Daniel Scheffler'
...
...
gms_preprocessing/misc/environment.py
View file @
e2b11a34
...
...
@@ -8,7 +8,7 @@ except ImportError:
import
gdal
from
logging
import
Logger
from
..config
import
GMS_config
as
CFG
from
..
options.
config
import
GMS_config
as
CFG
from
.spatial_index_mediator
import
SpatialIndexMediatorServer
from
.exceptions
import
GMSEnvironmentError
,
MissingNonPipLibraryWarning
...
...
gms_preprocessing/misc/exception_handler.py
View file @
e2b11a34
...
...
@@ -11,7 +11,7 @@ from typing import Union, List # noqa F401 # flake8 issue
from
..model.gms_object
import
GMS_object
# noqa F401 # flake8 issue
from
..model.gms_object
import
failed_GMS_object
from
..config
import
GMS_config
as
CFG
from
..
options.
config
import
GMS_config
as
CFG
from
..misc
import
database_tools
as
DB_T
from
.definition_dicts
import
db_jobs_statistics_def
,
proc_chain
...
...
gms_preprocessing/misc/helper_functions.py
View file @
e2b11a34
...
...
@@ -29,7 +29,7 @@ from matplotlib import pyplot as plt
from
subprocess
import
Popen
,
PIPE
from
xml.etree.ElementTree
import
QName
from
..config
import
GMS_config
as
CFG
from
..
options.
config
import
GMS_config
as
CFG
from
.
import
database_tools
as
DB_T
from
..algorithms.L1A_P
import
L1A_object
from
..algorithms.L1B_P
import
L1B_object
...
...
gms_preprocessing/misc/path_generator.py
View file @
e2b11a34
...
...
@@ -7,7 +7,7 @@ import warnings
import
uuid
from
logging
import
Logger
from
..config
import
GMS_config
as
CFG
from
..
options.
config
import
GMS_config
as
CFG
from
.definition_dicts
import
get_GMS_sensorcode
...
...
gms_preprocessing/model/gms_object.py
View file @
e2b11a34
...
...
@@ -38,7 +38,7 @@ from ..model.metadata import METADATA, get_dict_LayerOptTherm, metaDict_to_metaO
from
..model.dataset
import
Dataset
from
..misc
import
path_generator
as
PG
from
..misc
import
database_tools
as
DB_T
from
..config
import
GMS_config
as
CFG
from
..
options.
config
import
GMS_config
as
CFG
from
..algorithms
import
geoprocessing
as
GEOP
from
..io
import
input_reader
as
INP_R
from
..io
import
output_writer
as
OUT_W
...
...
gms_preprocessing/model/metadata.py
View file @
e2b11a34
...
...
@@ -22,7 +22,7 @@ from pyorbital import astronomy
from
py_tools_ds.geo.map_info
import
geotransform2mapinfo
from
py_tools_ds.geo.projection
import
WKT2EPSG
from
gms_preprocessing.config
import
GMS_config
as
CFG
from
gms_preprocessing.
options.
config
import
GMS_config
as
CFG
from
gms_preprocessing.io.input_reader
import
open_specific_file_within_archive
,
Solar_Irradiance_reader
,
SRF_reader
from
gms_preprocessing.io.output_writer
import
enviHdr_keyOrder
from
gms_preprocessing.algorithms
import
geoprocessing
as
GEOP
...
...
gms_preprocessing/model/mgrs_tile.py
View file @
e2b11a34
...
...
@@ -4,7 +4,7 @@ import shapely
from
py_tools_ds.geo.coord_trafo
import
reproject_shapelyGeometry
from
shapely.geometry
import
Polygon
from
gms_preprocessing.config
import
GMS_config
as
CFG
from
gms_preprocessing.
options.
config
import
GMS_config
as
CFG
from
gms_preprocessing.misc
import
database_tools
as
DB_T
from
gms_preprocessing.misc
import
helper_functions
as
HLP_F
from
gms_preprocessing.algorithms
import
geoprocessing
as
GEOP
...
...
gms_preprocessing/options/config.py
View file @
e2b11a34
...
...
@@ -206,18 +206,18 @@ class JobConfig(object):
else
:
# software test mode, the repository should be self-contained -> use only relative paths
self
.
path_fileserver
=
self
.
absP
(
'../tests/data/'
)
self
.
path_archive
=
self
.
absP
(
'../tests/data/archive_data/'
)
self
.
path_procdata_scenes
=
self
.
absP
(
'../tests/data/output_scenes/'
)
self
.
path_procdata_MGRS
=
self
.
absP
(
'../tests/data/output_mgrs_tiles/'
)
self
.
path_earthSunDist
=
self
.
absP
(
'./database/earth_sun_distance/Earth_Sun_distances_per_day_edited.csv'
,
)
self
.
path_SRFs
=
self
.
absP
(
'./database/srf/'
)
self
.
path_cloud_classif
=
self
.
absP
(
'./database/cloud_classifier/'
)
self
.
path_fileserver
=
self
.
absP
(
'../
../
tests/data/'
)
self
.
path_archive
=
self
.
absP
(
'../
../
tests/data/archive_data/'
)
self
.
path_procdata_scenes
=
self
.
absP
(
'../
../
tests/data/output_scenes/'
)
self
.
path_procdata_MGRS
=
self
.
absP
(
'../
../
tests/data/output_mgrs_tiles/'
)
self
.
path_earthSunDist
=
self
.
absP
(
'.
.
/database/earth_sun_distance/Earth_Sun_distances_per_day_edited.csv'
)
self
.
path_SRFs
=
self
.
absP
(
'.
.
/database/srf/'
)
self
.
path_cloud_classif
=
self
.
absP
(
'.
.
/database/cloud_classifier/'
)
self
.
path_solar_irr
=
self
.
absP
(
'./database/solar_irradiance/SUNp1fontenla__350-2500nm_@0.1nm_converted.txt'
)
self
.
path_ECMWF_db
=
self
.
absP
(
'../tests/data/processed_ECMWF/'
)
self
.
path_benchmarks
=
self
.
absP
(
'./benchmarks/'
)
self
.
path_job_logs
=
self
.
absP
(
'./logs/job_logs/'
)
'.
.
/database/solar_irradiance/SUNp1fontenla__350-2500nm_@0.1nm_converted.txt'
)
self
.
path_ECMWF_db
=
self
.
absP
(
'../
../
tests/data/processed_ECMWF/'
)
self
.
path_benchmarks
=
self
.
absP
(
'.
.
/benchmarks/'
)
self
.
path_job_logs
=
self
.
absP
(
'.
.
/logs/job_logs/'
)
###########################
# processor configuration #
...
...
gms_preprocessing/processing/multiproc.py
View file @
e2b11a34
...
...
@@ -3,7 +3,7 @@
from
multiprocessing
import
Pool
from
itertools
import
chain
from
..config
import
GMS_config
as
CFG
from
..
options.
config
import
GMS_config
as
CFG
__author__
=
'Daniel Scheffler'
...
...
Prev
1
2
Next
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment