Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Daniel Scheffler
py_tools_ds
Commits
28ce2c12
Commit
28ce2c12
authored
Mar 22, 2019
by
Daniel Scheffler
Browse files
Merge branch 'bugfix/fix_mp_deadlock' into 'master'
Bugfix/fix mp deadlock See merge request
!9
parents
30dbee81
02973a4b
Pipeline
#3817
failed with stages
in 16 minutes and 2 seconds
Changes
3
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
py_tools_ds/geo/raster/reproject.py
View file @
28ce2c12
...
...
@@ -5,6 +5,7 @@ import multiprocessing
import
os
from
tempfile
import
TemporaryDirectory
from
typing
import
Union
,
Tuple
,
List
,
Any
# noqa: F401
import
sys
# custom
try
:
...
...
@@ -818,8 +819,8 @@ def _initializer(lats, lons, data):
class
SensorMapGeometryTransformer3D
(
object
):
def
__init__
(
self
,
lons
,
lats
,
resamp_alg
=
'nearest'
,
radius_of_influence
=
30
,
**
opts
):
# type: (np.ndarray, np.ndarray, str, int, Any) -> None
def
__init__
(
self
,
lons
,
lats
,
resamp_alg
=
'nearest'
,
radius_of_influence
=
30
,
mp_alg
=
'auto'
,
**
opts
):
# type: (np.ndarray, np.ndarray, str, int,
str,
Any) -> None
"""Get an instance of SensorMapGeometryTransformer.
:param lons: 3D longitude array corresponding to the 3D sensor geometry array
...
...
@@ -829,6 +830,10 @@ class SensorMapGeometryTransformer3D(object):
- resamp_alg: resampling algorithm ('nearest', 'bilinear', 'gauss', 'custom')
- radius_of_influence: <float> Cut off distance in meters (default: 30)
NOTE: keyword is named 'radius' in case of bilinear resampling
- mp_alg multiprocessing algorithm
'bands': parallelize over bands using multiprocessing lib
'tiles': parallelize over tiles using OpenMP
'auto': automatically choose the algorithm
- sigmas: <list of floats or float> [ONLY 'gauss'] List of sigmas to use for the gauss
weighting of each channel 1 to k, w_k = exp(-dist^2/sigma_k^2). If only one channel
is resampled sigmas is a single float value.
...
...
@@ -867,7 +872,14 @@ class SensorMapGeometryTransformer3D(object):
# bands: multiprocessing uses multiprocessing.Pool, implemented in to_map_geometry / to_sensor_geometry
# tiles: multiprocessing uses OpenMP implemented in pykdtree which is used by pyresample
self
.
opts
[
'nprocs'
]
=
opts
.
get
(
'nprocs'
,
multiprocessing
.
cpu_count
())
self
.
mp_alg
=
'bands'
if
self
.
lons
.
shape
[
2
]
>=
opts
[
'nprocs'
]
else
'tiles'
self
.
mp_alg
=
(
'bands'
if
self
.
lons
.
shape
[
2
]
>=
opts
[
'nprocs'
]
else
'tiles'
)
if
mp_alg
==
'auto'
else
mp_alg
# override self.mp_alg if SensorMapGeometryTransformer3D is called by nosetest or unittest
is_called_by_nose_cmd
=
'nosetest'
in
sys
.
argv
[
0
]
if
self
.
opts
[
'nprocs'
]
>
1
and
self
.
mp_alg
==
'bands'
and
is_called_by_nose_cmd
:
warnings
.
warn
(
"mp_alg='bands' causes deadlocks if SensorMapGeometryTransformer3D is called within a "
"nosetest console call. Using mp_alg='tiles'."
)
self
.
mp_alg
=
'tiles'
@
staticmethod
def
_to_map_geometry_2D
(
kwargs_dict
):
...
...
py_tools_ds/version.py
View file @
28ce2c12
__version__
=
'0.14.1
6
'
__versionalias__
=
'20190322_0
2
'
__version__
=
'0.14.1
7
'
__versionalias__
=
'20190322_0
3
'
tests/test_geo/test_raster/test_reproject.py
View file @
28ce2c12
...
...
@@ -120,7 +120,9 @@ class Test_SensorMapGeometryTransformer3D(TestCase):
def
test_to_map_geometry_lonlat_3D_geolayer
(
self
):
SMGT
=
SensorMapGeometryTransformer3D
(
lons
=
self
.
lons_3D
,
lats
=
self
.
lats_3D
,
resamp_alg
=
'nearest'
)
# resamp_alg='nearest',
resamp_alg
=
'bilinear'
,
)
# to Lon/Lat
data_mapgeo_3D
,
dem_gt
,
dem_prj
=
SMGT
.
to_map_geometry
(
self
.
data_sensor_geo_3D
,
tgt_prj
=
4326
)
...
...
@@ -137,7 +139,8 @@ class Test_SensorMapGeometryTransformer3D(TestCase):
def
test_to_sensor_geometry
(
self
):
SMGT
=
SensorMapGeometryTransformer3D
(
lons
=
self
.
lons_3D
,
lats
=
self
.
lats_3D
,
resamp_alg
=
'nearest'
)
resamp_alg
=
'nearest'
,
)
dem_sensors_geo
=
SMGT
.
to_sensor_geometry
(
self
.
data_map_geo_3D
,
src_prj
=
32632
,
src_extent
=
self
.
dem_area_extent_coarse_subset_utm
)
self
.
assertIsInstance
(
dem_sensors_geo
,
np
.
ndarray
)
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment