Merge branch 'bugfix/fix_mp_deadlock' into 'master'

Bugfix/fix mp deadlock

See merge request !9

py_tools_ds/geo/raster/reproject.py

@@ -5,6 +5,7 @@ import multiprocessing
 import os
 from tempfile import TemporaryDirectory
 from typing import Union, Tuple, List, Any  # noqa: F401
+import sys
 
 # custom
 try:
@@ -818,8 +819,8 @@ def _initializer(lats, lons, data):
 
 
 class SensorMapGeometryTransformer3D(object):
-    def __init__(self, lons, lats, resamp_alg='nearest', radius_of_influence=30, **opts):
-        # type: (np.ndarray, np.ndarray, str, int, Any) -> None
+    def __init__(self, lons, lats, resamp_alg='nearest', radius_of_influence=30, mp_alg='auto', **opts):
+        # type: (np.ndarray, np.ndarray, str, int, str, Any) -> None
         """Get an instance of SensorMapGeometryTransformer.
 
         :param lons:    3D longitude array corresponding to the 3D sensor geometry array
@@ -829,6 +830,10 @@ class SensorMapGeometryTransformer3D(object):
             - resamp_alg:           resampling algorithm ('nearest', 'bilinear', 'gauss', 'custom')
             - radius_of_influence:  <float> Cut off distance in meters (default: 30)
                                     NOTE: keyword is named 'radius' in case of bilinear resampling
+            - mp_alg                multiprocessing algorithm
+                                    'bands': parallelize over bands using multiprocessing lib
+                                    'tiles': parallelize over tiles using OpenMP
+                                    'auto': automatically choose the algorithm
             - sigmas:               <list of floats or float> [ONLY 'gauss'] List of sigmas to use for the gauss
                                     weighting of each channel 1 to k, w_k = exp(-dist^2/sigma_k^2). If only one channel
                                     is resampled sigmas is a single float value.
@@ -867,7 +872,14 @@ class SensorMapGeometryTransformer3D(object):
         #      bands: multiprocessing uses multiprocessing.Pool, implemented in to_map_geometry / to_sensor_geometry
         #      tiles: multiprocessing uses OpenMP implemented in pykdtree which is used by pyresample
         self.opts['nprocs'] = opts.get('nprocs', multiprocessing.cpu_count())
-        self.mp_alg = 'bands' if self.lons.shape[2] >= opts['nprocs'] else 'tiles'
+        self.mp_alg = ('bands' if self.lons.shape[2] >= opts['nprocs'] else 'tiles') if mp_alg == 'auto' else mp_alg
+
+        # override self.mp_alg if SensorMapGeometryTransformer3D is called by nosetest or unittest
+        is_called_by_nose_cmd = 'nosetest' in sys.argv[0]
+        if self.opts['nprocs'] > 1 and self.mp_alg == 'bands' and is_called_by_nose_cmd:
+            warnings.warn("mp_alg='bands' causes deadlocks if SensorMapGeometryTransformer3D is called within a "
+                          "nosetest console call. Using mp_alg='tiles'.")
+            self.mp_alg = 'tiles'
 
     @staticmethod
     def _to_map_geometry_2D(kwargs_dict):

py_tools_ds/version.py

-__version__ = '0.14.16'
-__versionalias__ = '20190322_02'
+__version__ = '0.14.17'
+__versionalias__ = '20190322_03'

tests/test_geo/test_raster/test_reproject.py

@@ -120,7 +120,9 @@ class Test_SensorMapGeometryTransformer3D(TestCase):
     def test_to_map_geometry_lonlat_3D_geolayer(self):
         SMGT = SensorMapGeometryTransformer3D(lons=self.lons_3D,
                                               lats=self.lats_3D,
-                                              resamp_alg='nearest')
+                                              # resamp_alg='nearest',
+                                              resamp_alg='bilinear',
+                                              )
 
         # to Lon/Lat
         data_mapgeo_3D, dem_gt, dem_prj = SMGT.to_map_geometry(self.data_sensor_geo_3D, tgt_prj=4326)
@@ -137,7 +139,8 @@ class Test_SensorMapGeometryTransformer3D(TestCase):
     def test_to_sensor_geometry(self):
         SMGT = SensorMapGeometryTransformer3D(lons=self.lons_3D,
                                               lats=self.lats_3D,
-                                              resamp_alg='nearest')
+                                              resamp_alg='nearest',
+                                              )
         dem_sensors_geo = SMGT.to_sensor_geometry(self.data_map_geo_3D,
                                                   src_prj=32632, src_extent=self.dem_area_extent_coarse_subset_utm)
         self.assertIsInstance(dem_sensors_geo, np.ndarray)