From ef68c8626e135baccda5b7823388e899a184b780 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Fri, 22 Mar 2019 17:33:29 +0100 Subject: [PATCH 1/5] Added parameter 'mp_alg'. --- py_tools_ds/geo/raster/reproject.py | 10 +++++++--- tests/test_geo/test_raster/test_reproject.py | 8 ++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/py_tools_ds/geo/raster/reproject.py b/py_tools_ds/geo/raster/reproject.py index a07f6c5..2ef7109 100755 --- a/py_tools_ds/geo/raster/reproject.py +++ b/py_tools_ds/geo/raster/reproject.py @@ -818,8 +818,8 @@ def _initializer(lats, lons, data): class SensorMapGeometryTransformer3D(object): - def __init__(self, lons, lats, resamp_alg='nearest', radius_of_influence=30, **opts): - # type: (np.ndarray, np.ndarray, str, int, Any) -> None + def __init__(self, lons, lats, resamp_alg='nearest', radius_of_influence=30, mp_alg='auto', **opts): + # type: (np.ndarray, np.ndarray, str, int, str, Any) -> None """Get an instance of SensorMapGeometryTransformer. :param lons: 3D longitude array corresponding to the 3D sensor geometry array @@ -829,6 +829,10 @@ class SensorMapGeometryTransformer3D(object): - resamp_alg: resampling algorithm ('nearest', 'bilinear', 'gauss', 'custom') - radius_of_influence: Cut off distance in meters (default: 30) NOTE: keyword is named 'radius' in case of bilinear resampling + - mp_alg multiprocessing algorithm + 'bands': parallelize over bands using multiprocessing lib + 'tiles': parallelize over tiles using OpenMP + 'auto': automatically choose the algorithm - sigmas: [ONLY 'gauss'] List of sigmas to use for the gauss weighting of each channel 1 to k, w_k = exp(-dist^2/sigma_k^2). If only one channel is resampled sigmas is a single float value. @@ -867,7 +871,7 @@ class SensorMapGeometryTransformer3D(object): # bands: multiprocessing uses multiprocessing.Pool, implemented in to_map_geometry / to_sensor_geometry # tiles: multiprocessing uses OpenMP implemented in pykdtree which is used by pyresample self.opts['nprocs'] = opts.get('nprocs', multiprocessing.cpu_count()) - self.mp_alg = 'bands' if self.lons.shape[2] >= opts['nprocs'] else 'tiles' + self.mp_alg = ('bands' if self.lons.shape[2] >= opts['nprocs'] else 'tiles') if mp_alg == 'auto' else mp_alg @staticmethod def _to_map_geometry_2D(kwargs_dict): diff --git a/tests/test_geo/test_raster/test_reproject.py b/tests/test_geo/test_raster/test_reproject.py index dd02c73..01a49f7 100644 --- a/tests/test_geo/test_raster/test_reproject.py +++ b/tests/test_geo/test_raster/test_reproject.py @@ -120,7 +120,9 @@ class Test_SensorMapGeometryTransformer3D(TestCase): def test_to_map_geometry_lonlat_3D_geolayer(self): SMGT = SensorMapGeometryTransformer3D(lons=self.lons_3D, lats=self.lats_3D, - resamp_alg='nearest') + resamp_alg='nearest', + mp_alg='tiles' # 'bands', i.e., Pool() causes problems with nosetests + ) # to Lon/Lat data_mapgeo_3D, dem_gt, dem_prj = SMGT.to_map_geometry(self.data_sensor_geo_3D, tgt_prj=4326) @@ -137,7 +139,9 @@ class Test_SensorMapGeometryTransformer3D(TestCase): def test_to_sensor_geometry(self): SMGT = SensorMapGeometryTransformer3D(lons=self.lons_3D, lats=self.lats_3D, - resamp_alg='nearest') + resamp_alg='nearest', + mp_alg='tiles' # 'bands', i.e., Pool() causes problems with nosetests) + ) dem_sensors_geo = SMGT.to_sensor_geometry(self.data_map_geo_3D, src_prj=32632, src_extent=self.dem_area_extent_coarse_subset_utm) self.assertIsInstance(dem_sensors_geo, np.ndarray) -- GitLab From 92636c0ddd6a34abd4f7fec24f46ef009dd80f73 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Fri, 22 Mar 2019 18:32:10 +0100 Subject: [PATCH 2/5] Force mp_alg = 'tiles' in case SensorMapGeometryTransformer3D is called within a nosetest. --- py_tools_ds/geo/raster/reproject.py | 8 ++++++++ tests/test_geo/test_raster/test_reproject.py | 5 ++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/py_tools_ds/geo/raster/reproject.py b/py_tools_ds/geo/raster/reproject.py index 2ef7109..518e19a 100755 --- a/py_tools_ds/geo/raster/reproject.py +++ b/py_tools_ds/geo/raster/reproject.py @@ -5,6 +5,7 @@ import multiprocessing import os from tempfile import TemporaryDirectory from typing import Union, Tuple, List, Any # noqa: F401 +import inspect # custom try: @@ -873,6 +874,13 @@ class SensorMapGeometryTransformer3D(object): self.opts['nprocs'] = opts.get('nprocs', multiprocessing.cpu_count()) self.mp_alg = ('bands' if self.lons.shape[2] >= opts['nprocs'] else 'tiles') if mp_alg == 'auto' else mp_alg + # override self.mp_alg if SensorMapGeometryTransformer3D is called by nosetests + is_called_by_nose = any(x[0].f_globals['__name__'].startswith('nose.') for x in inspect.stack()) + if self.opts['nprocs'] > 1 and self.mp_alg == 'bands' and is_called_by_nose: + warnings.warn("mp_alg='bands' causes deadlocks if SensorMapGeometryTransformer3D is called within a " + "nosetest. Using mp_alg='tiles'.") + self.mp_alg = 'tiles' + @staticmethod def _to_map_geometry_2D(kwargs_dict): # type: (dict) -> Tuple[np.ndarray, tuple, str, int] diff --git a/tests/test_geo/test_raster/test_reproject.py b/tests/test_geo/test_raster/test_reproject.py index 01a49f7..ed9786d 100644 --- a/tests/test_geo/test_raster/test_reproject.py +++ b/tests/test_geo/test_raster/test_reproject.py @@ -120,8 +120,8 @@ class Test_SensorMapGeometryTransformer3D(TestCase): def test_to_map_geometry_lonlat_3D_geolayer(self): SMGT = SensorMapGeometryTransformer3D(lons=self.lons_3D, lats=self.lats_3D, - resamp_alg='nearest', - mp_alg='tiles' # 'bands', i.e., Pool() causes problems with nosetests + # resamp_alg='nearest', + resamp_alg='bilinear', ) # to Lon/Lat @@ -140,7 +140,6 @@ class Test_SensorMapGeometryTransformer3D(TestCase): SMGT = SensorMapGeometryTransformer3D(lons=self.lons_3D, lats=self.lats_3D, resamp_alg='nearest', - mp_alg='tiles' # 'bands', i.e., Pool() causes problems with nosetests) ) dem_sensors_geo = SMGT.to_sensor_geometry(self.data_map_geo_3D, src_prj=32632, src_extent=self.dem_area_extent_coarse_subset_utm) -- GitLab From 8795c815a0823d8fb0b642a3867f15b167005e79 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Fri, 22 Mar 2019 18:50:27 +0100 Subject: [PATCH 3/5] Also include unittest in mp_alg overriding. --- py_tools_ds/geo/raster/reproject.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/py_tools_ds/geo/raster/reproject.py b/py_tools_ds/geo/raster/reproject.py index 518e19a..b6532aa 100755 --- a/py_tools_ds/geo/raster/reproject.py +++ b/py_tools_ds/geo/raster/reproject.py @@ -874,9 +874,10 @@ class SensorMapGeometryTransformer3D(object): self.opts['nprocs'] = opts.get('nprocs', multiprocessing.cpu_count()) self.mp_alg = ('bands' if self.lons.shape[2] >= opts['nprocs'] else 'tiles') if mp_alg == 'auto' else mp_alg - # override self.mp_alg if SensorMapGeometryTransformer3D is called by nosetests + # override self.mp_alg if SensorMapGeometryTransformer3D is called by nosetest or unittest is_called_by_nose = any(x[0].f_globals['__name__'].startswith('nose.') for x in inspect.stack()) - if self.opts['nprocs'] > 1 and self.mp_alg == 'bands' and is_called_by_nose: + is_called_by_unittest = any(x[0].f_globals['__name__'].startswith('unittest.') for x in inspect.stack()) + if self.opts['nprocs'] > 1 and self.mp_alg == 'bands' and (is_called_by_nose or is_called_by_unittest): warnings.warn("mp_alg='bands' causes deadlocks if SensorMapGeometryTransformer3D is called within a " "nosetest. Using mp_alg='tiles'.") self.mp_alg = 'tiles' -- GitLab From 7437dbae9eb83e3106cb0c7e45562b4817e18145 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Fri, 22 Mar 2019 19:03:55 +0100 Subject: [PATCH 4/5] Restricted mp_alg overriding to nosetest console calls only. --- py_tools_ds/geo/raster/reproject.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/py_tools_ds/geo/raster/reproject.py b/py_tools_ds/geo/raster/reproject.py index b6532aa..dabd802 100755 --- a/py_tools_ds/geo/raster/reproject.py +++ b/py_tools_ds/geo/raster/reproject.py @@ -5,7 +5,7 @@ import multiprocessing import os from tempfile import TemporaryDirectory from typing import Union, Tuple, List, Any # noqa: F401 -import inspect +import sys # custom try: @@ -875,11 +875,10 @@ class SensorMapGeometryTransformer3D(object): self.mp_alg = ('bands' if self.lons.shape[2] >= opts['nprocs'] else 'tiles') if mp_alg == 'auto' else mp_alg # override self.mp_alg if SensorMapGeometryTransformer3D is called by nosetest or unittest - is_called_by_nose = any(x[0].f_globals['__name__'].startswith('nose.') for x in inspect.stack()) - is_called_by_unittest = any(x[0].f_globals['__name__'].startswith('unittest.') for x in inspect.stack()) - if self.opts['nprocs'] > 1 and self.mp_alg == 'bands' and (is_called_by_nose or is_called_by_unittest): + is_called_by_nose_cmd = 'nosetest' in sys.argv[0] + if self.opts['nprocs'] > 1 and self.mp_alg == 'bands' and is_called_by_nose_cmd: warnings.warn("mp_alg='bands' causes deadlocks if SensorMapGeometryTransformer3D is called within a " - "nosetest. Using mp_alg='tiles'.") + "nosetest console call. Using mp_alg='tiles'.") self.mp_alg = 'tiles' @staticmethod -- GitLab From 02973a4b0a86babaf6293c4fd0278538304948fa Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Fri, 22 Mar 2019 19:09:36 +0100 Subject: [PATCH 5/5] Updated version info. --- py_tools_ds/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py_tools_ds/version.py b/py_tools_ds/version.py index cd2074a..1ba72bf 100644 --- a/py_tools_ds/version.py +++ b/py_tools_ds/version.py @@ -1,2 +1,2 @@ -__version__ = '0.14.16' -__versionalias__ = '20190322_02' +__version__ = '0.14.17' +__versionalias__ = '20190322_03' -- GitLab