Commit c5756a9c authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Merge branch 'bugfix/fix_openmp_deadlock' into 'master'

Bugfix/fix openmp deadlock

Closes #6

See merge request !5
parents 55a65077 af8a3c14
Pipeline #12324 passed with stages
in 1 minute and 14 seconds
......@@ -56,7 +56,7 @@ test_sensormapgeo_install:
# resolve some requirements with conda
# - conda config --set channel_priority strict # otherwise gdal or libgdal may be installed from defaults channel
- conda install --yes -q -c conda-forge numpy gdal pyresample pyqt scikit-image pyproj lxml geopandas ipython
- conda install --yes -q -c conda-forge numpy gdal pyresample pyqt scikit-image pyproj lxml geopandas ipython _openmp_mutex=*=*llvm*
# run installer
- python setup.py install
......
......@@ -2,6 +2,14 @@
History
=======
0.4.4 (2020-09-04)
------------------
* Fixed issue #6 (Deadlock within SensorMapGeometryTransformer3D when running in multiprocessing for resampling
algorithms 'near' and 'gauss'.)
* Added pebble to pip requirements.
0.4.3 (2020-09-02)
------------------
......
......@@ -100,9 +100,8 @@ class SensorMapGeometryTransformer(object):
# NOTE: If pykdtree is built with OpenMP support (default) the number of threads is controlled with the
# standard OpenMP environment variable OMP_NUM_THREADS. The nprocs argument has no effect on pykdtree.
os.environ['OMP_NUM_THREADS'] = '%d' % (self.opts['nprocs'] if 'nprocs' in self.opts else 1)
if 'nprocs' in self.opts:
if self.opts['nprocs'] > 1:
os.environ['OMP_NUM_THREADS'] = '%d' % opts['nprocs']
del self.opts['nprocs']
self.lats = lats
......
......@@ -26,7 +26,11 @@
from typing import Union, Tuple
import multiprocessing
from concurrent.futures import TimeoutError as Timeout
from warnings import warn
import platform
from pebble import ProcessPool, ProcessExpired
import numpy as np
from pyproj import CRS
from py_tools_ds.geo.coord_trafo import transform_any_prj
......@@ -221,11 +225,31 @@ class SensorMapGeometryTransformer3D(object):
) for band in range(data.shape[2])]
if self.opts['nprocs'] > 1 and self.mp_alg == 'bands':
# NOTE: We use imap here as it directly returns the results when available (works like a generator).
# NOTE: The pebble ProcessPool directly returns the results when available (works like a generator).
# This saves a lot of memory compared with map. We also don't use an initializer to share the input
# arrays because this would allocate the memory for the input arrays of all bands at once.
with multiprocessing.Pool(self.opts['nprocs']) as pool:
result = [res for res in pool.imap_unordered(self._to_map_geometry_2D, args)]
# NOTE: Use a multiprocessing imap iterator here when the OpenMP is finally fixed in the pyresample side:
# with multiprocessing.Pool(self.opts['nprocs']) as pool:
# return [res for res in pool.imap_unordered(self._to_map_geometry_2D, args)]
try:
# this may cause a deadlock with the GNU OpenMP build, thus each WORKER has a timeout of 10 seconds
with ProcessPool() as pool:
future = pool.map(self._to_map_geometry_2D, args, timeout=10)
result = [i for i in future.result()]
except (Timeout, ProcessExpired):
# use mp_alg='tiles' instead which uses OpenMP under the hood
msg = "Switched multiprocessing algorithm from 'bands' to 'tiles' due to a timeout in 'bands' mode. "
if platform.system() == 'Linux':
msg += "Consider using the LLVM instead of the GNU build of OpenMP to fix this issue (install, " \
"e.g., by 'conda install -c conda-forge _openmp_mutex=*=1_llvm'."
warn(msg, RuntimeWarning)
init_opts = self.opts.copy()
for argset in args:
argset.update(dict(init_opts=init_opts))
result = [self._to_map_geometry_2D(argsdict) for argsdict in args]
else:
result = [self._to_map_geometry_2D(argsdict) for argsdict in args]
......
......@@ -22,6 +22,6 @@
# You should have received a copy of the GNU Lesser General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
__version__ = '0.4.3'
__versionalias__ = '20200902.01'
__version__ = '0.4.4'
__versionalias__ = '20200904.01'
__author__ = 'Daniel Scheffler'
......@@ -37,7 +37,7 @@ version = {}
with open("sensormapgeo/version.py") as version_file:
exec(version_file.read(), version)
requirements = ['numpy', 'gdal', 'pyresample>=1.11.0', 'py_tools_ds>=0.14.26', 'pyproj>=2.2']
requirements = ['numpy', 'gdal', 'pyresample>=1.11.0', 'py_tools_ds>=0.14.26', 'pyproj>=2.2', 'pebble']
setup_requirements = []
......@@ -46,7 +46,7 @@ test_requirements = ['coverage', 'nose', 'nose-htmloutput', 'rednose']
setup(
author="Daniel Scheffler",
author_email='daniel.scheffler@gfz-potsdam.de',
python_requires='>=3.5',
python_requires='>=3.6',
classifiers=[
'Development Status :: 2 - Pre-Alpha',
'Intended Audience :: Developers',
......
......@@ -9,6 +9,7 @@ dependencies:
- numpy
- gdal
- pyresample>=1.11.0
- _openmp_mutex=*=*llvm* [Unix] # fixes a deadlock (https://gitext.gfz-potsdam.de/EnMAP/sensormapgeo/-/issues/6)
- pyproj>=2.2
# py_tools_ds
......@@ -20,6 +21,8 @@ dependencies:
- pip:
- py_tools_ds>=0.14.26
- pebble
- sphinx-argparse
- flake8
- pycodestyle
......
......@@ -39,6 +39,7 @@ from sensormapgeo import SensorMapGeometryTransformer, SensorMapGeometryTransfor
tests_path = os.path.abspath(os.path.join(__path__[0], "..", "tests"))
rsp_algs = ['nearest', 'bilinear', 'gauss']
mp_algs = ['bands', 'tiles']
class Test_SensorMapGeometryTransformer(TestCase):
......@@ -194,10 +195,12 @@ class Test_SensorMapGeometryTransformer3D(TestCase):
def test_to_map_geometry_lonlat_3D_geolayer(self):
for rsp_alg in rsp_algs:
for mp_alg in mp_algs:
SMGT = SensorMapGeometryTransformer3D(lons=self.lons_3D,
lats=self.lats_3D,
# resamp_alg='nearest',
resamp_alg=rsp_alg,
mp_alg=mp_alg
)
# to Lon/Lat
......@@ -225,10 +228,12 @@ class Test_SensorMapGeometryTransformer3D(TestCase):
def test_to_map_geometry_utm_3D_geolayer(self):
for rsp_alg in rsp_algs:
for mp_alg in mp_algs:
SMGT = SensorMapGeometryTransformer3D(lons=self.lons_3D,
lats=self.lats_3D,
# resamp_alg='nearest',
resamp_alg=rsp_alg,
mp_alg=mp_alg
)
# to Lon/Lat
......@@ -260,12 +265,15 @@ class Test_SensorMapGeometryTransformer3D(TestCase):
def test_to_sensor_geometry(self):
for rsp_alg in rsp_algs:
for mp_alg in mp_algs:
SMGT = SensorMapGeometryTransformer3D(lons=self.lons_3D,
lats=self.lats_3D,
resamp_alg=rsp_alg,
mp_alg=mp_alg
)
dem_sensors_geo = SMGT.to_sensor_geometry(self.data_map_geo_3D,
src_prj=32632, src_extent=self.dem_area_extent_coarse_subset_utm)
src_prj=32632,
src_extent=self.dem_area_extent_coarse_subset_utm)
self.assertIsInstance(dem_sensors_geo, np.ndarray)
self.assertEqual(dem_sensors_geo.shape, (150, 1000, 2))
self.assertEqual(self.data_map_geo_3D.dtype, dem_sensors_geo.dtype)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment