diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8a0f230c6661b25c1c4724fecec5d1d8263b01ad..b290d9dc844a5c707b008c6597a713eea32de278 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,16 @@ test_enpt: - source /root/miniconda3/bin/activate enpt - export GDAL_DATA=/root/miniconda3/envs/enpt/share/gdal - export PYTHONPATH=$PYTHONPATH:/root # /root <- here are the sicor tables + + # update sicor + # - conda install -y -q -c conda-forge basemap + # - rm -rf context/sicor + # - git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git ./context/sicor + # - cd ./context/sicor + # - make download-tables + # - python setup.py install + # - cd ../../ + # run nosetests - make nosetests # test are called here # create the docs @@ -36,6 +46,7 @@ test_styles: - make lint artifacts: paths: + - tests/data/test_outputs/*.log - tests/linting/flake8.log - tests/linting/pycodestyle.log - tests/linting/pydocstyle.log @@ -48,16 +59,28 @@ test_enpt_install: - source /root/miniconda3/bin/activate - conda create -y -q --name enpt_test python=3 - source activate enpt_test + # install some dependencies that cause trouble when installed via pip - conda install -y -c conda-forge scipy # scikit-image, matplotlib + # install not pip-installable deps of geoarray - conda install -y -c conda-forge numpy scikit-image matplotlib pandas gdal rasterio pyproj basemap shapely - conda install -y -c conda-forge 'icu=58.*' lxml # fixes bug for conda-forge gdal build + + # install sicor + - conda install -y -q -c conda-forge pygrib h5py pytables pyfftw numba llvmlite + - rm -rf context/sicor + - git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git ./context/sicor + - cd ./context/sicor + - make install + - cd ../../ + # install enpt - make install - cd .. - pwd - ls + # test importability - python -c "import enpt; print(enpt)" - python -c "from enpt.model.images import EnMAPL1Product_SensorGeo" diff --git a/README.rst b/README.rst index 70f97e150b6aff4119fa19686b4aa82c52ca15a6..b5da1c238b3d0a38cc51be60b447a6702a55f7d3 100644 --- a/README.rst +++ b/README.rst @@ -6,7 +6,7 @@ EnPT - EnMAP Processing Tools Please check the documentation_ for usage and in depth information. -Licence +License ------- Free software: GNU General Public License v3 @@ -25,7 +25,7 @@ See also the latest coverage_ report and the nosetests_ HTML report. Credits ---------- +------- This software was funded from BMBF under EnMAP ... diff --git a/enpt/execution/controller.py b/enpt/execution/controller.py index d8d051bf967e4296d0cb4bc7d43843d9eafa1e41..a6c048866ccfef3a29d3295f2b51cf687dd0ebff 100644 --- a/enpt/execution/controller.py +++ b/enpt/execution/controller.py @@ -11,7 +11,7 @@ import warnings from ..options.config import EnPTConfig from ..io.reader import L1B_Reader -from ..processors.radiometric_transform import Radiometric_Transformer +from ..processors import Radiometric_Transformer from ..model.images import EnMAPL1Product_SensorGeo @@ -57,7 +57,7 @@ class EnPT_Controller(object): if not os.path.isdir(path_enmap_image) and \ not (os.path.exists(path_enmap_image) and path_enmap_image.endswith('.zip')): raise ValueError("The parameter 'path_enmap_image' must be a directory or the path to an existing zip " - "archive.") + "archive. Received %s." % path_enmap_image) # extract L1B image archive if needed if path_enmap_image.endswith('.zip'): @@ -79,9 +79,12 @@ class EnPT_Controller(object): # run transformation to TOARef self.L1_obj = RT.transform_TOARad2TOARef(self.L1_obj) + def run_geometry_processor(self): + pass + def run_atmospheric_correction(self): """Run atmospheric correction only.""" - pass + self.L1_obj.run_AC() def write_output(self): if self.cfg.output_dir: @@ -90,7 +93,8 @@ class EnPT_Controller(object): def run_all_processors(self): """Run all processors at once.""" try: - self.run_toaRad2toaRef() + # self.run_toaRad2toaRef() + self.run_geometry_processor() self.run_atmospheric_correction() self.write_output() finally: diff --git a/enpt/model/images.py b/enpt/model/images.py index a6551ed3ed9c69f021903050759f2bbd7c8afd8a..c1020b52ff7da72834cf085262bfd7e2fa8589fd 100644 --- a/enpt/model/images.py +++ b/enpt/model/images.py @@ -47,7 +47,7 @@ class _EnMAP_Image(object): self.paths = SimpleNamespace() # protected attributes - self._arr = None + self._data = None self._mask_nodata = None self._mask_clouds = None self._mask_clouds_confidence = None @@ -61,7 +61,7 @@ class _EnMAP_Image(object): self.basename = '' @property - def data(self): + def data(self) -> GeoArray: """Return the actual EnMAP image data. Bundled with all the corresponding metadata. @@ -105,26 +105,25 @@ class _EnMAP_Image(object): :return: instance of geoarray.GeoArray """ - # TODO this must return a subset if self.subset is not None - return self._arr + return self._data @data.setter def data(self, *geoArr_initArgs): if geoArr_initArgs[0] is not None: # TODO this must be able to handle subset inputs in tiled processing - if self._arr and len(geoArr_initArgs[0]) and isinstance(geoArr_initArgs[0], np.ndarray): - self._arr = GeoArray(geoArr_initArgs[0], geotransform=self._arr.gt, projection=self._arr.prj) + if self._data and len(geoArr_initArgs[0]) and isinstance(geoArr_initArgs[0], np.ndarray): + self._data = GeoArray(geoArr_initArgs[0], geotransform=self._data.gt, projection=self._data.prj) else: - self._arr = GeoArray(*geoArr_initArgs) + self._data = GeoArray(*geoArr_initArgs) else: del self.data @data.deleter def data(self): - self._arr = None + self._data = None @property - def mask_clouds(self): + def mask_clouds(self) -> GeoArray: """Return the cloud mask. Bundled with all the corresponding metadata. @@ -155,7 +154,7 @@ class _EnMAP_Image(object): self._mask_clouds = None @property - def mask_clouds_confidence(self): + def mask_clouds_confidence(self) -> GeoArray: """Return pixelwise information on the cloud mask confidence. Bundled with all the corresponding metadata. @@ -190,7 +189,7 @@ class _EnMAP_Image(object): self._mask_clouds_confidence = None @property - def dem(self): + def dem(self) -> GeoArray: """Return an SRTM DEM in the exact dimension an pixel grid of self.arr. :return: geoarray.GeoArray @@ -220,7 +219,7 @@ class _EnMAP_Image(object): self._dem = None @property - def ac_errors(self): + def ac_errors(self) -> GeoArray: """Return error information calculated by the atmospheric correction. :return: geoarray.GeoArray @@ -251,7 +250,7 @@ class _EnMAP_Image(object): self._ac_errors = None @property - def deadpixelmap(self): + def deadpixelmap(self) -> GeoArray: """Return the dead pixel map. Bundled with all the corresponding metadata. Dimensions: (bands x columns). @@ -472,6 +471,11 @@ class EnMAPL1Product_SensorGeo(object): if self.swir.detector_meta.unitcode != 'TOARad': self.swir.DN2TOARadiance() + def run_AC(self): + from ..processors import AtmosphericCorrector + AC = AtmosphericCorrector(config=self.cfg) + AC.run_ac(self) + def save(self, outdir: str, suffix="") -> str: """Save this product to disk using almost the same format as for reading. @@ -553,7 +557,7 @@ class EnMAP_Detector_MapGeo(_EnMAP_Image): super(EnMAP_Detector_MapGeo, self).__init__() @property - def mask_nodata(self): + def mask_nodata(self) -> GeoArray: """Return the no data mask. Bundled with all the corresponding metadata. @@ -587,7 +591,7 @@ class EnMAP_Detector_MapGeo(_EnMAP_Image): def mask_nodata(self): self._mask_nodata = None - def calc_mask_nodata(self, fromBand=None, overwrite=False): + def calc_mask_nodata(self, fromBand=None, overwrite=False) -> GeoArray: """Calculate a no data mask with (values: 0=nodata; 1=data). :param fromBand: index of the band to be used (if None, all bands are used) diff --git a/enpt/model/metadata.py b/enpt/model/metadata.py index cb32a0a66060a98a632a0c662fa94427adce264d..2861d4b925587c11089d4d8ecedf44d16330abbf 100644 --- a/enpt/model/metadata.py +++ b/enpt/model/metadata.py @@ -65,6 +65,7 @@ class EnMAP_Metadata_L1B_Detector_SensorGeo(object): self.geom_view_azimuth = None # type: float # viewing azimuth angle self.geom_sun_zenith = None # type: float # sun zenith angle self.geom_sun_azimuth = None # type: float # sun azimuth angle + self.mu_sun = None # type: float # needed by SICOR for TOARad > TOARef conversion self.lat_UL_UR_LL_LR = None # type: list # latitude coordinates for UL, UR, LL, LR self.lon_UL_UR_LL_LR = None # type: list # longitude coordinates for UL, UR, LL, LR self.lats = None # type: np.ndarray # 2D array of latitude coordinates according to given lon/lat sampling @@ -106,6 +107,7 @@ class EnMAP_Metadata_L1B_Detector_SensorGeo(object): xml.findall("%s/illumination_geometry/zenith_angle" % lbl)[0].text.split()[0]) self.geom_sun_azimuth = np.float( xml.findall("%s/illumination_geometry/azimuth_angle" % lbl)[0].text.split()[0]) + self.mu_sun = np.cos(np.deg2rad(self.geom_sun_zenith)) self.lat_UL_UR_LL_LR = \ [float(xml.findall("%s/geometry/bounding_box/%s_northing" % (lbl, corner))[0].text.split()[0]) for corner in ("UL", "UR", "LL", "LR")] diff --git a/enpt/options/config.py b/enpt/options/config.py index c20b9ba390401a0b4404a39cd33a5535e6fcc566..9d016ab7f9e3c0145f9aa6b063d9d1542c47756f 100644 --- a/enpt/options/config.py +++ b/enpt/options/config.py @@ -17,6 +17,8 @@ from collections import OrderedDict, Mapping import numpy as np from multiprocessing import cpu_count +import sicor + from .options_schema import \ enpt_schema_input, \ enpt_schema_config_output, \ @@ -31,6 +33,14 @@ path_enptlib = os.path.dirname(pkgutil.get_loader("enpt").path) path_options_default = os.path.join(path_enptlib, 'options', 'options_default.json') +config_for_testing = dict( + path_l1b_enmap_image=os.path.abspath( + os.path.join(path_enptlib, '..', 'tests', 'data', 'EnMAP_Level_1B', 'AlpineTest1_CWV2_SM0.zip')), + log_level='DEBUG', + output_dir=os.path.join(path_enptlib, '..', 'tests', 'data', 'test_outputs') +) + + class EnPTConfig(object): def __init__(self, json_config='', **user_opts): """Create a job configuration. @@ -72,7 +82,7 @@ class EnPTConfig(object): # output options # ################## - self.output_dir = self.absPath(gp('output_dir')) + self.output_dir = self.absPath(gp('output_dir', fallback=os.path.abspath(os.path.curdir))) ########################### # processor configuration # @@ -81,6 +91,7 @@ class EnPTConfig(object): # toa_ref self.path_earthSunDist = self.absPath(gp('path_earthSunDist')) self.path_solar_irr = self.absPath(gp('path_solar_irr')) + self.scale_factor_toa_ref = gp('scale_factor_toa_ref') # geometry self.enable_keystone_correction = gp('enable_keystone_correction') @@ -88,8 +99,10 @@ class EnPTConfig(object): self.path_reference_image = gp('path_reference_image') # atmospheric_correction + self.sicor_cache_dir = gp('sicor_cache_dir', fallback=sicor.__path__[0]) self.auto_download_ecmwf = gp('auto_download_ecmwf') self.enable_cloud_screening = gp('enable_cloud_screening') + self.scale_factor_boa_ref = gp('scale_factor_boa_ref'), # smile self.run_smile_P = gp('run_smile_P') diff --git a/enpt/options/options_default.json b/enpt/options/options_default.json index fc9476f6db2a32b55062851c298e7daeab10584f..598bded633c23191756652fe3ee8687cc9216b50 100644 --- a/enpt/options/options_default.json +++ b/enpt/options/options_default.json @@ -16,7 +16,8 @@ "processors": { "toa_ref": { "path_earthSunDist": "./resources/earth_sun_distance/Earth_Sun_distances_per_day_edited__1980_2030.csv", /*input path of the earth sun distance model*/ - "path_solar_irr": "./resources/solar_irradiance/SUNp1fontenla__350-2500nm_@0.1nm_converted.txt" /*input path of the solar irradiance model*/ + "path_solar_irr": "./resources/solar_irradiance/SUNp1fontenla__350-2500nm_@0.1nm_converted.txt", /*input path of the solar irradiance model*/ + "scale_factor_toa_ref": 10000 /*scale factor to be applied to TOA reflectance result*/ }, "geometry": { @@ -26,8 +27,12 @@ }, "atmospheric_correction": { + "sicor_cache_dir": "", /*directory to be used to stored sicor cache files + NOTE: SICOR stores intermediate results there that need to computed only once + for atmospheric correction of multiple EnMAP images. (default: 'auto')*/ "auto_download_ecmwf": false, - "enable_cloud_screening": false + "enable_cloud_screening": false, + "scale_factor_boa_ref": 10000 /*scale factor to be applied to BOA reflectance result*/ }, "smile": { diff --git a/enpt/options/options_schema.py b/enpt/options/options_schema.py index b10a7151040562a4d2d426d89a03fd4900cbb029..7816d618acd95ea5cf2ccc947b8bd22ed6344131 100644 --- a/enpt/options/options_schema.py +++ b/enpt/options/options_schema.py @@ -41,6 +41,7 @@ enpt_schema_input = dict( atmospheric_correction=dict( type='dict', required=False, schema=dict( + sicor_cache_dir=dict(type='string', required=False), auto_download_ecmwf=dict(type='boolean', required=False), enable_cloud_screening=dict(type='boolean', required=False), )), @@ -82,6 +83,7 @@ parameter_mapping = dict( # processors > toa_ref path_earthSunDist=('processors', 'toa_ref', 'path_earthSunDist'), path_solar_irr=('processors', 'toa_ref', 'path_solar_irr'), + scale_factor_toa_ref=('processors', 'toa_ref', 'scale_factor_toa_ref'), # processors > geometry enable_keystone_correction=('processors', 'geometry', 'enable_keystone_correction'), @@ -89,8 +91,10 @@ parameter_mapping = dict( path_reference_image=('processors', 'geometry', 'path_reference_image'), # processors > atmospheric_correction + sicor_cache_dir=('processors', 'atmospheric_correction', 'sicor_cache_dir'), auto_download_ecmwf=('processors', 'atmospheric_correction', 'auto_download_ecmwf'), enable_cloud_screening=('processors', 'atmospheric_correction', 'enable_cloud_screening'), + scale_factor_boa_ref=('processors', 'atmospheric_correction', 'scale_factor_boa_ref'), # processors > smile run_smile_P=('processors', 'smile', 'run_processor'), diff --git a/enpt/processors/__init__.py b/enpt/processors/__init__.py index ffcaa00784346cb00d3c33f54ed70f5053727758..5ce1f347d7370a8350f3b231896af1bc4756e3c8 100644 --- a/enpt/processors/__init__.py +++ b/enpt/processors/__init__.py @@ -1,2 +1,10 @@ # -*- coding: utf-8 -*- """EnPT 'processors' module containing all EnPT processor sub-modules.""" + +from .radiometric_transform.radiometric_transform import Radiometric_Transformer +from .atmospheric_correction.atmospheric_correction import AtmosphericCorrector + +__all__ = [ + "Radiometric_Transformer", + "AtmosphericCorrector" +] diff --git a/enpt/processors/atmospheric_correction/__init__.py b/enpt/processors/atmospheric_correction/__init__.py index b6134021198291185cf0ebaf2fe443fd1592cdc9..7e1cd0163368b1d06e9f22e8c3564ef70c3305c6 100644 --- a/enpt/processors/atmospheric_correction/__init__.py +++ b/enpt/processors/atmospheric_correction/__init__.py @@ -1,2 +1,6 @@ # -*- coding: utf-8 -*- """EnPT 'atmospheric correction module.""" + +from .atmospheric_correction import AtmosphericCorrector + +__all__ = ['AtmosphericCorrector'] diff --git a/enpt/processors/atmospheric_correction/atmospheric_correction.py b/enpt/processors/atmospheric_correction/atmospheric_correction.py index a55ee1726475bcf622563d18b194615ebc0ae5b1..d263a5cb05a842a3298814f1c40f4cd85244f877 100644 --- a/enpt/processors/atmospheric_correction/atmospheric_correction.py +++ b/enpt/processors/atmospheric_correction/atmospheric_correction.py @@ -3,3 +3,69 @@ Performs the atmospheric correction of EnMAP L1B data. """ +import pprint +import numpy as np + +from sicor.sicor_enmap import sicor_ac_enmap +from sicor.options import get_options as get_ac_options + +from ...model.images import EnMAPL1Product_SensorGeo +from ...options.config import EnPTConfig +from ...utils.path_generator import get_path_ac_options + + +class AtmosphericCorrector(object): + """Class for performing atmospheric correction of EnMAP L1 images using SICOR.""" + + def __init__(self, config: EnPTConfig=None): + """Create an instance of AtmosphericCorrector.""" + self.cfg = config + + @staticmethod + def get_ac_options(buffer_dir): + path_opts = get_path_ac_options() + + try: + options = get_ac_options(path_opts) + + # adjust options + options['EnMAP']['buffer_dir'] = buffer_dir + for vv in options["RTFO"].values(): + vv["hash_formats"] = dict(spr='%.0f', + coz='%.0f,', + cwv='%.0f,', + tmp='%0f,', + tau_a='%.2f,', + vza='%.0f,') + options["ECMWF"]["path_db"] = "./ecmwf" + + return options + + except FileNotFoundError: + raise FileNotFoundError('Could not locate options file for atmospheric correction at %s.' % path_opts) + + def run_ac(self, enmap_ImageL1: EnMAPL1Product_SensorGeo) -> EnMAPL1Product_SensorGeo: + options = self.get_ac_options(buffer_dir=self.cfg.sicor_cache_dir) + enmap_ImageL1.logger.debug('AC options: \n' + pprint.pformat(options)) + + # run AC + enmap_ImageL1.logger.info("Starting atmospheric correction for VNIR and SWIR detector. " + "Source radiometric unit code is '%s'." % enmap_ImageL1.meta.vnir.unitcode) + enmap_l2a_sens_geo, state, cwv_map, ch4_map = sicor_ac_enmap(enmap_l1b=enmap_ImageL1, options=options, + logger=enmap_ImageL1.logger, debug=True) + + # join results + enmap_ImageL1.logger.info('Joining results of atmospheric correction.') + + for in_detector, out_detector in zip([enmap_ImageL1.vnir, enmap_ImageL1.swir], + [enmap_l2a_sens_geo.vnir, enmap_l2a_sens_geo.swir]): + in_detector.data = (out_detector.data[:] * self.cfg.scale_factor_boa_ref).astype(np.int16) + # NOTE: geotransform and projection are missing due to sensor geometry + + del in_detector.data_l2a # FIXME sicor sets data_l2a to float array -> not needed + del in_detector.unit # FIXME sicor sets unit to '1' -> not needed + + in_detector.detector_meta.unit = '0-%d' % self.cfg.scale_factor_boa_ref + in_detector.detector_meta.unitcode = 'BOARef' + + return enmap_ImageL1 diff --git a/enpt/processors/radiometric_transform/__init__.py b/enpt/processors/radiometric_transform/__init__.py index c0335e5880e607742ecdce93364bc9b988b8b255..53c484986288ff340b4dd1564d52446024200cf2 100644 --- a/enpt/processors/radiometric_transform/__init__.py +++ b/enpt/processors/radiometric_transform/__init__.py @@ -1,6 +1,2 @@ # -*- coding: utf-8 -*- """EnPT 'radiometric transform' module containing eveything related to radiometric transformations.""" - -from .radiometric_transform import Radiometric_Transformer - -__all__ = ['Radiometric_Transformer'] diff --git a/enpt/processors/radiometric_transform/radiometric_transform.py b/enpt/processors/radiometric_transform/radiometric_transform.py index e23bf06ad6645c1b900f853aedcbb8be94553939..a09ce72fe2b52859fcc5638188ae785838a8dcad 100644 --- a/enpt/processors/radiometric_transform/radiometric_transform.py +++ b/enpt/processors/radiometric_transform/radiometric_transform.py @@ -21,8 +21,7 @@ class Radiometric_Transformer(object): self.solarIrr = config.path_solar_irr # path of model for solar irradiance self.earthSunDist = config.path_earthSunDist # path of model for earth sun distance - @staticmethod - def transform_TOARad2TOARef(enmap_ImageL1: EnMAPL1Product_SensorGeo, scale_factor: int=10000): + def transform_TOARad2TOARef(self, enmap_ImageL1: EnMAPL1Product_SensorGeo): """Transform top-of-atmosphere radiance to top-of-atmosphere reflectance. NOTE: The following formula is used: @@ -30,7 +29,6 @@ class Radiometric_Transformer(object): (solIrr * math.cos(zenithAngleDeg)) :param enmap_ImageL1: instance of the class 'EnMAPL1Product_ImGeo' - :param scale_factor: scale factor to be applied to TOA reflectance result :return: """ for detectorName in enmap_ImageL1.detector_attrNames: @@ -41,7 +39,7 @@ class Radiometric_Transformer(object): # compute TOA reflectance constant = \ - scale_factor * math.pi * enmap_ImageL1.meta.earthSunDist ** 2 / \ + self.cfg.scale_factor_toa_ref * math.pi * enmap_ImageL1.meta.earthSunDist ** 2 / \ (math.cos(math.radians(detector.detector_meta.geom_sun_zenith))) solIrr = np.array([detector.detector_meta.solar_irrad[band] for band in detector.detector_meta.srf.bands])\ .reshape(1, 1, detector.data.bands) @@ -49,7 +47,7 @@ class Radiometric_Transformer(object): # update EnMAP image detector.data = toaRef - detector.detector_meta.unit = '0-%d' % scale_factor + detector.detector_meta.unit = '0-%d' % self.cfg.scale_factor_toa_ref detector.detector_meta.unitcode = 'TOARef' return enmap_ImageL1 diff --git a/enpt/utils/path_generator.py b/enpt/utils/path_generator.py index 0b8c6f009dfd5c51bbb43caf6a6f71f24da80354..2f7490415dd150bd5238cbbc6f00b45434944d83 100644 --- a/enpt/utils/path_generator.py +++ b/enpt/utils/path_generator.py @@ -48,3 +48,11 @@ class PathGenL1BProduct(object): def _find_in_metaxml(self, expression): return self.xml.findall(expression)[0].text.replace("\n", "").strip() + + +def get_path_ac_options() -> str: + """Returns the path of the options json file needed for atmospheric correction.""" + from sicor import options + path_ac = os.path.join(os.path.dirname(options.__file__), 'sicor_enmap_user_options.json') + + return path_ac diff --git a/requirements.txt b/requirements.txt index 74df5e22cd3455e93aec5ff1f411d83f9ceb9eb5..0688135e3afeffb27fd556ab7fb171264a010f5a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ spectral>=0.16 cerberus jsmin matplotlib +git+https://gitext.gfz-potsdam.de/EnMAP/sicor.git diff --git a/requirements_dev.txt b/requirements_dev.txt index 9147e4e727a09f724c9fe96fe4cf48eb24114859..985c960e33b05ad4d195c4bd1137681cd6cec791 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -6,5 +6,3 @@ flake8==2.6.0 tox==2.3.1 coverage==4.1 Sphinx==1.4.8 - - diff --git a/setup.py b/setup.py index 685e01803769c1a01f6291eab9de9ba5a08b64d8..21764784acd921011c35666ec79a7dba52829e54 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,7 @@ with open("enpt/version.py") as version_file: requirements = [ # put package requirements here 'numpy', 'scipy', 'geoarray>=0.7.15', 'spectral>=0.16', 'cerberus', 'jsmin', 'matplotlib' + # 'sicor', # pip install git+https://gitext.gfz-potsdam.de/EnMAP/sicor.git ] test_requirements = ['coverage', 'nose', 'nose-htmloutput', 'rednose'] diff --git a/tests/__init__.py b/tests/__init__.py index 60939742c5d817b679945cb617dd28742b832a1f..40a96afc6ff09d58a702b76e3f7dd412fe975e26 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,10 +1 @@ # -*- coding: utf-8 -*- - -import os - -enptRepo_rootpath = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) - -config_for_testing = dict( - path_l1b_enmap_image=os.path.join(enptRepo_rootpath, 'tests', 'data', 'EnMAP_Level_1B', 'AlpineTest1_CWV2_SM0.zip'), - output_dir=os.path.join(enptRepo_rootpath, 'tests', 'data', 'test_outputs') -) diff --git a/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh b/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh index 11194540bc7ca81074889a2f597433d5225557ec..4acaa483e8e813ab73c60e6ffc1a9090a29d4bee 100755 --- a/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh +++ b/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh @@ -2,9 +2,19 @@ context_dir="./context" dockerfile="enpt_ci.docker" -tag="enpt_ci:latest" +tag="enpt_ci:0.4.0b4" gitlab_runner="enpt_gitlab_CI_runner" +# get sicor project +rm -rf context/sicor +# git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git ./context/sicor +git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git --branch feature/improve_enmap --single-branch ./context/sicor + +# download sicor cache (fastens SICOR CI tests a lot, but cache needs to be updated manually using a local sicor repo: +# 1. clone a fresh copy of sicor or delete sicor/sicor/aerosol_0_ch4_34d3778719cc87188787de09bb8f870d16050078.pkl.zip +# 2. run a sicor test including sicor_ac or enmap_ac (recreates cache file) -> upload newly created cache file +# wget http://ouo.io/uCQxof -P ./context/ + echo "#### Build runner docker image" sudo docker rmi ${tag} sudo docker build -f ${context_dir}/${dockerfile} -m 20G -t ${tag} ${context_dir} diff --git a/tests/gitlab_CI_docker/context/enpt_ci.docker b/tests/gitlab_CI_docker/context/enpt_ci.docker index e9d1020e5f6ec9afafe169a185bb0471786c6f01..a64bdeda24ee687a02cee66661871702b9643dca 100644 --- a/tests/gitlab_CI_docker/context/enpt_ci.docker +++ b/tests/gitlab_CI_docker/context/enpt_ci.docker @@ -4,12 +4,12 @@ FROM centos:7 RUN yum update -y && \ yum install -y wget vim bzip2 gcc gcc-c++ make libgl1-mesa-glx mesa-libGL qt5-qtbase-gui git nano tree gdb texlive -# change version numbers for future upgrades -# ENV miniconda_dl 'Miniconda3-latest-Linux-x86_64.sh' +# change version numbers for future upgrades # currently Anaconda 4.5.1 (worked fine) +ENV miniconda_dl 'Miniconda3-latest-Linux-x86_64.sh' # use miniconda 4.3.31 as workaround for "IsADirectoryError(21, 'Is a directory')" with version 4.4.10 (currently latest) -ENV miniconda_dl 'Miniconda3-4.3.31-Linux-x86_64.sh' +# ENV miniconda_dl 'Miniconda3-4.3.31-Linux-x86_64.sh' ENV envconfig 'environment_enpt.yml' -ENV git_lfs_v='2.1.1' +ENV git_lfs_v='2.4.1' RUN /bin/bash -i -c "cd /root; wget https://repo.continuum.io/miniconda/$miniconda_dl ; \ bash -i /root/$miniconda_dl -b ; \ @@ -30,3 +30,19 @@ RUN /bin/bash -i -c "wget https://github.com/git-lfs/git-lfs/releases/download/v tar -zxvf git-lfs-linux-amd64-${git_lfs_v}.tar.gz; \ cd git-lfs-${git_lfs_v}; \ bash ./install.sh" + +# copy sicor code to /tmp +COPY sicor /tmp/sicor + +# install sicor (in pip development mode so that its root directory in /tmp/sicor matching with the subsequent COPY command) +RUN bash -i -c "source /root/miniconda3/bin/activate enpt; \ + cd /tmp/sicor/ ; \ + make clean ; \ + make requirements ; \ + make download-tables ; \ + pip install -e . --no-cache-dir" + +# copy sicor cache files to sicor root directory (speeds up SICOR CI tests because table subsets dont have to be created each time) +# -> sicor root directory is the default directory of these cache files if sicor_cache_dir is not set in EnPT options +COPY *.zip /tmp/sicor/sicor + diff --git a/tests/gitlab_CI_docker/context/environment_enpt.yml b/tests/gitlab_CI_docker/context/environment_enpt.yml index f1b7bbfdbfe3fa09f4f634f7fc2430fd83614ff4..4f67e4e3ec73f3f72335f73b08522ae43eeef68a 100644 --- a/tests/gitlab_CI_docker/context/environment_enpt.yml +++ b/tests/gitlab_CI_docker/context/environment_enpt.yml @@ -14,7 +14,7 @@ dependencies: - scikit-image - rasterio - pyproj - - lxml + # - lxml - geopandas - ipython - matplotlib @@ -22,19 +22,38 @@ dependencies: - shapely - holoviews - bokeh + + # arosics + - pyfftw + - pykrige + + # sicor + - glymur + - pygrib + - cachetools + - pyhdf + - h5py + - pytables + - pip: - scipy - geoarray>=0.7.15 - spectral>=0.16 - cerberus - jsmin + - utm + - lxml + - imageio - sphinx-argparse + - pycodestyle<2.4.0 # version 2.4.0 causes ImportError: module 'pycodestyle' has no attribute 'break_around_binary_operator' - flake8 - - pycodestyle - pylint - pydocstyle - nose - nose2 - nose-htmloutput - coverage - - rednose \ No newline at end of file + - rednose + + # sicor + - https://software.ecmwf.int/wiki/download/attachments/56664858/ecmwf-api-client-python.tgz \ No newline at end of file diff --git a/tests/linting/pydocstyle.log b/tests/linting/pydocstyle.log index 9db016230957433bd57cf2cc8f5574f32de48c55..358d86ea5f908fd85c01cc66a6608854307fdbce 100644 --- a/tests/linting/pydocstyle.log +++ b/tests/linting/pydocstyle.log @@ -1,10 +1,12 @@ enpt/version.py:1 at module level: D100: Missing docstring in public module -enpt/model/metadata.py:185 in public method `calc_solar_irradiance_CWL_FWHM_per_band`: +enpt/model/images.py:474 in public method `run_AC`: D102: Missing docstring in public method -enpt/model/metadata.py:248 in public method `get_earth_sun_distance`: +enpt/model/metadata.py:210 in public method `calc_solar_irradiance_CWL_FWHM_per_band`: + D102: Missing docstring in public method +enpt/model/metadata.py:268 in public method `get_earth_sun_distance`: D202: No blank lines allowed after function docstring (found 1) -enpt/model/metadata.py:248 in public method `get_earth_sun_distance`: +enpt/model/metadata.py:268 in public method `get_earth_sun_distance`: D400: First line should end with a period (not ')') enpt/model/srf.py:11 in public class `SRF`: D101: Missing docstring in public class @@ -16,45 +18,65 @@ enpt/model/srf.py:114 in public method `__getitem__`: D105: Missing docstring in magic method enpt/model/srf.py:117 in public method `__iter__`: D105: Missing docstring in magic method +enpt/utils/path_generator.py:53 in public function `get_path_ac_options`: + D401: First line should be in imperative mood ('Return', not 'Returns') +enpt/utils/logging.py:83 in public method `__getstate__`: + D105: Missing docstring in magic method +enpt/utils/logging.py:87 in public method `__setstate__`: + D401: First line should be in imperative mood ('Define', not 'Defines') +enpt/utils/logging.py:138 in public method `__del__`: + D105: Missing docstring in magic method +enpt/utils/logging.py:141 in public method `__enter__`: + D105: Missing docstring in magic method +enpt/utils/logging.py:144 in public method `__exit__`: + D105: Missing docstring in magic method +enpt/processors/atmospheric_correction/atmospheric_correction.py:25 in public method `get_ac_options`: + D102: Missing docstring in public method +enpt/processors/atmospheric_correction/atmospheric_correction.py:47 in public method `run_ac`: + D102: Missing docstring in public method enpt/execution/controller.py:50 in public method `read_L1B_data`: D205: 1 blank line required between summary line and description (found 0) enpt/execution/controller.py:50 in public method `read_L1B_data`: D400: First line should end with a period (not ':') -enpt/options/config.py:34 in public class `EnPTConfig`: +enpt/execution/controller.py:82 in public method `run_geometry_processor`: + D102: Missing docstring in public method +enpt/execution/controller.py:89 in public method `write_output`: + D102: Missing docstring in public method +enpt/options/config.py:42 in public class `EnPTConfig`: D101: Missing docstring in public class -enpt/options/config.py:35 in public method `__init__`: +enpt/options/config.py:43 in public method `__init__`: D202: No blank lines allowed after function docstring (found 1) -enpt/options/config.py:102 in public method `absPath`: +enpt/options/config.py:121 in public method `absPath`: D102: Missing docstring in public method -enpt/options/config.py:105 in public method `get_parameter`: +enpt/options/config.py:124 in public method `get_parameter`: D102: Missing docstring in public method -enpt/options/config.py:168 in public method `to_dict`: +enpt/options/config.py:187 in public method `to_dict`: D202: No blank lines allowed after function docstring (found 1) -enpt/options/config.py:182 in public method `to_jsonable_dict`: +enpt/options/config.py:201 in public method `to_jsonable_dict`: D102: Missing docstring in public method -enpt/options/config.py:193 in public method `__repr__`: +enpt/options/config.py:212 in public method `__repr__`: D105: Missing docstring in magic method -enpt/options/config.py:197 in public function `json_to_python`: +enpt/options/config.py:216 in public function `json_to_python`: D103: Missing docstring in public function -enpt/options/config.py:230 in public function `python_to_json`: +enpt/options/config.py:249 in public function `python_to_json`: D103: Missing docstring in public function -enpt/options/config.py:252 in public class `EnPTValidator`: +enpt/options/config.py:271 in public class `EnPTValidator`: D101: Missing docstring in public class -enpt/options/config.py:253 in public method `__init__`: +enpt/options/config.py:272 in public method `__init__`: D205: 1 blank line required between summary line and description (found 0) -enpt/options/config.py:253 in public method `__init__`: +enpt/options/config.py:272 in public method `__init__`: D400: First line should end with a period (not 'r') -enpt/options/config.py:261 in public method `validate`: +enpt/options/config.py:280 in public method `validate`: D102: Missing docstring in public method -enpt/options/config.py:266 in public function `get_options`: +enpt/options/config.py:285 in public function `get_options`: D202: No blank lines allowed after function docstring (found 1) enpt/options/__init__.py:1 at module level: D104: Missing docstring in public package -enpt/options/options_schema.py:93 in public function `get_updated_schema`: +enpt/options/options_schema.py:110 in public function `get_updated_schema`: D103: Missing docstring in public function -enpt/options/options_schema.py:94 in private nested function `deep_update`: +enpt/options/options_schema.py:111 in private nested function `deep_update`: D202: No blank lines allowed after function docstring (found 1) -enpt/options/options_schema.py:94 in private nested function `deep_update`: +enpt/options/options_schema.py:111 in private nested function `deep_update`: D400: First line should end with a period (not 'e') -enpt/options/options_schema.py:113 in public function `get_param_from_json_config`: +enpt/options/options_schema.py:130 in public function `get_param_from_json_config`: D103: Missing docstring in public function diff --git a/tests/test_cli_parser.py b/tests/test_cli_parser.py index b9e43c38c813a3540193ba1ff7a99b7e1aa29196..c866e2c7ac0dae9ba5fe3e9f77c4e0d9adf7a0d6 100644 --- a/tests/test_cli_parser.py +++ b/tests/test_cli_parser.py @@ -13,10 +13,10 @@ import os from runpy import run_path from multiprocessing import cpu_count -from enpt import __path__ +import enpt -path_run_enpt = os.path.abspath(os.path.join(__path__[0], '..', 'bin', 'enpt_cli.py')) +path_run_enpt = os.path.abspath(os.path.join(enpt.__path__[0], '..', 'bin', 'enpt_cli.py')) class Test_CLIParser(TestCase): diff --git a/tests/test_config.py b/tests/test_config.py index 66a78207887ac9a947ffc200f5a4370e816fc01e..6f13d53bcd3df080b88daff343f3c7e0a1e2888f 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python # -*- coding: utf-8 -*- """ @@ -11,7 +12,7 @@ from json import \ dumps, \ JSONDecodeError -from unittest import TestCase +from unittest import TestCase, main from enpt.options.config import \ get_options, \ @@ -87,3 +88,7 @@ class Test_EnPTConfig(TestCase): # check validity EnPTValidator(allow_unknown=True, schema=enpt_schema_config_output).validate(params) + + +if __name__ == '__main__': + main() diff --git a/tests/test_controller.py b/tests/test_controller.py index 195de5b1862e31c6ca344c7027da33282b44efae..2c53f8350308e0f8d2d3442f188e86356ef5fcc4 100644 --- a/tests/test_controller.py +++ b/tests/test_controller.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python # -*- coding: utf-8 -*- """ @@ -7,12 +8,11 @@ test_controller Tests for `execution.controller` module. """ -from unittest import TestCase +from unittest import TestCase, main import shutil from enpt.execution.controller import EnPT_Controller - -from . import config_for_testing +from enpt.options.config import config_for_testing class Test_EnPT_Controller(TestCase): @@ -20,7 +20,12 @@ class Test_EnPT_Controller(TestCase): self.CTR = EnPT_Controller(**config_for_testing) def tearDown(self): - shutil.rmtree(self.CTR.cfg.output_dir) + # NOTE: ignore_errors deletes the folder, regardless of whether it contains read-only files + shutil.rmtree(self.CTR.cfg.output_dir, ignore_errors=True) def test_run_all_processors(self): self.CTR.run_all_processors() + + +if __name__ == '__main__': + main() diff --git a/tests/test_l1b_reader.py b/tests/test_l1b_reader.py index 62a60265519fa1a0e0d255ea6a5767f1ded1eebe..780a77d74300246ca462aee4849cc23c9ebcce3c 100644 --- a/tests/test_l1b_reader.py +++ b/tests/test_l1b_reader.py @@ -17,8 +17,7 @@ import zipfile from datetime import datetime import shutil -from enpt.options.config import EnPTConfig -from . import config_for_testing +from enpt.options.config import EnPTConfig, config_for_testing class Test_L1B_Reader(unittest.TestCase): diff --git a/tests/test_radiometric_transform.py b/tests/test_radiometric_transform.py index 61c8a0a9d532b5daa994ce4fbf65cdf5c760710b..2433d77ae0ed1430e07e23c3707e66bc80d599c3 100644 --- a/tests/test_radiometric_transform.py +++ b/tests/test_radiometric_transform.py @@ -1,23 +1,21 @@ -# -*- coding: utf-8 -*- +#!/usr/bin/env python +# -*- coding: utf-8 -*- import os -import unittest +from unittest import TestCase, main from glob import glob import tempfile import zipfile from datetime import datetime -from enpt.processors.radiometric_transform import Radiometric_Transformer -from enpt.options.config import EnPTConfig +from enpt.processors import Radiometric_Transformer +from enpt.options.config import EnPTConfig, config_for_testing -from . import config_for_testing - -class Test_Radiometric_Transformer(unittest.TestCase): +class Test_Radiometric_Transformer(TestCase): def setUp(self): """Set up the needed test data""" - self.cfg = EnPTConfig(**config_for_testing) self.pathList_testimages = glob(os.path.join(os.path.dirname(__file__), "data", "EnMAP_Level_1B", "*.zip")) self.RT = Radiometric_Transformer(config=self.cfg) @@ -48,3 +46,7 @@ class Test_Radiometric_Transformer(unittest.TestCase): self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) self.assertTrue(L1_obj.vnir.detector_meta.unitcode == 'TOARef') self.assertTrue(L1_obj.swir.detector_meta.unitcode == 'TOARef') + + +if __name__ == "__main__": + main()