diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 92164cb14d1c5adfbf9ed6ac0eb0d40bb2111823..61f52c86924a00aaf694094167de8facf879127e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,7 +15,17 @@ test_enpt: - export PYTHONPATH=$PYTHONPATH:/root # /root <- here are the sicor tables # update py_tools_ds - - pip install 'py_tools_ds>=0.14.8' + - pip install 'py_tools_ds>=0.14.14' + + # switch sicor branch + - mkdir /tmp/sicor_tables + - mv /tmp/sicor/sicor/tables/*.h5 /tmp/sicor_tables + - mv /tmp/sicor/sicor/tables/EnMAP_LUT* /tmp/sicor_tables + - rm -rf /tmp/sicor + - git clone -b feature/improve_speed https://gitext.gfz-potsdam.de/EnMAP/sicor.git /tmp/sicor + - mv /tmp/sicor_tables/* /tmp/sicor/sicor/tables + - rm -rf /tmp/sicor_tables/ + - pip install -e /tmp/sicor # update sicor # - conda install -y -q -c conda-forge basemap @@ -60,6 +70,7 @@ test_enpt_install: stage: test script: - source /root/miniconda3/bin/activate + - conda update -n base -c defaults conda - conda create -y -q --name enpt_test python=3 - source activate enpt_test @@ -68,10 +79,11 @@ test_enpt_install: # install not pip-installable deps of geoarray - conda install -y -c conda-forge numpy scikit-image matplotlib pandas gdal rasterio pyproj basemap shapely - - conda install -y -c conda-forge 'icu=58.*' # fixes bug for conda-forge gdal build + # - conda install -y -c conda-forge 'icu=58.*' # fixes bug for conda-forge gdal build # install py_tools_ds - - pip install 'py_tools_ds>=0.14.8' + - conda install -y -c conda-forge pyresample + - pip install 'py_tools_ds>=0.14.14' # install sicor - conda install -y -q -c conda-forge pygrib h5py pytables pyfftw numba llvmlite scikit-learn diff --git a/HISTORY.rst b/HISTORY.rst index c9d8dab6df63c886a242b104355f301e122c5342..ee3e476b3d17280232a26f36ba7728dacae6413e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,12 +2,13 @@ History ======= -0.x.x (coming soon) +0.8.x (coming soon) ------------------- -New features: +New features / improvements: -* TBD +* enhanced AC results due to updated SICOR implementation + (currently dependent from SICOR branch "feature/three_phases_of_water_retieval") 0.7.0 (2019-01-21) diff --git a/README.rst b/README.rst index b5da1c238b3d0a38cc51be60b447a6702a55f7d3..ab5a71f51cf467eebdff492bed07a7d492cd41de 100644 --- a/README.rst +++ b/README.rst @@ -33,6 +33,6 @@ This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypack .. _Cookiecutter: https://github.com/audreyr/cookiecutter .. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage -.. _documentation: http://segl.gitext.gfz-potsdam.de/EnPT/doc/ -.. _coverage: http://segl.gitext.gfz-potsdam.de/EnPT/coverage/ -.. _nosetests: http://segl.gitext.gfz-potsdam.de/EnPT/nosetests_reports/nosetests.html \ No newline at end of file +.. _documentation: https://enmap.gitext.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/EnPT/doc/ +.. _coverage: https://enmap.gitext.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/EnPT/coverage/ +.. _nosetests: https://enmap.gitext.gfz-potsdam.de/GFZ_Tools_EnMAP_BOX/EnPT/nosetests_reports/nosetests.html \ No newline at end of file diff --git a/bin/enpt_cli.py b/bin/enpt_cli.py index eb890e5e361fafef3c73bf2e4ea49da37a7ca020..e7f8496366bcf56874beef4e8b4b6a40bda11e37 100644 --- a/bin/enpt_cli.py +++ b/bin/enpt_cli.py @@ -68,10 +68,10 @@ def get_enpt_argparser(): add('--enable_ac', type=bool, default=True, help="Enable atmospheric correction using SICOR algorithm (default: True). If False, the L2A output contains " "top-of-atmosphere reflectance") - add('--sicor_cache_dir', type=str, default=None, - help='SICOR cache directory') add('--auto_download_ecmwf', type=bool, default=False, help='Automatically download ECMWF data for atmospheric correction') + add('--enable_ice_retrieval', type=bool, default=True, + help='Enable ice retrieval (default); increases accuracy of water vapour retrieval') add('--enable_cloud_screening', type=bool, default=False, help='Enable cloud screening during atmospheric correction') add('--scale_factor_boa_ref', type=int, default=10000, diff --git a/enpt/execution/controller.py b/enpt/execution/controller.py index a1001f2e73bb3b53bcf439ec4ff31f55b7352cf4..f2fd38ae7282dbddb58329695cd21ad3d148b1dd 100644 --- a/enpt/execution/controller.py +++ b/enpt/execution/controller.py @@ -121,10 +121,10 @@ class EnPT_Controller(object): # self.run_toaRad2toaRef() # this is only needed for geometry processor but AC expects radiance self.run_dem_processor() if self.cfg.enable_ac: - self.L1_obj.logger.info('Skipping atmospheric correction as configured and ' - 'computing top-of-atmosphere reflectance instead.') self.run_atmospheric_correction() else: + self.L1_obj.logger.info('Skipping atmospheric correction as configured and ' + 'computing top-of-atmosphere reflectance instead.') self.run_toaRad2toaRef() self.run_geometry_processor() self.run_orthorectification() diff --git a/enpt/model/images.py b/enpt/model/images.py index 0984818c01fe84835c9d7f93c9e4b031902cac8d..048ac7affa9d152939bba991915673fb13bded28 100644 --- a/enpt/model/images.py +++ b/enpt/model/images.py @@ -388,7 +388,7 @@ class EnMAP_Detector_SensorGeo(_EnMAP_Image): algo = self.cfg.deadpix_P_algorithm method_spectral, method_spatial = self.cfg.deadpix_P_interp_spectral, self.cfg.deadpix_P_interp_spatial self.logger.info("Correcting dead pixels of %s detector...\n" - "Used algorithm / interpolation: %s / %s" + "Used algorithm: %s interpolation in the %s domain" % (self.detector_name, algo, method_spectral if algo == 'spectral' else method_spatial)) self.data = \ diff --git a/enpt/model/metadata.py b/enpt/model/metadata.py index 299c4031d7c6855c02f0499628c981024e533618..4afb3595fc8bff77efea7dbd09647d4da523a3dd 100644 --- a/enpt/model/metadata.py +++ b/enpt/model/metadata.py @@ -801,7 +801,7 @@ class EnMAP_Metadata_L2A_MapGeo(object): # FIXME this is the size of the VNIR/SWIR stack size = [F['size'] for F in self.fileinfos if os.path.splitext(F['name'])[0].endswith('-SPECTRAL_IMAGE')][0] xml.find("product/image/%s/size" % lbl).text = str(size) - # FIXME DLR data dimensions equal either L2A data nor L1B data + # FIXME DLR data dimensions equal neither L2A data nor L1B data xml.find("product/image/%s/channels" % lbl).text = str(detMetaL1B.nwvl) xml.find("product/image/%s/dimension/rows" % lbl).text = str(self.nrows) xml.find("product/image/%s/dimension/columns" % lbl).text = str(self.ncols) diff --git a/enpt/model/srf.py b/enpt/model/srf.py index 038fd1869abd4864215c817c5dfb599f9d0358f3..4057743b39fafe4fb367f968f381bf4f3bac4f59 100644 --- a/enpt/model/srf.py +++ b/enpt/model/srf.py @@ -43,8 +43,8 @@ class SRF(object): :param cwl: target center wavelength position :param fwhm: target band width (full width half maximum) - :param wvl_min: minimum wavelength to compute spectral respnse for - :param wvl_max: maximum wavelength to compute spectral respnse for + :param wvl_min: minimum wavelength to compute spectral response for + :param wvl_max: maximum wavelength to compute spectral response for :param wvl_res: spectral resolution at which spectral response is to be computed :param normalize: whether to normalize the output spectral response to values between 0 and 1 :return: 2D numpy.ndarray: rows: response per wavelength; columns: wavelength/response diff --git a/enpt/options/config.py b/enpt/options/config.py index 14f0f8f8e6f26dfaca24c8fdb959bc5a889213dc..478e595d9da85e911c7a33103476bc20cef17524 100644 --- a/enpt/options/config.py +++ b/enpt/options/config.py @@ -17,8 +17,6 @@ from collections import OrderedDict, Mapping import numpy as np from multiprocessing import cpu_count -import sicor - from .options_schema import \ enpt_schema_input, \ enpt_schema_config_output, \ @@ -45,7 +43,9 @@ config_for_testing = dict( n_lines_to_append=50, disable_progress_bars=True, is_dlr_dataformat=False, - enable_ac=False + enable_ac=False, + enable_ice_retrieval=False, + CPUs=16 ) @@ -66,7 +66,9 @@ config_for_testing_dlr = dict( disable_progress_bars=True, is_dlr_dataformat=True, enable_ac=False, - ortho_resampAlg='gauss' + enable_ice_retrieval=False, + CPUs=32, + ortho_resampAlg='gauss', ) @@ -143,8 +145,8 @@ class EnPTConfig(object): # atmospheric_correction self.enable_ac = gp('enable_ac') - self.sicor_cache_dir = gp('sicor_cache_dir', fallback=sicor.__path__[0]) self.auto_download_ecmwf = gp('auto_download_ecmwf') + self.enable_ice_retrieval = gp('enable_ice_retrieval') self.enable_cloud_screening = gp('enable_cloud_screening') self.scale_factor_boa_ref = gp('scale_factor_boa_ref'), diff --git a/enpt/options/options_default.json b/enpt/options/options_default.json index 474a2792897b124fc47a3dfad06fe481c736b12a..a1327714fd26ef5a1a0566b79090faf4443bf1d0 100644 --- a/enpt/options/options_default.json +++ b/enpt/options/options_default.json @@ -38,11 +38,9 @@ "atmospheric_correction": { "enable_ac": true, /*enable atmospheric correction using SICOR algorithm (default: True). If False, the L2A output contains top-of-atmosphere reflectance.*/ - "sicor_cache_dir": "", /*directory to be used to stored sicor cache files - NOTE: SICOR stores intermediate results there that need to computed only once - for atmospheric correction of multiple EnMAP images. (default: 'auto')*/ - "auto_download_ecmwf": false, - "enable_cloud_screening": false, + "auto_download_ecmwf": false, /*FIXME this might be not needed anymore in future*/ + "enable_ice_retrieval": true, /*enable ice retrieval (default); increases accuracy of water vapour retrieval*/ + "enable_cloud_screening": false, /*FIXME this is currently not implemented*/ "scale_factor_boa_ref": 10000 /*scale factor to be applied to BOA reflectance result*/ }, diff --git a/enpt/options/options_schema.py b/enpt/options/options_schema.py index f10af6d29c5c5d3fc98e96a094b1e63f0bca7afb..1da9b4bcc39e2aa148ba388aff1437d331e33d9d 100644 --- a/enpt/options/options_schema.py +++ b/enpt/options/options_schema.py @@ -46,8 +46,8 @@ enpt_schema_input = dict( type='dict', required=False, schema=dict( enable_ac=dict(type='boolean', required=False), - sicor_cache_dir=dict(type='string', required=False), auto_download_ecmwf=dict(type='boolean', required=False), + enable_ice_retrieval=dict(type='boolean', required=False), enable_cloud_screening=dict(type='boolean', required=False), )), @@ -106,8 +106,8 @@ parameter_mapping = dict( # processors > atmospheric_correction enable_ac=('processors', 'atmospheric_correction', 'enable_ac'), - sicor_cache_dir=('processors', 'atmospheric_correction', 'sicor_cache_dir'), auto_download_ecmwf=('processors', 'atmospheric_correction', 'auto_download_ecmwf'), + enable_ice_retrieval=('processors', 'atmospheric_correction', 'enable_ice_retrieval'), enable_cloud_screening=('processors', 'atmospheric_correction', 'enable_cloud_screening'), scale_factor_boa_ref=('processors', 'atmospheric_correction', 'scale_factor_boa_ref'), diff --git a/enpt/processors/atmospheric_correction/atmospheric_correction.py b/enpt/processors/atmospheric_correction/atmospheric_correction.py index 541ef6e49c3a2a7006252de0fc2f0912b8206178..b9a553429b1e60edc22cbcb64720e37fe863d39f 100644 --- a/enpt/processors/atmospheric_correction/atmospheric_correction.py +++ b/enpt/processors/atmospheric_correction/atmospheric_correction.py @@ -5,7 +5,10 @@ Performs the atmospheric correction of EnMAP L1B data. """ import pprint import numpy as np +from multiprocessing import cpu_count +from os import path +import sicor from sicor.sicor_enmap import sicor_ac_enmap from sicor.options import get_options as get_ac_options @@ -21,23 +24,23 @@ class AtmosphericCorrector(object): """Create an instance of AtmosphericCorrector.""" self.cfg = config - def get_ac_options(self, buffer_dir): + def get_ac_options(self): path_opts = get_path_ac_options() try: - options = get_ac_options(path_opts) + options = get_ac_options(path_opts, validation=False) # FIXME validation is currently not implemented # adjust options - options['EnMAP']['buffer_dir'] = buffer_dir - for vv in options["RTFO"].values(): - vv["hash_formats"] = dict(spr='%.0f', - coz='%.0f,', - cwv='%.0f,', - tmp='%0f,', - tau_a='%.2f,', - vza='%.0f,') - vv["disable_progress_bars"] = self.cfg.disable_progress_bars - options["ECMWF"]["path_db"] = "./ecmwf" + # FIXME this path should be already known to sicor + options["EnMAP"]["Retrieval"]["fn_LUT"] = \ + path.join(path.abspath(sicor.__path__[0]), 'tables', 'EnMAP_LUT_MOD5_formatted_1nm') + # options["ECMWF"]["path_db"] = "./ecmwf" # disbled as it is not needed at the moment + # TODO disable_progress_bars? + + # always use the fast implementation (the slow implementation was only a temporary solution) + options["EnMAP"]["Retrieval"]["fast"] = True + options["EnMAP"]["Retrieval"]["ice"] = self.cfg.enable_ice_retrieval + options["EnMAP"]["Retrieval"]["cpu"] = self.cfg.CPUs or cpu_count() return options @@ -45,29 +48,34 @@ class AtmosphericCorrector(object): raise FileNotFoundError('Could not locate options file for atmospheric correction at %s.' % path_opts) def run_ac(self, enmap_ImageL1: EnMAPL1Product_SensorGeo) -> EnMAPL1Product_SensorGeo: - options = self.get_ac_options(buffer_dir=self.cfg.sicor_cache_dir) + options = self.get_ac_options() enmap_ImageL1.logger.debug('AC options: \n' + pprint.pformat(options)) # run AC enmap_ImageL1.logger.info("Starting atmospheric correction for VNIR and SWIR detector. " "Source radiometric unit code is '%s'." % enmap_ImageL1.meta.vnir.unitcode) - enmap_l2a_sens_geo, state = sicor_ac_enmap(enmap_l1b=enmap_ImageL1, options=options, - logger=enmap_ImageL1.logger) + + # run SICOR + # NOTE: - enmap_l2a_vnir, enmap_l2a_swir: reflectance between 0 and 1 + # - cwv_model, cwc_model, toa_model have the SWIR geometry + # - currently, the fast method is implemented, + # -> otherwise options["EnMAP"]["Retrieval"]["fast"] must be false + # - ice_model is None if self.cfg.enable_ice_retrieval is False + enmap_l2a_vnir, enmap_l2a_swir, cwv_model, cwc_model, ice_model, toa_model, se, scem, srem = \ + sicor_ac_enmap(enmap_l1b=enmap_ImageL1, options=options, logger=enmap_ImageL1.logger) # join results enmap_ImageL1.logger.info('Joining results of atmospheric correction.') for in_detector, out_detector in zip([enmap_ImageL1.vnir, enmap_ImageL1.swir], - [enmap_l2a_sens_geo.vnir, enmap_l2a_sens_geo.swir]): - in_detector.data = (out_detector.data[:] * self.cfg.scale_factor_boa_ref).astype(np.int16) + [enmap_l2a_vnir, enmap_l2a_swir]): + in_detector.data = (out_detector * self.cfg.scale_factor_boa_ref).astype(np.int16) # NOTE: geotransform and projection are missing due to sensor geometry - del in_detector.data_l2a # FIXME sicor sets data_l2a to float array -> not needed - del in_detector.unit # FIXME sicor sets unit to '1' -> not needed - in_detector.detector_meta.unit = '0-%d' % self.cfg.scale_factor_boa_ref in_detector.detector_meta.unitcode = 'BOARef' # FIXME what about mask_clouds, mask_clouds_confidence, ac_errors? + # FIXME use cwv_model, cwc_model, toa_model also for EnPT? return enmap_ImageL1 diff --git a/enpt/processors/orthorectification/orthorectification.py b/enpt/processors/orthorectification/orthorectification.py index f285e4a341edc85a5aab30f2ae005d543bae408b..64010945752695d2bf2f85b1a724eb31df599665 100644 --- a/enpt/processors/orthorectification/orthorectification.py +++ b/enpt/processors/orthorectification/orthorectification.py @@ -66,13 +66,16 @@ class Orthorectifier(object): # transform VNIR and SWIR to map geometry GeoTransformer = Geometry_Transformer if lons_vnir.ndim == 2 else Geometry_Transformer_3D + enmap_ImageL1.logger.info('Orthorectifying VNIR data...') GT_vnir = GeoTransformer(lons=lons_vnir, lats=lats_vnir, **kw_init) vnir_mapgeo, vnir_gt, vnir_prj = GT_vnir.to_map_geometry(enmap_ImageL1.vnir.data[:], **kw_trafo) + enmap_ImageL1.logger.info('Orthorectifying SWIR data...') GT_swir = GeoTransformer(lons=lons_swir, lats=lats_swir, **kw_init) swir_mapgeo, swir_gt, swir_prj = GT_swir.to_map_geometry(enmap_ImageL1.swir.data[:], **kw_trafo) # combine VNIR and SWIR + enmap_ImageL1.logger.info('Merging VNIR and SWIR data...') L2_obj.data = self._get_VNIR_SWIR_stack(vnir_mapgeo, swir_mapgeo, vnir_gt, swir_gt, vnir_prj, swir_prj, enmap_ImageL1.meta.vnir.wvl_center, enmap_ImageL1.meta.swir.wvl_center) @@ -82,6 +85,7 @@ class Orthorectifier(object): ** kw_init) # FIXME cloud mask applies to BOTH detectors + enmap_ImageL1.logger.info('Orthorectifying cloud mask...') L2_obj.mask_clouds = GeoArray(*GT_2D.to_map_geometry(enmap_ImageL1.vnir.mask_clouds, **kw_trafo)) # TODO transform mask_clouds_confidence, ac_errors, pixel masks @@ -89,6 +93,7 @@ class Orthorectifier(object): # metadata adjustments # ######################## + enmap_ImageL1.logger.info('Generating L2A metadata...') L2_obj.meta = EnMAP_Metadata_L2A_MapGeo(config=self.cfg, meta_l1b=enmap_ImageL1.meta, dims_mapgeo=L2_obj.data.shape, diff --git a/enpt/utils/logging.py b/enpt/utils/logging.py index 90d351375e7c8fc27652fbc814e8593c140ed6c1..0bfa14abf00476d9b5f6eb16727416b39d6a8586 100644 --- a/enpt/utils/logging.py +++ b/enpt/utils/logging.py @@ -26,6 +26,12 @@ class EnPT_Logger(logging.Logger): # private attributes self._captured_stream = '' + # attributes that need to be present in order to unpickle the logger via __setstate_ + self.name_logfile = name_logfile + self.fmt_suffix = fmt_suffix + self.path_logfile = path_logfile + self.log_level = log_level + super(EnPT_Logger, self).__init__(name_logfile) self.path_logfile = path_logfile @@ -54,9 +60,9 @@ class EnPT_Logger(logging.Logger): # create StreamHandler self.streamObj = StringIO() - self.streamHandler = logging.StreamHandler(stream=self.streamObj) - self.streamHandler.setFormatter(self.formatter_fileH) - self.streamHandler.set_name('StringIO handler') + streamHandler = logging.StreamHandler(stream=self.streamObj) + streamHandler.setFormatter(self.formatter_fileH) + streamHandler.set_name('StringIO handler') # create ConsoleHandler for logging levels DEGUG and INFO -> logging to sys.stdout consoleHandler_out = logging.StreamHandler(stream=sys.stdout) # by default it would go to sys.stderr @@ -76,7 +82,7 @@ class EnPT_Logger(logging.Logger): if not self.handlers: if fileHandler: self.addHandler(fileHandler) - self.addHandler(self.streamHandler) + self.addHandler(streamHandler) self.addHandler(consoleHandler_out) self.addHandler(consoleHandler_err) @@ -118,9 +124,8 @@ class EnPT_Logger(logging.Logger): try: if handler.get_name() == 'StringIO handler': self.streamObj.flush() - self.streamHandler.flush() self.removeHandler(handler) # if not called with '[:]' the StreamHandlers are left open - # handler.flush() + handler.flush() handler.close() except PermissionError: warnings.warn('Could not properly close logfile due to a PermissionError: %s' % sys.exc_info()[1]) @@ -128,8 +133,6 @@ class EnPT_Logger(logging.Logger): if self.handlers[:]: warnings.warn('Not all logging handlers could be closed. Remaining handlers: %s' % self.handlers[:]) - # print('sh', self.streamHandler) - def view_logfile(self): """View the log file written to disk.""" with open(self.path_logfile) as inF: diff --git a/enpt/utils/path_generator.py b/enpt/utils/path_generator.py index d85c8b4c2646178e9fa892e5f941d19c767d7e16..1fdd75e2c97e9b2b4a862e1fa2ac00c3f4a69e18 100644 --- a/enpt/utils/path_generator.py +++ b/enpt/utils/path_generator.py @@ -58,6 +58,8 @@ class PathGenL1BProduct(object): def get_path_ac_options() -> str: """Returns the path of the options json file needed for atmospheric correction.""" from sicor import options - path_ac = os.path.join(os.path.dirname(options.__file__), 'sicor_enmap_user_options.json') + path_ac = os.path.join(os.path.dirname(options.__file__), 'enmap_options.json') + # FIXME temporarily disabled because not implemented at the moment: + # path_ac = os.path.join(os.path.dirname(options.__file__), 'sicor_enmap_user_options.json') return path_ac diff --git a/requirements.txt b/requirements.txt index 26262868a1fb4904bb9bce579d8eaee2653bd507..34bae14158efdeb673a11fdd12d9f33051c6c2b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ numpy pandas scipy geoarray>=0.8.9 -py_tools_ds>=0.14.8 +py_tools_ds>=0.14.14 cerberus jsmin matplotlib diff --git a/setup.py b/setup.py index b5aa6c8e1a0e1333622d60d9fb0691bb3c289539..f0f60bde6487ca3a650cfc62c92795b5c44797e5 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ with open("enpt/version.py") as version_file: exec(version_file.read(), version) requirements = [ # put package requirements here - 'numpy', 'pandas', 'scipy', 'geoarray>=0.8.9', 'py_tools_ds>=0.14.8', 'cerberus', 'jsmin', 'matplotlib', 'tqdm', + 'numpy', 'pandas', 'scipy', 'geoarray>=0.8.9', 'py_tools_ds>=0.14.14', 'cerberus', 'jsmin', 'matplotlib', 'tqdm', 'utm', 'lxml', 'numpy-indexed' # 'sicor', # pip install git+https://gitext.gfz-potsdam.de/EnMAP/sicor.git ] diff --git a/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh b/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh index ec2072ff1593075c8910ecd520e3a108086ba4b4..7e5eebef3cbebc9319670cf0e552b18cbcb308fb 100755 --- a/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh +++ b/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh @@ -2,7 +2,7 @@ context_dir="./context" dockerfile="enpt_ci.docker" -tag="enpt_ci:0.7.0" +tag="enpt_ci:0.7.1" gitlab_runner="enpt_gitlab_CI_runner" # get sicor project diff --git a/tests/gitlab_CI_docker/context/enpt_ci.docker b/tests/gitlab_CI_docker/context/enpt_ci.docker index fe09f8c42e1c87b4fcf87ba604a8cf6b5d8c0abb..411e87615efdb53af0d603d41934e8722141e429 100644 --- a/tests/gitlab_CI_docker/context/enpt_ci.docker +++ b/tests/gitlab_CI_docker/context/enpt_ci.docker @@ -46,3 +46,5 @@ RUN bash -i -c "source /root/miniconda3/bin/activate enpt; \ # -> sicor root directory is the default directory of these cache files if sicor_cache_dir is not set in EnPT options COPY *.zip /tmp/sicor/sicor +# copy AC LUT for EnMAP to SICOR tables directory +COPY EnMAP_LUT* /tmp/sicor/sicor/tables diff --git a/tests/gitlab_CI_docker/context/environment_enpt.yml b/tests/gitlab_CI_docker/context/environment_enpt.yml index 00aa1bef807d9f1c9b8a48100da3bc04b6deea78..e00340468bb94d8c123f7355d1a221bb9c62956c 100644 --- a/tests/gitlab_CI_docker/context/environment_enpt.yml +++ b/tests/gitlab_CI_docker/context/environment_enpt.yml @@ -26,6 +26,7 @@ dependencies: - shapely - holoviews - bokeh + - pyresample # arosics - pyfftw @@ -44,7 +45,7 @@ dependencies: - pip: - scipy - geoarray>=0.8.9 - - py_tools_ds>=0.14.8 + - py_tools_ds>=0.14.14 - cerberus - jsmin - tqdm diff --git a/tests/linting/pydocstyle.log b/tests/linting/pydocstyle.log index 6c091c5d33f93714ea59e499e91d8cedacb5fc8b..402b1727f27c97ce746aa76883e0aa51c7908001 100644 --- a/tests/linting/pydocstyle.log +++ b/tests/linting/pydocstyle.log @@ -148,7 +148,7 @@ enpt/processors/dem_preprocessing/dem_preprocessing.py:65 in public method `to_s D102: Missing docstring in public method enpt/processors/dead_pixel_correction/dead_pixel_correction.py:18 in public class `Dead_Pixel_Corrector`: D101: Missing docstring in public class -enpt/processors/atmospheric_correction/atmospheric_correction.py:24 in public method `get_ac_options`: +enpt/processors/atmospheric_correction/atmospheric_correction.py:25 in public method `get_ac_options`: D102: Missing docstring in public method enpt/processors/atmospheric_correction/atmospheric_correction.py:47 in public method `run_ac`: D102: Missing docstring in public method @@ -172,33 +172,33 @@ enpt/execution/controller.py:94 in public method `run_geometry_processor`: D102: Missing docstring in public method enpt/execution/controller.py:106 in public method `write_output`: D102: Missing docstring in public method -enpt/options/config.py:79 in public class `EnPTConfig`: +enpt/options/config.py:81 in public class `EnPTConfig`: D101: Missing docstring in public class -enpt/options/config.py:80 in public method `__init__`: +enpt/options/config.py:82 in public method `__init__`: D202: No blank lines allowed after function docstring (found 1) -enpt/options/config.py:170 in public method `absPath`: +enpt/options/config.py:172 in public method `absPath`: D102: Missing docstring in public method -enpt/options/config.py:173 in public method `get_parameter`: +enpt/options/config.py:175 in public method `get_parameter`: D102: Missing docstring in public method -enpt/options/config.py:236 in public method `to_dict`: +enpt/options/config.py:238 in public method `to_dict`: D202: No blank lines allowed after function docstring (found 1) -enpt/options/config.py:250 in public method `to_jsonable_dict`: +enpt/options/config.py:252 in public method `to_jsonable_dict`: D102: Missing docstring in public method -enpt/options/config.py:261 in public method `__repr__`: +enpt/options/config.py:263 in public method `__repr__`: D105: Missing docstring in magic method -enpt/options/config.py:265 in public function `json_to_python`: +enpt/options/config.py:267 in public function `json_to_python`: D103: Missing docstring in public function -enpt/options/config.py:298 in public function `python_to_json`: +enpt/options/config.py:300 in public function `python_to_json`: D103: Missing docstring in public function -enpt/options/config.py:320 in public class `EnPTValidator`: +enpt/options/config.py:322 in public class `EnPTValidator`: D101: Missing docstring in public class -enpt/options/config.py:321 in public method `__init__`: +enpt/options/config.py:323 in public method `__init__`: D205: 1 blank line required between summary line and description (found 0) -enpt/options/config.py:321 in public method `__init__`: +enpt/options/config.py:323 in public method `__init__`: D400: First line should end with a period (not 'r') -enpt/options/config.py:329 in public method `validate`: +enpt/options/config.py:331 in public method `validate`: D102: Missing docstring in public method -enpt/options/config.py:334 in public function `get_options`: +enpt/options/config.py:336 in public function `get_options`: D202: No blank lines allowed after function docstring (found 1) enpt/options/__init__.py:1 at module level: D104: Missing docstring in public package diff --git a/tests/test_l1b_reader.py b/tests/test_l1b_reader.py index e3603ab783bbc14cd8ec7eb22578bc45e61d4ce7..328e9e0607b44f991a19ea8d980cde99c7c16236 100644 --- a/tests/test_l1b_reader.py +++ b/tests/test_l1b_reader.py @@ -122,7 +122,7 @@ class Test_L1B_Reader(unittest.TestCase): n_lines)) print("-----------------------------------------------------------------------------------------------") - if n_lines is "all": + if n_lines == "all": n_lines = None print("")