From d05b47c41a50d88c430b1ff650080f13c2ac7090 Mon Sep 17 00:00:00 2001 From: Stephane Guillaso Date: Wed, 16 May 2018 17:24:06 +0200 Subject: [PATCH 01/11] Initial commit with modified functions --- HISTORY.rst | 9 + enpt/io/reader.py | 154 ++++++++++----- enpt/model/images.py | 396 ++++++++++++++++++++++++++++++++------- enpt/model/metadata.py | 282 +++++++++++++++++++++------- tests/__init__.py | 4 +- tests/test_l1b_reader.py | 166 ++++++++++++---- 6 files changed, 794 insertions(+), 217 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index ee822f1..7160ca0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,15 @@ History ======= +0.x.x (2018-05-16) +------------------ +New features: + +* Use a new header file +* Add a second image at the bottom of the main image (if possible), select a possible number of line +* Save extended image + + 0.2.0 (2017-08-24) ------------------ diff --git a/enpt/io/reader.py b/enpt/io/reader.py index 54ad8ca..49e1528 100644 --- a/enpt/io/reader.py +++ b/enpt/io/reader.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """Reader module for reading all kinds of EnMAP images.""" -from datetime import datetime +# from datetime import datetime import logging import tempfile import zipfile @@ -9,58 +9,75 @@ import numpy as np from scipy.interpolate import interp1d from ..model.images import EnMAPL1Product_SensorGeo -from ..model.metadata import EnMAP_Metadata_L1B_SensorGeo +# from ..model.metadata import EnMAP_Metadata_L1B_SensorGeo from ..options.config import EnPTConfig class L1B_Reader(object): """Reader for EnMAP Level-1B products.""" - def __init__(self, config: EnPTConfig, logger: logging.Logger=None): + def __init__(self, + config: EnPTConfig, + logger: logging.Logger=None, + root_dir_main: str=None, + root_dir_ext: str=None, + n_line_ext: int=None): + # Add option to init as suggested. """Get an instance of L1B_Reader. :param config: instance of EnPTConfig class :param logger: instance of logging.Logger (NOTE: This logger is only used to log messages within L1B_Reader. It is not appended to the read L1B EnMAP object). + :param root_dir_main: Root directory of EnMAP Level-1B product (the main image) + :param root_dir_ext: Root directory of EnMAP Level-1B product (to extend the main image) + :param n_line_ext: [Optional] add number of line to be added to the main image from the extended image """ self.cfg = config self.logger = logger or logging.getLogger(__name__) - def read_inputdata(self, root_dir, observation_time: datetime, lon_lat_smpl: tuple=(15, 15), nsmile_coef: int=5, + # read data if root_dir_main is given or not + if root_dir_main is not None: + self.read_inputdata(root_dir_main, root_dir_ext, n_line_ext) + + def read_inputdata(self, + root_dir_main, + root_dir_ext: str=None, + n_line_ext: int=None, + lon_lat_smpl: tuple=(15,15), compute_snr: bool=True): - # TODO move to init? - """Read L1B, DEM and spatial reference data. - - :param root_dir: Root directory of EnMAP Level-1B product - :param observation_time: datetime of observation time (currently missing in metadata) - :param lon_lat_smpl: number if sampling points in lon, lat fields - :param nsmile_coef: number of polynomial coefficients for smile - :param compute_snr: whether to compute SNR or not (default: True) - :return: instance of EnMAPL1Product_MapGeo + # All information are read from data itself now + # In case of multiple files, temporary files are created to store them. + """ + Read L1B EnMAP data. Extend the image by adding a second image [entire, partial] + :param root_dir_main: Root directory of the main EnMAP Level-1B product + :param root_dir_ext: Root directory of the extended EnMAP Level-1B product [optional] + :param n_line_ext: Number of line to be added to the main image [if None, use the whole image] + :param lon_lat_smpl: number if sampling points in lon, lat fields + :param compute_snr: whether to compute SNR or not (default: True) + :return: instance of EnMAPL1Product_SensorGeo """ - # get a new instance of EnMAPL1Product_MapGeo - L1_obj = EnMAPL1Product_SensorGeo(root_dir, config=self.cfg) - - # read metadata - L1_obj.meta = EnMAP_Metadata_L1B_SensorGeo(L1_obj.paths.metaxml, config=self.cfg, logger=L1_obj.logger) - L1_obj.meta.read_metadata(observation_time=observation_time, lon_lat_smpl=lon_lat_smpl, nsmile_coef=nsmile_coef) - - # read VNIR data - # call L1_obj.vnir.arr.setter which sets L1_obj.swir.arr to an instance of GeoArray class - L1_obj.vnir.data = L1_obj.paths.vnir.data - L1_obj.vnir.mask_clouds = L1_obj.paths.vnir.mask_clouds - L1_obj.vnir.deadpixelmap = L1_obj.paths.vnir.deadpixelmap - L1_obj.vnir.detector_meta = L1_obj.meta.vnir - - # read SWIR data - # call L1_obj.swir.arr.setter which sets L1_obj.swir.arr to an instance of GeoArray class - L1_obj.swir.data = L1_obj.paths.swir.data - L1_obj.swir.mask_clouds = L1_obj.paths.swir.mask_clouds - L1_obj.swir.deadpixelmap = L1_obj.paths.swir.deadpixelmap - L1_obj.swir.detector_meta = L1_obj.meta.swir - - # compute radiance - L1_obj.DN2TOARadiance() + self.logger.info("Reading Input Data") + + # Get a new instance of the EnMAPL1Product_SensorGeo for the main image + l1b_main_obj = EnMAPL1Product_SensorGeo(root_dir_main, config=self.cfg, logger=self.logger, + lon_lat_smpl=lon_lat_smpl) + + # load data from the main object + l1b_main_obj.vnir.data = l1b_main_obj.paths.vnir.data + l1b_main_obj.vnir.mask_clouds = l1b_main_obj.paths.vnir.mask_clouds + l1b_main_obj.vnir.deadpixelmap = l1b_main_obj.paths.vnir.deadpixelmap + l1b_main_obj.swir.data = l1b_main_obj.paths.swir.data + l1b_main_obj.swir.mask_clouds = l1b_main_obj.paths.swir.mask_clouds + l1b_main_obj.swir.deadpixelmap = l1b_main_obj.paths.swir.deadpixelmap + l1b_main_obj.DN2TOARadiance() + + # in case of a second file, we create new files that will be temporary save into a temporary directory + # and their path will be stored into a the paths of l1b_main_obj + # NOTE: We do the following hypothesis: + # - The dead pixel map will not change when acquiring 2 adjacent images. + if root_dir_ext is not None: + l1b_ext_obj = EnMAPL1Product_SensorGeo(root_dir_ext, config=self.cfg, lon_lat_smpl=lon_lat_smpl) + l1b_main_obj.append_new_image(l1b_ext_obj, n_line_ext) # compute SNR if compute_snr: @@ -68,12 +85,65 @@ class L1B_Reader(object): zipfile.ZipFile(self.cfg.path_l1b_snr_model, "r") as zf: zf.extractall(tmpDir) - if L1_obj.meta.vnir.unitcode == 'TOARad': - L1_obj.vnir.detector_meta.calc_snr_from_radiance(rad_data=L1_obj.vnir.data, dir_snr_models=tmpDir) - if L1_obj.meta.swir.unitcode == 'TOARad': - L1_obj.swir.detector_meta.calc_snr_from_radiance(rad_data=L1_obj.swir.data, dir_snr_models=tmpDir) - - return L1_obj + if l1b_main_obj.meta.vnir.unitcode == 'TOARad': + l1b_main_obj.vnir.detector_meta.calc_snr_from_radiance(rad_data=l1b_main_obj.vnir.data, + dir_snr_models=tmpDir) + if l1b_main_obj.meta.swir.unitcode == 'TOARad': + l1b_main_obj.swir.detector_meta.calc_snr_from_radiance(rad_data=l1b_main_obj.swir.data, + dir_snr_models=tmpDir) + + + # Return the l1b_main_obj + return l1b_main_obj + + # def read_inputdata_old(self, root_dir, observation_time: datetime, lon_lat_smpl: tuple=(15, 15), nsmile_coef: int=5, + # compute_snr: bool=True): + # # TODO move to init? --> This has been added in the init phase (will call the new read_inputdata method + # """Read L1B, DEM and spatial reference data. + # + # :param root_dir: Root directory of EnMAP Level-1B product + # :param observation_time: datetime of observation time (currently missing in metadata) + # :param lon_lat_smpl: number if sampling points in lon, lat fields + # :param nsmile_coef: number of polynomial coefficients for smile + # :param compute_snr: whether to compute SNR or not (default: True) + # :return: instance of EnMAPL1Product_MapGeo + # """ + # # get a new instance of EnMAPL1Product_MapGeo + # L1_obj = EnMAPL1Product_SensorGeo(root_dir, config=self.cfg) + # + # # read metadata + # L1_obj.meta = EnMAP_Metadata_L1B_SensorGeo(L1_obj.paths.metaxml, config=self.cfg, logger=L1_obj.logger) + # L1_obj.meta.read_metadata(observation_time=observation_time, lon_lat_smpl=lon_lat_smpl, nsmile_coef=nsmile_coef) + # + # # read VNIR data + # # call L1_obj.vnir.arr.setter which sets L1_obj.swir.arr to an instance of GeoArray class + # L1_obj.vnir.data = L1_obj.paths.vnir.data + # L1_obj.vnir.mask_clouds = L1_obj.paths.vnir.mask_clouds + # L1_obj.vnir.deadpixelmap = L1_obj.paths.vnir.deadpixelmap + # L1_obj.vnir.detector_meta = L1_obj.meta.vnir + # + # # read SWIR data + # # call L1_obj.swir.arr.setter which sets L1_obj.swir.arr to an instance of GeoArray class + # L1_obj.swir.data = L1_obj.paths.swir.data + # L1_obj.swir.mask_clouds = L1_obj.paths.swir.mask_clouds + # L1_obj.swir.deadpixelmap = L1_obj.paths.swir.deadpixelmap + # L1_obj.swir.detector_meta = L1_obj.meta.swir + # + # # compute radiance + # L1_obj.DN2TOARadiance() + # + # # compute SNR + # if compute_snr: + # with tempfile.TemporaryDirectory(dir=self.cfg.working_dir) as tmpDir,\ + # zipfile.ZipFile(self.cfg.path_l1b_snr_model, "r") as zf: + # zf.extractall(tmpDir) + # + # if L1_obj.meta.vnir.unitcode == 'TOARad': + # L1_obj.vnir.detector_meta.calc_snr_from_radiance(rad_data=L1_obj.vnir.data, dir_snr_models=tmpDir) + # if L1_obj.meta.swir.unitcode == 'TOARad': + # L1_obj.swir.detector_meta.calc_snr_from_radiance(rad_data=L1_obj.swir.data, dir_snr_models=tmpDir) + # + # return L1_obj def validate_input(self): """Validate user inputs.""" diff --git a/enpt/model/images.py b/enpt/model/images.py index a6551ed..10939b3 100644 --- a/enpt/model/images.py +++ b/enpt/model/images.py @@ -4,13 +4,22 @@ import logging from types import SimpleNamespace import numpy as np -from os import path, sep, makedirs +from os import path, makedirs from shutil import copyfile -from xml.etree import ElementTree +from lxml import etree +from glob import glob +import utm +from scipy.interpolate import interp2d + +# Use to generate preview +import imageio +from skimage import exposure from geoarray import GeoArray, NoDataMask, CloudMask -from ..utils.path_generator import PathGenL1BProduct +# Not used (thank to pycharm) +# from enpt.options.config import EnPTConfig +# from ..utils.path_generator import PathGenL1BProduct from ..utils.logging import EnPT_Logger from ..model.metadata import EnMAP_Metadata_L1B_SensorGeo, EnMAP_Metadata_L1B_Detector_SensorGeo from ..options.config import EnPTConfig @@ -55,6 +64,7 @@ class _EnMAP_Image(object): self._deadpixelmap = None self._ac_options = {} self._ac_errors = None + self._subset = None # defaults self.entity_ID = '' @@ -105,8 +115,11 @@ class _EnMAP_Image(object): :return: instance of geoarray.GeoArray """ - # TODO this must return a subset if self.subset is not None - return self._arr + if self._subset is None: + return self._arr + + return GeoArray(self._arr.arr[self._subset[0]:self._subset[1], self._subset[2]:self._subset[3], :], + geotransform=self._arr.gt, projection=self._arr.prj) @data.setter def data(self, *geoArr_initArgs): @@ -287,7 +300,7 @@ class _EnMAP_Image(object): class EnMAP_Detector_SensorGeo(_EnMAP_Image): - """Class representing a single detector of an EnMAP image (as sensor geometry). + """Class representing a single detector of an EnMAP image (in sensor geometry). NOTE: - Inherits all attributes from _EnMAP_Image class. @@ -298,12 +311,13 @@ class EnMAP_Detector_SensorGeo(_EnMAP_Image): """ - def __init__(self, detector_name: str, root_dir: str, config: EnPTConfig, logger=None): + def __init__(self, detector_name: str, root_dir: str, config: EnPTConfig, logger=None, meta=None): """Get an instance of _EnMAP_Detector_SensorGeo. :param detector_name: 'VNIR' or 'SWIR' :param root_dir: :param logger: + :param meta: import meta if already loaded """ if detector_name not in ['VNIR', 'SWIR']: raise ValueError("'detector_name' must be 'VNIR' or 'SWIR'. Received %s!" % detector_name) @@ -315,25 +329,27 @@ class EnMAP_Detector_SensorGeo(_EnMAP_Image): # get all attributes of base class "_EnMAP_Image" super(EnMAP_Detector_SensorGeo, self).__init__() - self.paths = self.get_paths() + # instance an empty metadata object - self.detector_meta = \ - EnMAP_Metadata_L1B_Detector_SensorGeo(self.detector_name, config=self.cfg, logger=self.logger) + if meta is None: + self.detector_meta = \ + EnMAP_Metadata_L1B_Detector_SensorGeo(self.detector_name, config=self.cfg, logger=self.logger) + else: + self.detector_meta = meta def get_paths(self): - """Get all file paths associated with the current instance of _EnMAP_Detector_SensorGeo. - - :return: types.SimpleNamespace """ - pathGen = PathGenL1BProduct(self._root_dir, self.detector_name) + Get all file paths associated with the current instance of EnMAP_Detector_SensorGeo + These information are reading from the detector_meta + :return: paths as SimpleNamespace + """ paths = SimpleNamespace() paths.root_dir = self._root_dir - paths.metaxml = pathGen.get_path_metaxml() - paths.data = pathGen.get_path_data() - paths.mask_clouds = pathGen.get_path_cloudmask() - paths.deadpixelmap = pathGen.get_path_deadpixelmap() - paths.quicklook = pathGen.get_path_quicklook() - + paths.metaxml = glob(path.join(self._root_dir, "*_header.xml"))[0] + paths.data = path.join(self._root_dir, self.detector_meta.data_filename) + paths.mask_clouds = path.join(self._root_dir, self.detector_meta.cloud_mask_filename) + paths.deadpixelmap = path.join(self._root_dir, self.detector_meta.dead_pixel_filename) + paths.quicklook = path.join(self._root_dir, self.detector_meta.quicklook_filename) return paths def DN2TOARadiance(self): @@ -357,6 +373,25 @@ class EnMAP_Detector_SensorGeo(_EnMAP_Image): code=self.detector_meta.unitcode) ) + def generate_quicklook(self, filename): + """ + Generate image quicklook and save into a file + :param filename: file path to store the image + :return: None + """ + p2 = np.percentile(self.data[:, :, self.detector_meta.preview_bands[0]], 2) + p98 = np.percentile(self.data[:, :, self.detector_meta.preview_bands[0]], 98) + red_rescaled = exposure.rescale_intensity(self.data[:, :, self.detector_meta.preview_bands[0]], (p2, p98)) + p2 = np.percentile(self.data[:, :, self.detector_meta.preview_bands[1]], 2) + p98 = np.percentile(self.data[:, :, self.detector_meta.preview_bands[1]], 98) + green_rescaled = exposure.rescale_intensity(self.data[:, :, self.detector_meta.preview_bands[1]], (p2, p98)) + p2 = np.percentile(self.data[:, :, self.detector_meta.preview_bands[2]], 2) + p98 = np.percentile(self.data[:, :, self.detector_meta.preview_bands[2]], 98) + blue_rescaled = exposure.rescale_intensity(self.data[:, :, self.detector_meta.preview_bands[2]], (p2, p98)) + pix = np.dstack((red_rescaled, green_rescaled, blue_rescaled)) + pix = np.uint8(pix * 255) + imageio.imwrite(filename, pix) + class EnMAPL1Product_SensorGeo(object): """Class for EnPT EnMAP object in sensor geometry. @@ -377,7 +412,7 @@ class EnMAPL1Product_SensorGeo(object): """ - def __init__(self, root_dir: str, config: EnPTConfig, logger=None): + def __init__(self, root_dir: str, config: EnPTConfig, logger=None, lon_lat_smpl=None): """Get instance of EnPT EnMAP object in sensor geometry. :param root_dir: Root directory of EnMAP Level-1B product @@ -391,12 +426,68 @@ class EnMAPL1Product_SensorGeo(object): self.cfg = config if logger: self.logger = logger - self.vnir = EnMAP_Detector_SensorGeo('VNIR', root_dir, config=self.cfg, logger=self.logger) - self.swir = EnMAP_Detector_SensorGeo('SWIR', root_dir, config=self.cfg, logger=self.logger) - self.paths = self.get_paths() - self.meta = EnMAP_Metadata_L1B_SensorGeo(self.paths.metaxml, config=self.cfg, logger=self.logger) + + # Define the path. get_paths will be removed (see comments below) + # self.paths = self.get_paths() return glob(os.path.join(self.root_dir, "*_header.xml"))[0] + # self.paths = SimpleNamespace() + + # Read metadata here in order to get all information needed by to get paths. + # self.paths.metaxml = glob(path.join(root_dir, "*_header.xml"))[0] + self.meta = EnMAP_Metadata_L1B_SensorGeo(glob(path.join(root_dir, "*_header.xml"))[0], + config=self.cfg, logger=self.logger) + self.meta.read_metadata(lon_lat_smpl) + + # define VNIR and SWIR detector + self.vnir = EnMAP_Detector_SensorGeo('VNIR', root_dir, config=self.cfg, logger=self.logger, meta=self.meta.vnir) + self.swir = EnMAP_Detector_SensorGeo('SWIR', root_dir, config=self.cfg, logger=self.logger, meta=self.meta.swir) + + # Get the paths according information delivered in the metadata + self.paths = self.get_paths(root_dir) + # self.paths.root_dir = root_dir + # + # # Get paths for vnir + # self.paths.vnir = SimpleNamespace() + # self.paths.vnir.root_dir = root_dir + # self.paths.vnir.metaxml = glob(path.join(root_dir, "*_header.xml"))[0] + # self.paths.vnir.data = path.join(root_dir, self.meta.vnir.data_filename) + # self.paths.vnir.mask_clouds = path.join(root_dir, self.meta.vnir.cloud_mask_filename) + # self.paths.vnir.deadpixelmap = path.join(root_dir, self.meta.vnir.dead_pixel_filename) + # self.paths.vnir.quicklook = path.join(root_dir, self.meta.vnir.quicklook_filename) + # + # # Get paths for swir + # self.paths.swir = SimpleNamespace() + # self.paths.swir.root_dir = root_dir + # self.paths.swir.metaxml = glob(path.join(root_dir, "*_header.xml"))[0] + # self.paths.swir.data = path.join(root_dir, self.meta.swir.data_filename) + # self.paths.swir.mask_clouds = path.join(root_dir, self.meta.swir.cloud_mask_filename) + # self.paths.swir.deadpixelmap = path.join(root_dir, self.meta.swir.dead_pixel_filename) + # self.paths.swir.quicklook = path.join(root_dir, self.meta.swir.quicklook_filename) + self.detector_attrNames = ['vnir', 'swir'] + # Create a temporary tmp directory to store some data if needed + # self.tmpdir = tempfile.mkdtemp(dir=self.cfg.working_dir) + + # def __del__(self): + # """ + # Remove the (un)used temporary directory + # :return: None + # """ + # print("shutil.rmtree(self.tmpdir)") + + def get_paths(self, root_dir): + """ + Get all file paths associated with the current instance of EnMAPL1Product_SensorGeo + :param root_dir: directory where the data are located + :return: paths.SimpleNamespace() + """ + paths = SimpleNamespace() + paths.vnir = self.vnir.get_paths() + paths.swir = self.swir.get_paths() + paths.root_dir = root_dir + paths.metaxml = glob(path.join(root_dir, "*_header.xml"))[0] + return paths + @property def logger(self) -> EnPT_Logger: """Get a an instance of enpt.utils.logging.EnPT_Logger. @@ -449,18 +540,18 @@ class EnMAPL1Product_SensorGeo(object): assert isinstance(string, str), "'log' can only be set to a string. Got %s." % type(string) self.logger.captured_stream = string - def get_paths(self): - """Get all file paths associated with the current instance of EnMAPL1Product_SensorGeo. - - :return: types.SimpleNamespace() - """ - paths = SimpleNamespace() - paths.vnir = self.vnir.get_paths() - paths.swir = self.swir.get_paths() - paths.root_dir = paths.vnir.root_dir - paths.metaxml = paths.vnir.metaxml - - return paths + # def get_paths(self): + # """Get all file paths associated with the current instance of EnMAPL1Product_SensorGeo. + # + # :return: types.SimpleNamespace() + # """ + # paths = SimpleNamespace() + # paths.vnir = self.vnir.get_paths() + # paths.swir = self.swir.get_paths() + # paths.root_dir = paths.vnir.root_dir + # paths.metaxml = paths.vnir.metaxml + # + # return paths def DN2TOARadiance(self): """Convert self.vnir.data and self.swir.data from digital numbers to top-of-atmosphere radiance. @@ -469,56 +560,217 @@ class EnMAPL1Product_SensorGeo(object): """ if self.vnir.detector_meta.unitcode != 'TOARad': self.vnir.DN2TOARadiance() + self.meta.vnir.unitcode = self.vnir.detector_meta.unitcode + self.meta.vnir.unit = self.vnir.detector_meta.unit if self.swir.detector_meta.unitcode != 'TOARad': self.swir.DN2TOARadiance() + self.meta.swir.unitcode = self.swir.detector_meta.unitcode + self.meta.swir.unit = self.swir.detector_meta.unit + # Define a new save to take into account the fact that 2 images might be appended + # Here we save the radiance and not DN (otherwise there will be a problem with the concatened images) def save(self, outdir: str, suffix="") -> str: - """Save this product to disk using almost the same format as for reading. - - :param outdir: Path to output directory - :param suffix: Suffix to be appended to the output filename - :return: Root path of written product + """ + Save the product to disk using almost the same input format + :param outdir: path to the output directory + :param suffix: suffix to be appended to the output filename (???) + :return: root path (root directory) where products were written """ product_dir = path.join( path.abspath(outdir), "{name}{suffix}".format( - name=[ff for ff in self.paths.root_dir.split(sep) if ff != ''][-1], + name=[ff for ff in self.paths.root_dir.split(path.sep) if ff != ''][-1], suffix=suffix) ) self.logger.info("Write product to: %s" % product_dir) makedirs(product_dir, exist_ok=True) - for detector_name in self.detector_attrNames: - detector = getattr(self, detector_name) - detector_paths = getattr(self.paths, detector_name) + # We can hardcode the detectors (?) + # write the VNIR + self.vnir.data.save(product_dir + path.sep + self.meta.vnir.data_filename, fmt="ENVI") + self.vnir.mask_clouds.save(product_dir + path.sep + self.meta.vnir.cloud_mask_filename, fmt="GTiff") + self.vnir.deadpixelmap.save(product_dir + path.sep + self.meta.vnir.dead_pixel_filename, fmt="GTiff") + self.vnir.generate_quicklook(product_dir + path.sep + self.meta.vnir.quicklook_filename) + + # write the SWIR + self.swir.data.save(product_dir + path.sep + self.meta.swir.data_filename, fmt="ENVI") + self.swir.mask_clouds.save(product_dir + path.sep + self.meta.swir.cloud_mask_filename, fmt="GTiff") + self.swir.deadpixelmap.save(product_dir + path.sep + self.meta.swir.dead_pixel_filename, fmt="GTiff") + self.swir.generate_quicklook(product_dir + path.sep + self.meta.swir.quicklook_filename) + + # Update the xml file + xml = etree.parse(self.paths.metaxml) + xml.findall("ProductComponent/VNIRDetector/Data/Size/NRows")[0].text = str(self.meta.vnir.nrows) + xml.findall("ProductComponent/VNIRDetector/Data/Type/UnitCode")[0].text = self.meta.vnir.unitcode + xml.findall("ProductComponent/VNIRDetector/Data/Type/Unit")[0].text = self.meta.vnir.unit + xml.findall("ProductComponent/SWIRDetector/Data/Size/NRows")[0].text = str(self.meta.swir.nrows) + xml.findall("ProductComponent/SWIRDetector/Data/Type/UnitCode")[0].text = self.meta.swir.unitcode + xml.findall("ProductComponent/SWIRDetector/Data/Type/Unit")[0].text = self.meta.swir.unit + xml_string = etree.tostring(xml, pretty_print=True, xml_declaration=True, encoding='UTF-8') + xml_file = open(product_dir + path.sep + path.basename(self.paths.metaxml), "w") + xml_file.write(xml_string.decode("utf-8")) + xml_file.close() - for atts, fmt in ((("deadpixelmap", "mask_clouds"), "GTIff"), - (("data",), "ENVI")): - for att in atts: - getattr(detector, att).save( - path.join(product_dir, path.basename(getattr(detector_paths, att))), fmt=fmt) - - copyfile( - src=detector_paths.quicklook, - dst=path.join(product_dir, path.basename(detector_paths.quicklook)) - ) - - xml = ElementTree.parse(self.paths.metaxml) - for xml_name, real_name in (("detector1", "vnir"), ("detector2", "swir")): - ele = xml.getroot().find(xml_name) - - # add unitcode - new_ele = ElementTree.Element("unitcode") - new_ele.text = getattr(self.meta, real_name).unitcode - ele.append(new_ele) - - # add unit - new_ele = ElementTree.Element("unit") - new_ele.text = getattr(self.meta, real_name).unit - ele.append(new_ele) + return product_dir - xml.write(path.join(product_dir, path.basename(self.paths.metaxml))) - return product_dir + # def save_old(self, outdir: str, suffix="") -> str: + # """Save this product to disk using almost the same format as for reading. + # + # :param outdir: Path to output directory + # :param suffix: Suffix to be appended to the output filename + # :return: Root path of written product + # """ + # product_dir = path.join( + # path.abspath(outdir), "{name}{suffix}".format( + # name=[ff for ff in self.paths.root_dir.split(path.sep) if ff != ''][-1], + # suffix=suffix) + # ) + # self.logger.info("Write product to: %s" % product_dir) + # makedirs(product_dir, exist_ok=True) + # + # for detector_name in self.detector_attrNames: + # detector = getattr(self, detector_name) + # detector_paths = getattr(self.paths, detector_name) + # + # for atts, fmt in ((("deadpixelmap", "mask_clouds"), "GTIff"), + # (("data",), "ENVI")): + # for att in atts: + # getattr(detector, att).save( + # path.join(product_dir, path.basename(getattr(detector_paths, att))), fmt=fmt) + # + # copyfile( + # src=detector_paths.quicklook, + # dst=path.join(product_dir, path.basename(detector_paths.quicklook)) + # ) + # + # xml = ElementTree.parse(self.paths.metaxml) + # for xml_name, real_name in (("detector1", "vnir"), ("detector2", "swir")): + # ele = xml.getroot().find(xml_name) + # + # # add unitcode + # new_ele = ElementTree.Element("unitcode") + # new_ele.text = getattr(self.meta, real_name).unitcode + # ele.append(new_ele) + # + # # add unit + # new_ele = ElementTree.Element("unit") + # new_ele.text = getattr(self.meta, real_name).unit + # ele.append(new_ele) + # + # xml.write(path.join(product_dir, path.basename(self.paths.metaxml))) + # + # return product_dir + + def append_new_image(self, img2, n_lines: int=None): + """ + Check if a second image could pass with the first image. + In this version we assume that the image will be add below + If it is the case, the method will create temporary files that will be used in the following. + :param img2: + :param n_lines: number of line to be added + :return: None + """ + self.logger.info("Check new image %s" % img2.paths.root_dir) + + distance_min = 27.0 + distance_max = 34.0 + tag_vnir = False + tag_swir = False + + # check vnir bottom left + x1, y1, _, _ = utm.from_latlon(self.meta.vnir.lat_UL_UR_LL_LR[2], self.meta.vnir.lon_UL_UR_LL_LR[2]) + x2, y2, _, _ = utm.from_latlon(img2.meta.vnir.lat_UL_UR_LL_LR[0], img2.meta.vnir.lon_UL_UR_LL_LR[0]) + distance_left = np.sqrt((x1-x2)**2 + (y1-y2)**2) + + # check vnir bottom right + x1, y1, _, _ = utm.from_latlon(self.meta.vnir.lat_UL_UR_LL_LR[3], self.meta.vnir.lon_UL_UR_LL_LR[3]) + x2, y2, _, _ = utm.from_latlon(img2.meta.vnir.lat_UL_UR_LL_LR[1], img2.meta.vnir.lon_UL_UR_LL_LR[1]) + distance_right = np.sqrt((x1-x2)**2 + (y1-y2)**2) + + if distance_min < distance_left < distance_max and distance_min < distance_right < distance_max: + tag_vnir = True + + # check swir bottom left + x1, y1, _, _ = utm.from_latlon(self.meta.swir.lat_UL_UR_LL_LR[2], self.meta.swir.lon_UL_UR_LL_LR[2]) + x2, y2, _, _ = utm.from_latlon(img2.meta.swir.lat_UL_UR_LL_LR[0], img2.meta.swir.lon_UL_UR_LL_LR[0]) + distance_left = np.sqrt((x1-x2)**2 + (y1-y2)**2) + + # check swir bottom right + x1, y1, _, _ = utm.from_latlon(self.meta.swir.lat_UL_UR_LL_LR[3], self.meta.swir.lon_UL_UR_LL_LR[3]) + x2, y2, _, _ = utm.from_latlon(img2.meta.swir.lat_UL_UR_LL_LR[1], img2.meta.swir.lon_UL_UR_LL_LR[1]) + distance_right = np.sqrt((x1-x2)**2 + (y1-y2)**2) + + if distance_min < distance_left < distance_max and distance_min < distance_right < distance_max: + tag_swir = True + + if tag_vnir is False or tag_swir is False: + self.logger.warning("%s and %s don't fit to be appended" % (self.paths.root_dir, img2.paths.root_dir)) + return + + self.logger.info("Append new image: %s" % img2.paths.root_dir) + + # set new number of line + if n_lines is None: + n_lines = img2.meta.vnir.nrows + + if n_lines > img2.meta.vnir.nrows: + self.logger.warning("n_lines (%s) exceeds the total number of line of second image" % n_lines) + self.logger.warning("Set to the image number of line") + n_lines = img2.meta.vnir.nrows + + if n_lines < 50: # TODO: determine this values + self.logger.warning("A minimum of 50 lines is required, only %s were selected" % n_lines) + self.logger.warning("Set the number of line to 50") + n_lines = 50 + + + self.meta.vnir.nrows += n_lines + + # Create new corner coordinate - VNIR + ff = interp2d(x=[0, 1], y=[0, 1], z=[[img2.meta.vnir.lat_UL_UR_LL_LR[0], img2.meta.vnir.lat_UL_UR_LL_LR[1]], + [img2.meta.vnir.lat_UL_UR_LL_LR[2], img2.meta.vnir.lat_UL_UR_LL_LR[3]]], + kind='linear') + self.meta.vnir.lat_UL_UR_LL_LR[2] = ff(0, n_lines/img2.meta.vnir.nrows)[0] + self.meta.vnir.lat_UL_UR_LL_LR[3] = ff(1, n_lines/img2.meta.vnir.nrows)[0] + lon_lat_smpl = (15, 15) + self.meta.vnir.lats = self.meta.vnir.interpolate_corners(*self.meta.vnir.lat_UL_UR_LL_LR, *lon_lat_smpl) + + ff = interp2d(x=[0, 1], y=[0, 1], z=[[img2.meta.vnir.lon_UL_UR_LL_LR[0], img2.meta.vnir.lon_UL_UR_LL_LR[1]], + [img2.meta.vnir.lon_UL_UR_LL_LR[2], img2.meta.vnir.lon_UL_UR_LL_LR[3]]], + kind='linear') + self.meta.vnir.lon_UL_UR_LL_LR[2] = ff(0, n_lines/img2.meta.vnir.nrows)[0] + self.meta.vnir.lon_UL_UR_LL_LR[3] = ff(1, n_lines/img2.meta.vnir.nrows)[0] + self.meta.vnir.lons = self.meta.vnir.interpolate_corners(*self.meta.vnir.lon_UL_UR_LL_LR, *lon_lat_smpl) + + # Create new corner coordinate - SWIR + ff = interp2d(x=[0, 1], y=[0, 1], z=[[img2.meta.swir.lat_UL_UR_LL_LR[0], img2.meta.swir.lat_UL_UR_LL_LR[1]], + [img2.meta.swir.lat_UL_UR_LL_LR[2], img2.meta.swir.lat_UL_UR_LL_LR[3]]], + kind='linear') + self.meta.swir.lat_UL_UR_LL_LR[2] = ff(0, n_lines/img2.meta.swir.nrows)[0] + self.meta.swir.lat_UL_UR_LL_LR[3] = ff(1, n_lines/img2.meta.swir.nrows)[0] + lon_lat_smpl = (15, 15) + self.meta.swir.lats = self.meta.swir.interpolate_corners(*self.meta.swir.lat_UL_UR_LL_LR, *lon_lat_smpl) + ff = interp2d(x=[0, 1], y=[0, 1], z=[[img2.meta.vnir.lon_UL_UR_LL_LR[0], img2.meta.vnir.lon_UL_UR_LL_LR[1]], + [img2.meta.vnir.lon_UL_UR_LL_LR[2], img2.meta.vnir.lon_UL_UR_LL_LR[3]]], + kind='linear') + self.meta.swir.lon_UL_UR_LL_LR[2] = ff(0, n_lines/img2.meta.swir.nrows)[0] + self.meta.swir.lon_UL_UR_LL_LR[3] = ff(1, n_lines/img2.meta.swir.nrows)[0] + self.meta.swir.lons = self.meta.swir.interpolate_corners(*self.meta.swir.lon_UL_UR_LL_LR, *lon_lat_smpl) + + # append the vnir/swir image + img2.vnir.data = img2.paths.vnir.data + img2.vnir.data = img2.vnir.data[0:n_lines, :, :] + img2.swir.data = img2.paths.swir.data + img2.swir.data = img2.swir.data[0:n_lines, :, :] + img2.DN2TOARadiance() + self.vnir.data = np.append(self.vnir.data, img2.vnir.data, axis=0) + self.swir.data = np.append(self.swir.data, img2.swir.data, axis=0) + + # append the mask cloud + self.vnir.mask_clouds = np.append(self.vnir.mask_clouds, + GeoArray(img2.paths.vnir.mask_clouds)[0:n_lines, :], axis=0) + self.swir.mask_clouds = np.append(self.swir.mask_clouds, + GeoArray(img2.paths.swir.mask_clouds)[0:n_lines, :], axis=0) #################################### diff --git a/enpt/model/metadata.py b/enpt/model/metadata.py index cb32a0a..d9a8106 100644 --- a/enpt/model/metadata.py +++ b/enpt/model/metadata.py @@ -15,16 +15,27 @@ from ..options.config import EnPTConfig from .srf import SRF +# L1B_product_props = dict( +# xml_detector_label=dict( +# VNIR='detector1', +# SWIR='detector2'), +# fn_detector_suffix=dict( +# VNIR='D1', +# SWIR='D2') +# ) + +# Define a new L1B_product_props L1B_product_props = dict( xml_detector_label=dict( - VNIR='detector1', - SWIR='detector2'), + VNIR='VNIR', + SWIR='SWIR' + ), fn_detector_suffix=dict( VNIR='D1', - SWIR='D2') + SWIR='D2' + ) ) - ######################################################### # EnPT metadata objects for EnMAP data in sensor geometry ######################################################### @@ -49,6 +60,12 @@ class EnMAP_Metadata_L1B_Detector_SensorGeo(object): self.detector_label = L1B_product_props['xml_detector_label'][detector_name] self.logger = logger or logging.getLogger() + # These lines are used to load path information + self.data_filename = None # type: str # detector data filename + self.dead_pixel_filename = None # type: str # filename of the dead pixel file + self.quicklook_filename = None # type: str # filename of the quicklook file + self.cloud_mask_filename = None # type: str # filename of the cloud mask file + self.fwhm = None # type: np.ndarray # Full width half maximmum for each EnMAP band self.wvl_center = None # type: np.ndarray # Center wavelengths for each EnMAP band self.srf = None # type: SRF # SRF object holding the spectral response functions for each EnMAP band @@ -71,61 +88,136 @@ class EnMAP_Metadata_L1B_Detector_SensorGeo(object): self.lons = None # type: np.ndarray # 2D array of longitude coordinates according to given lon/lat sampling self.unit = '' # type: str # radiometric unit of pixel values self.unitcode = '' # type: str # code of radiometric unit + self.preview_bands = None self.snr = None # type: np.ndarray # Signal to noise ratio as computed from radiance data - def read_metadata(self, path_xml, lon_lat_smpl, nsmile_coef): - """Read the metadata of a specific EnMAP detector in sensor geometry. - - :param path_xml: file path of the metadata XML file - :param lon_lat_smpl: number if sampling points in lon, lat fields - :param nsmile_coef: number of polynomial coefficients for smile + # On this new version of read_data, we don't need anymore nsmile_coef (will be read from xml file) + def read_metadata(self, path_xml, lon_lat_smpl): + """ + Read the meadata of a specific EnMAP detector in sensor geometry + :param path_xml: file path of the metadata file + :param lon_lat_smpl: number if sampling in lon, lat fields + :return: None """ xml = ElementTree.parse(path_xml).getroot() - lbl = self.detector_label + lbl = self.detector_label + "Detector" self.logger.info("Reading metadata for %s detector..." % self.detector_name) - self.fwhm = np.array(xml.findall("%s/fwhm" % lbl)[0].text.replace("\n", "").split(), dtype=np.float) - self.wvl_center = np.array( - xml.findall("%s/centre_wavelength" % lbl)[0].text.replace("\n", "").split(), dtype=np.float) + # read data filenames + self.data_filename = xml.findall("ProductComponent/%s/Data/Filename" % lbl)[0].text + self.dead_pixel_filename = xml.findall("ProductComponent/%s/Sensor/DeadPixel/Filename" % lbl)[0].text + self.quicklook_filename = xml.findall("ProductComponent/%s/Preview/Filename" % lbl)[0].text + self.cloud_mask_filename = xml.findall("ProductComponent/%s/Data/CloudMaskMap/Filename" % lbl)[0].text + + # read preview bands + self.preview_bands = np.zeros(3, dtype=np.int) + self.preview_bands[0] = np.int(xml.findall("ProductComponent/%s/Preview/Bands/Red" % lbl)[0].text) + self.preview_bands[1] = np.int(xml.findall("ProductComponent/%s/Preview/Bands/Green" % lbl)[0].text) + self.preview_bands[2] = np.int(xml.findall("ProductComponent/%s/Preview/Bands/Blue" % lbl)[0].text) + + # read some basic information concerning the detector + self.nrows = np.int(xml.findall("ProductComponent/%s/Data/Size/NRows" % lbl)[0].text) + self.ncols = np.int(xml.findall("ProductComponent/%s/Data/Size/NCols" % lbl)[0].text) + self.geom_view_zenith = np.float(xml.findall("GeneralInfo/Geometry/Observation/ZenithAngle")[0].text) + self.geom_view_azimuth = np.float(xml.findall("GeneralInfo/Geometry/Observation/AzimuthAngle")[0].text) + self.geom_sun_zenith = np.float(xml.findall("GeneralInfo/Geometry/Illumination/ZenithAngle")[0].text) + self.geom_sun_azimuth = np.float(xml.findall("GeneralInfo/Geometry/Illumination/AzimuthAngle")[0].text) + self.unitcode = xml.findall("ProductComponent/%s/Data/Type/UnitCode" % lbl)[0].text + self.unit = xml.findall("ProductComponent/%s/Data/Type/Unit" % lbl)[0].text + + # Read image coordinates + scene_corner_coordinates = xml.findall("ProductComponent/%s/Data/SceneInformation/SceneCornerCoordinates" % lbl) + self.lat_UL_UR_LL_LR = [ + np.float(scene_corner_coordinates[0].findall("Latitude")[0].text), + np.float(scene_corner_coordinates[1].findall("Latitude")[0].text), + np.float(scene_corner_coordinates[2].findall("Latitude")[0].text), + np.float(scene_corner_coordinates[3].findall("Latitude")[0].text) + ] + self.lon_UL_UR_LL_LR = [ + np.float(scene_corner_coordinates[0].findall("Longitude")[0].text), + np.float(scene_corner_coordinates[1].findall("Longitude")[0].text), + np.float(scene_corner_coordinates[2].findall("Longitude")[0].text), + np.float(scene_corner_coordinates[3].findall("Longitude")[0].text) + ] + + # read the band related information: wavelength, fwhm + self.nwvl = np.int(xml.findall("ProductComponent/%s/Data/BandInformationList/NumberOfBands" % lbl)[0].text) + self.nsmile_coef = np.int(xml.findall( + "ProductComponent/%s/Data/BandInformationList/SmileInformation/NumberOfCoefficients" % lbl)[0].text) + self.fwhm = np.zeros(self.nwvl, dtype=np.float) + self.wvl_center = np.zeros(self.nwvl, dtype=np.float) + self.smile_coef = np.zeros((self.nwvl, self.nsmile_coef), dtype=np.float) + self.l_min = np.zeros(self.nwvl, dtype=np.float) + self.l_max = np.zeros(self.nwvl, dtype=np.float) + band_informations = xml.findall("ProductComponent/%s/Data/BandInformationList/BandInformation" % lbl) + for bi in band_informations: + k = np.int64(bi.attrib['Id']) - 1 + self.wvl_center[k] = np.float(bi.findall("CenterWavelength")[0].text) + self.fwhm[k] = np.float(bi.findall("FullWidthHalfMaximum")[0].text) + self.l_min[k] = np.float(bi.findall("L_min")[0].text) + self.l_max[k] = np.float(bi.findall("L_max")[0].text) + scl = bi.findall("Smile/Coefficient") + for sc in scl: + self.smile_coef[k, np.int64(sc.attrib['exponent'])] = np.float(sc.text) + self.smile = self.calc_smile() self.srf = SRF.from_cwl_fwhm(self.wvl_center, self.fwhm) self.solar_irrad = self.calc_solar_irradiance_CWL_FWHM_per_band() - self.nwvl = len(self.wvl_center) - self.nrows = np.int(xml.findall("%s/rows" % lbl)[0].text) - self.ncols = np.int(xml.findall("%s/columns" % lbl)[0].text) - self.smile_coef = np.array(xml.findall("%s/smile" % lbl)[0].text.replace("\n", "").split(), dtype=np.float) \ - .reshape((-1, nsmile_coef + 1))[:, 1:] - self.nsmile_coef = nsmile_coef - self.smile = self.calc_smile() - self.l_min = np.array(xml.findall("%s/L_min" % lbl)[0].text.split(), dtype=np.float) - self.l_max = np.array(xml.findall("%s/L_max" % lbl)[0].text.split(), dtype=np.float) - self.geom_view_zenith = np.float( - xml.findall("%s/observation_geometry/zenith_angle" % lbl)[0].text.split()[0]) - self.geom_view_azimuth = np.float( - xml.findall("%s/observation_geometry/azimuth_angle" % lbl)[0].text.split()[0]) - self.geom_sun_zenith = np.float( - xml.findall("%s/illumination_geometry/zenith_angle" % lbl)[0].text.split()[0]) - self.geom_sun_azimuth = np.float( - xml.findall("%s/illumination_geometry/azimuth_angle" % lbl)[0].text.split()[0]) - self.lat_UL_UR_LL_LR = \ - [float(xml.findall("%s/geometry/bounding_box/%s_northing" % (lbl, corner))[0].text.split()[0]) - for corner in ("UL", "UR", "LL", "LR")] - self.lon_UL_UR_LL_LR = \ - [float(xml.findall("%s/geometry/bounding_box/%s_easting" % (lbl, corner))[0].text.split()[0]) - for corner in ("UL", "UR", "LL", "LR")] self.lats = self.interpolate_corners(*self.lat_UL_UR_LL_LR, *lon_lat_smpl) self.lons = self.interpolate_corners(*self.lon_UL_UR_LL_LR, *lon_lat_smpl) - try: - self.unitcode = xml.findall("%s/unitcode" % lbl)[0].text - # '" ".join(xml.findall("%s/radiance_unit" % lbl)[0].text.split()) - self.unit = xml.findall("%s/unit" % lbl)[0].text - except IndexError: - self.unitcode = 'DN' - self.unit = 'none' - except Exception: - raise - - self.snr = None + # def read_metadata_old(self, path_xml, lon_lat_smpl, nsmile_coef): + # """Read the metadata of a specific EnMAP detector in sensor geometry. + # + # :param path_xml: file path of the metadata XML file + # :param lon_lat_smpl: number if sampling points in lon, lat fields + # :param nsmile_coef: number of polynomial coefficients for smile + # """ + # xml = ElementTree.parse(path_xml).getroot() + # lbl = self.detector_label + # self.logger.info("Reading metadata for %s detector..." % self.detector_name) + + # self.fwhm = np.array(xml.findall("%s/fwhm" % lbl)[0].text.replace("\n", "").split(), dtype=np.float) + # self.wvl_center = np.array( + # xml.findall("%s/centre_wavelength" % lbl)[0].text.replace("\n", "").split(), dtype=np.float) + # self.srf = SRF.from_cwl_fwhm(self.wvl_center, self.fwhm) + # self.solar_irrad = self.calc_solar_irradiance_CWL_FWHM_per_band() + # self.nwvl = len(self.wvl_center) + # self.nrows = np.int(xml.findall("%s/rows" % lbl)[0].text) + # self.ncols = np.int(xml.findall("%s/columns" % lbl)[0].text) + # self.smile_coef = np.array(xml.findall("%s/smile" % lbl)[0].text.replace("\n", "").split(), dtype=np.float) \ + # .reshape((-1, nsmile_coef + 1))[:, 1:] + # self.nsmile_coef = nsmile_coef + # self.smile = self.calc_smile() + # self.l_min = np.array(xml.findall("%s/L_min" % lbl)[0].text.split(), dtype=np.float) + # self.l_max = np.array(xml.findall("%s/L_max" % lbl)[0].text.split(), dtype=np.float) + # self.geom_view_zenith = np.float( + # xml.findall("%s/observation_geometry/zenith_angle" % lbl)[0].text.split()[0]) + # self.geom_view_azimuth = np.float( + # xml.findall("%s/observation_geometry/azimuth_angle" % lbl)[0].text.split()[0]) + # self.geom_sun_zenith = np.float( + # xml.findall("%s/illumination_geometry/zenith_angle" % lbl)[0].text.split()[0]) + # self.geom_sun_azimuth = np.float( + # xml.findall("%s/illumination_geometry/azimuth_angle" % lbl)[0].text.split()[0]) + # self.lat_UL_UR_LL_LR = \ + # [float(xml.findall("%s/geometry/bounding_box/%s_northing" % (lbl, corner))[0].text.split()[0]) + # for corner in ("UL", "UR", "LL", "LR")] + # self.lon_UL_UR_LL_LR = \ + # [float(xml.findall("%s/geometry/bounding_box/%s_easting" % (lbl, corner))[0].text.split()[0]) + # for corner in ("UL", "UR", "LL", "LR")] + # self.lats = self.interpolate_corners(*self.lat_UL_UR_LL_LR, *lon_lat_smpl) + # self.lons = self.interpolate_corners(*self.lon_UL_UR_LL_LR, *lon_lat_smpl) + # + # try: + # self.unitcode = xml.findall("%s/unitcode" % lbl)[0].text + # # '" ".join(xml.findall("%s/radiance_unit" % lbl)[0].text.split()) + # self.unit = xml.findall("%s/unit" % lbl)[0].text + # except IndexError: + # self.unitcode = 'DN' + # self.unit = 'none' + # except Exception: + # raise + # + # self.snr = None def calc_smile(self): """Compute smile for each EnMAP column. @@ -195,8 +287,12 @@ class EnMAP_Metadata_L1B_Detector_SensorGeo(object): @staticmethod def interpolate_corners(ul: float, ur: float, ll: float, lr: float, nx: int, ny: int): """Compute interpolated field from corner values of a scalar field given at: ul, ur, ll, lr. - - :param nx, ny: final shape + :param ul: tbd + :param ur: tbd + :param ll: tbd + :param lr: tbd + :param nx: final shape (x-axis direction) + :param ny: final shape (y-axis direction) """ ff = interp2d(x=[0, 1], y=[0, 1], z=[[ul, ur], [ll, lr]], kind='linear') rr = np.zeros((nx, ny), dtype=np.float) @@ -232,6 +328,10 @@ class EnMAP_Metadata_L1B_SensorGeo(object): Attributes: - logger(logging.Logger): None or logging instance - observation_datetime(datetime.datetime): datetime of observation time (currently missing in metadata) + - geom_view_zenith: tbd + - geom_view_azimuth: tbd + - geom_sun_zenith: tbd + - geom_sun_azimuth: tbd - vnir(EnMAP_Metadata_VNIR_SensorGeo) - swir(EnMAP_Metadata_SWIR_SensorGeo) @@ -247,28 +347,56 @@ class EnMAP_Metadata_L1B_SensorGeo(object): self.logger = logger or logging.getLogger() self._path_xml = path_metaxml - # defaults + # defaults - Common self.observation_datetime = None # type: datetime # Date and Time of image observation + self.geom_view_zenith = None # type: float # tbd + self.geom_view_azimuth = None # type: float # tbd + self.geom_sun_zenith = None # type: float # tbd + self.geom_sun_azimuth = None # type: float # tbd self.earthSunDist = None # type: float # earth-sun distance # TODO doc correct? self.vnir = None # type: EnMAP_Metadata_L1B_Detector_SensorGeo # metadata of VNIR only self.swir = None # type: EnMAP_Metadata_L1B_Detector_SensorGeo # metadata of SWIR only self.detector_attrNames = ['vnir', 'swir'] - def read_common_meta(self, observation_time: datetime=None): - """Read the metadata belonging to both, the VNIR and SWIR detector of the EnMAP L1B product in sensor geometry. - - :param observation_time: date and time of image observation (datetime.datetime) + # Read common metadata method + def read_common_meta(self, path_xml): + """Read the common metadata, principally stored in General Info + - the acquisition time + - the geometrical observation and illumination + :param path_xml: path to the main xml file + :return: None """ - # FIXME observation time is currently missing in the XML - self.observation_datetime = observation_time - self.earthSunDist = self.get_earth_sun_distance(self.observation_datetime) + + # load the metadata xml file + xml = ElementTree.parse(path_xml).getroot() + + # read the acquisition time + self.observation_datetime = xml.findall("GeneralInfo/ProductInfo/ProductStartTime")[0].text + + # get the distance earth sun from the acquisition date + self.earthSunDist = \ + self.get_earth_sun_distance(datetime.strptime(self.observation_datetime, '%Y-%m-%dT%H:%M:%S.%fZ')) + + # read Geometry (observation/illumination) angle + self.geom_view_zenith = np.float(xml.findall("GeneralInfo/Geometry/Observation/ZenithAngle")[0].text) + self.geom_view_azimuth = np.float(xml.findall("GeneralInfo/Geometry/Observation/AzimuthAngle")[0].text) + self.geom_sun_zenith = np.float(xml.findall("GeneralInfo/Geometry/Illumination/ZenithAngle")[0].text) + self.geom_sun_azimuth = np.float(xml.findall("GeneralInfo/Geometry/Illumination/AzimuthAngle")[0].text) + + # def read_common_meta_old(self, observation_time: datetime=None): + # """Read the metadata belonging to both, the VNIR and SWIR detector of the EnMAP L1B product in sensor geometry. + # + # :param observation_time: date and time of image observation (datetime.datetime) + # """ + # # FIXME observation time is currently missing in the XML -- DONE + # self.observation_datetime = observation_time + # self.earthSunDist = self.get_earth_sun_distance(self.observation_datetime) def get_earth_sun_distance(self, acqDate: datetime): """Get earth sun distance (requires file of pre calculated earth sun distance per day) :param acqDate: """ - if not os.path.exists(self.cfg.path_earthSunDist): self.logger.warning("\n\t WARNING: Earth Sun Distance is assumed to be " "1.0 because no database can be found at %s.""" % self.cfg.path_earthSunDist) @@ -286,15 +414,33 @@ class EnMAP_Metadata_L1B_SensorGeo(object): return float(EA_dist_dict[acqDate.strftime('%Y-%m-%d')]) - def read_metadata(self, observation_time: datetime, lon_lat_smpl, nsmile_coef): - """Read the metadata of the whole EnMAP L1B product in sensor geometry. - - :param observation_time: date and time of image observation (datetime.datetime) - :param lon_lat_smpl: number if sampling points in lon, lat fields - :param nsmile_coef: number of polynomial coefficients for smile + def read_metadata(self, lon_lat_smpl): + """ + Read the metadata of the entire EnMAP L1B product in sensor geometry + :param lon_lat_smpl: number if sampling point in lon, lat fields + :return: None """ - self.read_common_meta(observation_time) + + # first read common metadata + self.read_common_meta(self._path_xml) + + # define and read the VNIR metadata self.vnir = EnMAP_Metadata_L1B_Detector_SensorGeo('VNIR', config=self.cfg, logger=self.logger) - self.vnir.read_metadata(self._path_xml, lon_lat_smpl=lon_lat_smpl, nsmile_coef=nsmile_coef) + self.vnir.read_metadata(self._path_xml, lon_lat_smpl) + + # define and read the SWIR metadata self.swir = EnMAP_Metadata_L1B_Detector_SensorGeo('SWIR', config=self.cfg, logger=self.logger) - self.swir.read_metadata(self._path_xml, lon_lat_smpl=lon_lat_smpl, nsmile_coef=nsmile_coef) + self.swir.read_metadata(self._path_xml, lon_lat_smpl) + + # def read_metadata_old(self, observation_time: datetime, lon_lat_smpl, nsmile_coef): + # """Read the metadata of the whole EnMAP L1B product in sensor geometry. + # + # :param observation_time: date and time of image observation (datetime.datetime) + # :param lon_lat_smpl: number if sampling points in lon, lat fields + # :param nsmile_coef: number of polynomial coefficients for smile + # """ + # self.read_common_meta(observation_time) + # self.vnir = EnMAP_Metadata_L1B_Detector_SensorGeo('VNIR', config=self.cfg, logger=self.logger) + # self.vnir.read_metadata(self._path_xml, lon_lat_smpl=lon_lat_smpl, nsmile_coef=nsmile_coef) + # self.swir = EnMAP_Metadata_L1B_Detector_SensorGeo('SWIR', config=self.cfg, logger=self.logger) + # self.swir.read_metadata(self._path_xml, lon_lat_smpl=lon_lat_smpl, nsmile_coef=nsmile_coef) diff --git a/tests/__init__.py b/tests/__init__.py index 6093974..576285a 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -5,6 +5,8 @@ import os enptRepo_rootpath = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) config_for_testing = dict( - path_l1b_enmap_image=os.path.join(enptRepo_rootpath, 'tests', 'data', 'EnMAP_Level_1B', 'AlpineTest1_CWV2_SM0.zip'), + path_l1b_enmap_image=os.path.join(enptRepo_rootpath, 'tests', 'data', 'EnMAP_Level_1B_v2', 'Alpine1_CWV2_SM0'), + path_l1b_enmap_image_ext=os.path.join(enptRepo_rootpath, 'tests', 'data', 'EnMAP_Level_1B_v2', 'Alpine2_CWV2_SM0'), + path_l1b_enmap_image_ext2=os.path.join(enptRepo_rootpath, 'tests', 'data', 'EnMAP_Level_1B_v2', 'Alpine3_CWV2_SM0'), output_dir=os.path.join(enptRepo_rootpath, 'tests', 'data', 'test_outputs') ) diff --git a/tests/test_l1b_reader.py b/tests/test_l1b_reader.py index 62a6026..1490c23 100644 --- a/tests/test_l1b_reader.py +++ b/tests/test_l1b_reader.py @@ -12,9 +12,10 @@ import unittest from glob import glob import os from os import path +import sys import tempfile import zipfile -from datetime import datetime +# from datetime import datetime import shutil from enpt.options.config import EnPTConfig @@ -31,53 +32,150 @@ class Test_L1B_Reader(unittest.TestCase): def tearDown(self): shutil.rmtree(self.tmpdir) - shutil.rmtree(self.config.output_dir) + # shutil.rmtree(self.config.output_dir) def test_read_and_save_inputdata(self): from enpt.io.reader import L1B_Reader from enpt.model.images import EnMAPL1Product_SensorGeo - print("Test reading EnMAP Level-1B products") + print("") + print("################################################") + print("# #") + print("# Test reading EnMAP Level-1B products from SG #") + print("# #") + print("################################################") + print("") + print("") + + print("================================================") + print("Create the L1B_Reader new instance") + print("================================================") + print("") + print("") rd = L1B_Reader(config=self.config) - for l1b_file in self.pathList_testimages: - print("Tmp dir: %s" % self.tmpdir) - with zipfile.ZipFile(l1b_file, "r") as zf: - zf.extractall(self.tmpdir) - - root_dir = os.path.join(self.tmpdir, os.path.basename(l1b_file).split(".zip")[0]) - - ############### - # without snr # - ############### - - # read and write L1 data - L1_obj = rd.read_inputdata(root_dir, observation_time=datetime(2015, 12, 7, 10), compute_snr=False) + prods = (config_for_testing['path_l1b_enmap_image'], + config_for_testing['path_l1b_enmap_image_ext'], + config_for_testing['path_l1b_enmap_image_ext2']) + + # TEST FOR ONE IMAGE ONLY + print("=======================================================================================================") + print("Test: Read ONE image only") + print("=======================================================================================================") + print("") + + for prod in prods: + print("-------------------------------------") + print("Test with %s" % os.path.basename(prod)) + print("-------------------------------------") + print("") + print(" * Without SNR ") + print("") + L1_obj = rd.read_inputdata(prod, compute_snr=False) + self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) + self.assertIsNone(L1_obj.vnir.detector_meta.snr) + self.assertIsNone(L1_obj.swir.detector_meta.snr) + # root_dir_written_L1_data = L1_obj.save(path.join(self.tmpdir, "no_snr")) + # L1_obj = rd.read_inputdata(root_dir_written_L1_data, compute_snr=False) + # self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) + print("") + print(" * With SNR ") + print("") + L1_obj = rd.read_inputdata(prod) + self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) + self.assertIsNotNone(L1_obj.vnir.detector_meta.snr) + self.assertIsNotNone(L1_obj.swir.detector_meta.snr) + # root_dir_written_L1_data = L1_obj.save(path.join(self.config.output_dir, "with_snr")) + # L1_obj = rd.read_inputdata(root_dir_written_L1_data, compute_snr=False) + # self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) + print("") + print("") + + print("=======================================================================================================") + print("Test: Write ONE image only") + print("=======================================================================================================") + print("") + print("-----------------------------------------------------------------------------------------------") + print("Test with %s" % os.path.basename(prods[0])) + print("-----------------------------------------------------------------------------------------------") + print("") + print(" * With SNR ") + print("") + L1_obj = rd.read_inputdata(prods[0]) + root_dir_written_L1_data = L1_obj.save(path.join(self.config.output_dir, "with_snr")) + L1_obj = rd.read_inputdata(root_dir_written_L1_data, compute_snr=False) + self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) + shutil.rmtree(path.join(self.config.output_dir, "with_snr")) + print("") + print("") + + + # TEST FOR TWO IMAGES + print("=======================================================================================================") + print("Test: Read TWO images") + print("=======================================================================================================") + print("") + for prod1, prod2 in ((0, 1), + (1, 0), + (0, 2), + (2, 0), + (1, 2), + (2, 1)): + for n_lines in (10, 50, 100, 500, 1000, 1500): + print("-----------------------------------------------------------------------------------------------") + print("Test with %s and %s, with: %s lines" % (os.path.basename(prods[prod1]), + os.path.basename(prods[prod2]), + n_lines)) + print("-----------------------------------------------------------------------------------------------") + print("") + print(" * Without SNR") + print("") + L1_obj = rd.read_inputdata(prods[prod1], prods[prod2], n_lines, compute_snr=False) self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) self.assertIsNone(L1_obj.vnir.detector_meta.snr) self.assertIsNone(L1_obj.swir.detector_meta.snr) - root_dir_written_L1_data = L1_obj.save(path.join(self.tmpdir, "no_snr")) - - # read self written L1 data - L1_obj = rd.read_inputdata(root_dir_written_L1_data, observation_time=datetime(2015, 12, 7, 10), - compute_snr=False) - self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) - - ############ - # with snr # - ############ - - # read and write L1 data - L1_obj = rd.read_inputdata(root_dir, observation_time=datetime(2015, 12, 7, 10)) + # root_dir_written_L1_data = L1_obj.save(path.join(self.tmpdir, "no_snr")) + # L1_obj = rd.read_inputdata(root_dir_written_L1_data, compute_snr=False) + # self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) + print("") + print(" * with SNR ") + print("") + L1_obj = rd.read_inputdata(prods[prod1], prods[prod2], n_lines) self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) self.assertIsNotNone(L1_obj.vnir.detector_meta.snr) self.assertIsNotNone(L1_obj.swir.detector_meta.snr) - root_dir_written_L1_data = L1_obj.save(path.join(self.config.output_dir, "with_snr")) - - # read self written L1 data - L1_obj = rd.read_inputdata(root_dir_written_L1_data, observation_time=datetime(2015, 12, 7, 10)) - self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) + # root_dir_written_L1_data = \ + # L1_obj.save(path.join(self.config.output_dir, + # "with_snr_%s_%s_%s_lines" % (os.path.basename(prods[prod1]), + # os.path.basename(prods[prod2]), + # n_lines)) + # ) + # L1_obj = rd.read_inputdata(root_dir_written_L1_data, compute_snr=False) + # self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) + print("") + print("") + + print("=======================================================================================================") + print("Test: Write TWO images ") + print("=======================================================================================================") + print("") + print("-----------------------------------------------------------------------------------------------") + print("Test with %s and %s, with: %s lines" % (os.path.basename(prods[0]),os.path.basename(prods[1]), 500)) + print("-----------------------------------------------------------------------------------------------") + print("") + print(" * With SNR ") + print("") + L1_obj = rd.read_inputdata(prods[0], prods[1], 500) + root_dir_written_L1_data = L1_obj.save(path.join(self.config.output_dir, "with_snr")) + L1_obj = rd.read_inputdata(root_dir_written_L1_data, compute_snr=False) + self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) + shutil.rmtree(path.join(self.config.output_dir, "with_snr")) + print("") + print("") + + + return if __name__ == "__main__": -- GitLab From 86d195c359e66ea097b30f00ab4d21e7b291ce85 Mon Sep 17 00:00:00 2001 From: Stephane Guillaso Date: Tue, 5 Jun 2018 09:38:08 +0200 Subject: [PATCH 02/11] Updated test data with 2 images --- tests/__init__.py | 20 +-- .../AlpineTest1_CWV2_SM0.zip | 3 + .../AlpineTest2_CWV2_SM0.zip | 3 + tests/test_l1b_reader.py | 141 ++++++++---------- 4 files changed, 77 insertions(+), 90 deletions(-) create mode 100644 tests/data/EnMAP_Level_1B_new/AlpineTest1_CWV2_SM0.zip create mode 100644 tests/data/EnMAP_Level_1B_new/AlpineTest2_CWV2_SM0.zip diff --git a/tests/__init__.py b/tests/__init__.py index 576285a..9a0356e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- -import os - -enptRepo_rootpath = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) - -config_for_testing = dict( - path_l1b_enmap_image=os.path.join(enptRepo_rootpath, 'tests', 'data', 'EnMAP_Level_1B_v2', 'Alpine1_CWV2_SM0'), - path_l1b_enmap_image_ext=os.path.join(enptRepo_rootpath, 'tests', 'data', 'EnMAP_Level_1B_v2', 'Alpine2_CWV2_SM0'), - path_l1b_enmap_image_ext2=os.path.join(enptRepo_rootpath, 'tests', 'data', 'EnMAP_Level_1B_v2', 'Alpine3_CWV2_SM0'), - output_dir=os.path.join(enptRepo_rootpath, 'tests', 'data', 'test_outputs') -) +# import os +# +# enptRepo_rootpath = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +# +# config_for_testing = dict( +# path_l1b_enmap_image=os.path.join(enptRepo_rootpath, 'tests', 'data', 'EnMAP_Level_1B_v2', 'Alpine1_CWV2_SM0'), +# path_l1b_enmap_image_ext=os.path.join(enptRepo_rootpath, 'tests', 'data', 'EnMAP_Level_1B_v2', 'Alpine2_CWV2_SM0'), +# path_l1b_enmap_image_ext2=os.path.join(enptRepo_rootpath, 'tests', 'data', 'EnMAP_Level_1B_v2', 'Alpine3_CWV2_SM0'), +# output_dir=os.path.join(enptRepo_rootpath, 'tests', 'data', 'test_outputs') +# ) diff --git a/tests/data/EnMAP_Level_1B_new/AlpineTest1_CWV2_SM0.zip b/tests/data/EnMAP_Level_1B_new/AlpineTest1_CWV2_SM0.zip new file mode 100644 index 0000000..a283ed6 --- /dev/null +++ b/tests/data/EnMAP_Level_1B_new/AlpineTest1_CWV2_SM0.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed9d2db0510b1882f766b0439f12a94bfca2b068dedd11b3652cadfc1a5ed40c +size 46793447 diff --git a/tests/data/EnMAP_Level_1B_new/AlpineTest2_CWV2_SM0.zip b/tests/data/EnMAP_Level_1B_new/AlpineTest2_CWV2_SM0.zip new file mode 100644 index 0000000..3027098 --- /dev/null +++ b/tests/data/EnMAP_Level_1B_new/AlpineTest2_CWV2_SM0.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0d86d8cd3a75b66e33804128679db6b29a0678158498bafec17392a7dc2b9b4 +size 46055575 diff --git a/tests/test_l1b_reader.py b/tests/test_l1b_reader.py index efb6b42..4cd8dd0 100644 --- a/tests/test_l1b_reader.py +++ b/tests/test_l1b_reader.py @@ -12,7 +12,7 @@ import unittest from glob import glob import os from os import path -import sys +# import sys import tempfile import zipfile # from datetime import datetime @@ -25,19 +25,18 @@ class Test_L1B_Reader(unittest.TestCase): """Tests for L1B_Reader class..""" def setUp(self): - self.pathList_testimages = glob(os.path.join(os.path.dirname(__file__), "data", "EnMAP_Level_1B", "*.zip")) + self.pathList_testimages = glob(os.path.join(os.path.dirname(__file__), "data", "EnMAP_Level_1B_new", "*.zip")) self.config = EnPTConfig(**config_for_testing) self.tmpdir = tempfile.mkdtemp(dir=self.config.working_dir) def tearDown(self): shutil.rmtree(self.tmpdir) - # shutil.rmtree(self.config.output_dir) + shutil.rmtree(self.config.output_dir) def test_read_and_save_inputdata(self): from enpt.io.reader import L1B_Reader from enpt.model.images import EnMAPL1Product_SensorGeo - print("") print("################################################") print("# #") @@ -48,26 +47,44 @@ class Test_L1B_Reader(unittest.TestCase): print("") print("================================================") - print("Create the L1B_Reader new instance") + print("Unzip Test data files") print("================================================") print("") + for l1b_file in self.pathList_testimages: + with zipfile.ZipFile(l1b_file, "r") as zf: + zf.extractall(self.tmpdir) + prods = [os.path.join(self.tmpdir, os.path.basename(self.pathList_testimages[0]).split(".zip")[0]), + os.path.join(self.tmpdir, os.path.basename(self.pathList_testimages[1]).split(".zip")[0])] + print("Done!") print("") - rd = L1B_Reader(config=self.config) + print("") + - prods = (config_for_testing['path_l1b_enmap_image'], - config_for_testing['path_l1b_enmap_image_ext'], - config_for_testing['path_l1b_enmap_image_ext2']) + print("================================================") + print("Create the L1B_Reader new instance") + print("================================================") + print("") + rd = L1B_Reader(config=self.config) + print("Done!") + print("") + print("") # TEST FOR ONE IMAGE ONLY print("=======================================================================================================") - print("Test: Read ONE image only") + print("Test: Read and write ONE image only") print("=======================================================================================================") print("") - for prod in prods: + # for l1b_file in self.pathList_testimages: + # with zipfile.ZipFile(l1b_file, "r") as zf: + # + # zf.extractall(self.tmpdir) + # + # prod = os.path.join(self.tmpdir, os.path.basename(l1b_file).split(".zip")[0]) print("-------------------------------------") print("Test with %s" % os.path.basename(prod)) print("-------------------------------------") + print("Tmp dir: %s" % self.tmpdir) print("") print(" * Without SNR ") print("") @@ -75,9 +92,12 @@ class Test_L1B_Reader(unittest.TestCase): self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) self.assertIsNone(L1_obj.vnir.detector_meta.snr) self.assertIsNone(L1_obj.swir.detector_meta.snr) - # root_dir_written_L1_data = L1_obj.save(path.join(self.tmpdir, "no_snr")) - # L1_obj = rd.read_inputdata(root_dir_written_L1_data, compute_snr=False) - # self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) + root_dir_written_L1_data = L1_obj.save(path.join(self.tmpdir, "no_snr")) + + # read self written L1 data + L1_obj = rd.read_inputdata(root_dir_written_L1_data, compute_snr=False) + self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) + print("") print(" * With SNR ") print("") @@ -85,93 +105,54 @@ class Test_L1B_Reader(unittest.TestCase): self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) self.assertIsNotNone(L1_obj.vnir.detector_meta.snr) self.assertIsNotNone(L1_obj.swir.detector_meta.snr) - # root_dir_written_L1_data = L1_obj.save(path.join(self.config.output_dir, "with_snr")) - # L1_obj = rd.read_inputdata(root_dir_written_L1_data, compute_snr=False) - # self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) - print("") - print("") + root_dir_written_L1_data = L1_obj.save(path.join(self.tmpdir, "with_snr")) + L1_obj = rd.read_inputdata(root_dir_written_L1_data) + self.assertIsNotNone(L1_obj, EnMAPL1Product_SensorGeo) + # TEST FOR ONE IMAGE ONLY print("=======================================================================================================") - print("Test: Write ONE image only") + print("Test: read, join and write 2 images if possible!") print("=======================================================================================================") print("") - print("-----------------------------------------------------------------------------------------------") - print("Test with %s" % os.path.basename(prods[0])) - print("-----------------------------------------------------------------------------------------------") - print("") - print(" * With SNR ") - print("") - L1_obj = rd.read_inputdata(prods[0]) - root_dir_written_L1_data = L1_obj.save(path.join(self.config.output_dir, "with_snr")) - L1_obj = rd.read_inputdata(root_dir_written_L1_data, compute_snr=False) - self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) - shutil.rmtree(path.join(self.config.output_dir, "with_snr")) - print("") - print("") - - # TEST FOR TWO IMAGES - print("=======================================================================================================") - print("Test: Read TWO images") - print("=======================================================================================================") - print("") - for prod1, prod2 in ((0, 1), - (1, 0), - (0, 2), - (2, 0), - (1, 2), - (2, 1)): - for n_lines in (10, 50, 100, 500, 1000, 1500): + for k_prod1, k_prod2 in ((0, 1), (1, 0)): + for n_lines in (-1, 10, 50, 80, 100, 150): + tempdir = tempfile.mkdtemp(dir=self.config.working_dir) + if n_lines is -1: + n_lines = "all" print("-----------------------------------------------------------------------------------------------") - print("Test with %s and %s, with: %s lines" % (os.path.basename(prods[prod1]), - os.path.basename(prods[prod2]), + print("Test with %s and %s, with: %s lines" % (os.path.basename(prods[k_prod1]), + os.path.basename(prods[k_prod2]), n_lines)) print("-----------------------------------------------------------------------------------------------") + + if n_lines is "all": + n_lines = None + print("") print(" * Without SNR") print("") - L1_obj = rd.read_inputdata(prods[prod1], prods[prod2], n_lines, compute_snr=False) + L1_obj = rd.read_inputdata(prods[k_prod1], prods[k_prod2], n_lines, compute_snr=False) self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) self.assertIsNone(L1_obj.vnir.detector_meta.snr) self.assertIsNone(L1_obj.swir.detector_meta.snr) - # root_dir_written_L1_data = L1_obj.save(path.join(self.tmpdir, "no_snr")) - # L1_obj = rd.read_inputdata(root_dir_written_L1_data, compute_snr=False) - # self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) + root_dir_written_L1_data = L1_obj.save(path.join(tempdir, "no_snr")) + L1_obj = rd.read_inputdata(root_dir_written_L1_data, compute_snr=False) + self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) + print("") - print(" * with SNR ") + print(" * With SNR") print("") - L1_obj = rd.read_inputdata(prods[prod1], prods[prod2], n_lines) + L1_obj = rd.read_inputdata(prods[k_prod1], prods[k_prod2], n_lines) self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) self.assertIsNotNone(L1_obj.vnir.detector_meta.snr) self.assertIsNotNone(L1_obj.swir.detector_meta.snr) - # root_dir_written_L1_data = \ - # L1_obj.save(path.join(self.config.output_dir, - # "with_snr_%s_%s_%s_lines" % (os.path.basename(prods[prod1]), - # os.path.basename(prods[prod2]), - # n_lines)) - # ) - # L1_obj = rd.read_inputdata(root_dir_written_L1_data, compute_snr=False) - # self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) + root_dir_written_L1_data = L1_obj.save(path.join(tempdir, "with_snr")) + L1_obj = rd.read_inputdata(root_dir_written_L1_data) + self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) print("") print("") - - print("=======================================================================================================") - print("Test: Write TWO images ") - print("=======================================================================================================") - print("") - print("-----------------------------------------------------------------------------------------------") - print("Test with %s and %s, with: %s lines" % (os.path.basename(prods[0]),os.path.basename(prods[1]), 500)) - print("-----------------------------------------------------------------------------------------------") - print("") - print(" * With SNR ") - print("") - L1_obj = rd.read_inputdata(prods[0], prods[1], 500) - root_dir_written_L1_data = L1_obj.save(path.join(self.config.output_dir, "with_snr")) - L1_obj = rd.read_inputdata(root_dir_written_L1_data, compute_snr=False) - self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) - shutil.rmtree(path.join(self.config.output_dir, "with_snr")) - print("") - print("") + shutil.rmtree(tempdir) return -- GitLab From 68877a2809fec5badeca60b615f1d37c19e6ea8b Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Tue, 5 Jun 2018 16:21:03 +0200 Subject: [PATCH 03/11] Merge branch 'enhancement/revise_reader' of /home/gfz-fe/scheffler/python/EnPT with conflicts. --- enpt/model/images.py | 2 +- tests/test_l1b_reader.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/enpt/model/images.py b/enpt/model/images.py index 83188bc..b34dc1a 100644 --- a/enpt/model/images.py +++ b/enpt/model/images.py @@ -5,7 +5,7 @@ import logging from types import SimpleNamespace import numpy as np from os import path, makedirs -from shutil import copyfile +# from shutil import copyfile from lxml import etree from glob import glob import utm diff --git a/tests/test_l1b_reader.py b/tests/test_l1b_reader.py index 4cd8dd0..aba96f0 100644 --- a/tests/test_l1b_reader.py +++ b/tests/test_l1b_reader.py @@ -154,7 +154,6 @@ class Test_L1B_Reader(unittest.TestCase): print("") shutil.rmtree(tempdir) - return -- GitLab From 3a12f8ea5c63900dc21648c10219034d31137dae Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Mon, 11 Jun 2018 12:34:31 +0200 Subject: [PATCH 04/11] Fixed test_controller and test_radiometric_transform. Fixed code style issues. Fixed test_enpt_install. --- enpt/execution/controller.py | 3 +- enpt/io/reader.py | 10 +++--- requirements.txt | 3 ++ setup.py | 3 +- tests/__init__.py | 6 ++-- tests/linting/pydocstyle.log | 54 ++++++++++++++++++++++++----- tests/test_l1b_reader.py | 1 - tests/test_radiometric_transform.py | 4 +-- 8 files changed, 62 insertions(+), 22 deletions(-) diff --git a/enpt/execution/controller.py b/enpt/execution/controller.py index c41d153..6ddd5f5 100644 --- a/enpt/execution/controller.py +++ b/enpt/execution/controller.py @@ -2,7 +2,6 @@ """EnPT process controller module.""" import os -from datetime import datetime import tempfile import zipfile import shutil @@ -66,7 +65,7 @@ class EnPT_Controller(object): # run the reader RD = L1B_Reader(config=self.cfg) - L1_obj = RD.read_inputdata(path_enmap_image, observation_time=datetime(2015, 12, 7, 10)) + L1_obj = RD.read_inputdata(path_enmap_image) return L1_obj diff --git a/enpt/io/reader.py b/enpt/io/reader.py index 49e1528..1765cb0 100644 --- a/enpt/io/reader.py +++ b/enpt/io/reader.py @@ -43,7 +43,7 @@ class L1B_Reader(object): root_dir_main, root_dir_ext: str=None, n_line_ext: int=None, - lon_lat_smpl: tuple=(15,15), + lon_lat_smpl: tuple=(15, 15), compute_snr: bool=True): # All information are read from data itself now # In case of multiple files, temporary files are created to store them. @@ -92,12 +92,11 @@ class L1B_Reader(object): l1b_main_obj.swir.detector_meta.calc_snr_from_radiance(rad_data=l1b_main_obj.swir.data, dir_snr_models=tmpDir) - # Return the l1b_main_obj return l1b_main_obj - # def read_inputdata_old(self, root_dir, observation_time: datetime, lon_lat_smpl: tuple=(15, 15), nsmile_coef: int=5, - # compute_snr: bool=True): + # def read_inputdata_old(self, root_dir, observation_time: datetime, lon_lat_smpl: tuple=(15, 15), + # nsmile_coef: int=5, compute_snr: bool=True): # # TODO move to init? --> This has been added in the init phase (will call the new read_inputdata method # """Read L1B, DEM and spatial reference data. # @@ -113,7 +112,8 @@ class L1B_Reader(object): # # # read metadata # L1_obj.meta = EnMAP_Metadata_L1B_SensorGeo(L1_obj.paths.metaxml, config=self.cfg, logger=L1_obj.logger) - # L1_obj.meta.read_metadata(observation_time=observation_time, lon_lat_smpl=lon_lat_smpl, nsmile_coef=nsmile_coef) + # L1_obj.meta.read_metadata(observation_time=observation_time, lon_lat_smpl=lon_lat_smpl, + # nsmile_coef=nsmile_coef) # # # read VNIR data # # call L1_obj.vnir.arr.setter which sets L1_obj.swir.arr to an instance of GeoArray class diff --git a/requirements.txt b/requirements.txt index 480788d..a033335 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,7 @@ cerberus jsmin matplotlib tqdm +utm +lxml +imageio git+https://gitext.gfz-potsdam.de/EnMAP/sicor.git \ No newline at end of file diff --git a/setup.py b/setup.py index 0385891..c9c9878 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,8 @@ with open("enpt/version.py") as version_file: exec(version_file.read(), version) requirements = [ # put package requirements here - 'numpy', 'scipy', 'geoarray>=0.7.15', 'spectral>=0.16', 'cerberus', 'jsmin', 'matplotlib', 'tqdm' + 'numpy', 'scipy', 'geoarray>=0.7.15', 'spectral>=0.16', 'cerberus', 'jsmin', 'matplotlib', 'tqdm', 'utm', 'lxml', + 'imageio' # 'sicor', # pip install git+https://gitext.gfz-potsdam.de/EnMAP/sicor.git ] diff --git a/tests/__init__.py b/tests/__init__.py index 9a0356e..7359418 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -6,7 +6,9 @@ # # config_for_testing = dict( # path_l1b_enmap_image=os.path.join(enptRepo_rootpath, 'tests', 'data', 'EnMAP_Level_1B_v2', 'Alpine1_CWV2_SM0'), -# path_l1b_enmap_image_ext=os.path.join(enptRepo_rootpath, 'tests', 'data', 'EnMAP_Level_1B_v2', 'Alpine2_CWV2_SM0'), -# path_l1b_enmap_image_ext2=os.path.join(enptRepo_rootpath, 'tests', 'data', 'EnMAP_Level_1B_v2', 'Alpine3_CWV2_SM0'), +# path_l1b_enmap_image_ext=os.path.join(enptRepo_rootpath, 'tests', 'data', 'EnMAP_Level_1B_v2', +# 'Alpine2_CWV2_SM0'), +# path_l1b_enmap_image_ext2=os.path.join(enptRepo_rootpath, 'tests', 'data', 'EnMAP_Level_1B_v2', +# 'Alpine3_CWV2_SM0'), # output_dir=os.path.join(enptRepo_rootpath, 'tests', 'data', 'test_outputs') # ) diff --git a/tests/linting/pydocstyle.log b/tests/linting/pydocstyle.log index 8272d63..3af9382 100644 --- a/tests/linting/pydocstyle.log +++ b/tests/linting/pydocstyle.log @@ -1,13 +1,51 @@ enpt/version.py:1 at module level: D100: Missing docstring in public module -enpt/model/images.py:521 in public method `run_AC`: +enpt/io/reader.py:42 in public method `read_inputdata`: + D205: 1 blank line required between summary line and description (found 0) +enpt/io/reader.py:42 in public method `read_inputdata`: + D400: First line should end with a period (not ']') +enpt/model/images.py:344 in public method `get_paths`: + D205: 1 blank line required between summary line and description (found 0) +enpt/model/images.py:344 in public method `get_paths`: + D400: First line should end with a period (not 'o') +enpt/model/images.py:390 in public method `generate_quicklook`: + D205: 1 blank line required between summary line and description (found 0) +enpt/model/images.py:390 in public method `generate_quicklook`: + D400: First line should end with a period (not 'e') +enpt/model/images.py:492 in public method `get_paths`: + D205: 1 blank line required between summary line and description (found 0) +enpt/model/images.py:492 in public method `get_paths`: + D400: First line should end with a period (not 'o') +enpt/model/images.py:616 in public method `run_AC`: D102: Missing docstring in public method -enpt/model/metadata.py:210 in public method `calc_solar_irradiance_CWL_FWHM_per_band`: +enpt/model/images.py:623 in public method `save`: + D205: 1 blank line required between summary line and description (found 0) +enpt/model/images.py:623 in public method `save`: + D400: First line should end with a period (not 't') +enpt/model/images.py:714 in public method `append_new_image`: + D205: 1 blank line required between summary line and description (found 0) +enpt/model/metadata.py:97 in public method `read_metadata`: + D205: 1 blank line required between summary line and description (found 0) +enpt/model/metadata.py:97 in public method `read_metadata`: + D400: First line should end with a period (not 'y') +enpt/model/metadata.py:290 in public method `interpolate_corners`: + D205: 1 blank line required between summary line and description (found 0) +enpt/model/metadata.py:306 in public method `calc_solar_irradiance_CWL_FWHM_per_band`: D102: Missing docstring in public method -enpt/model/metadata.py:268 in public method `get_earth_sun_distance`: +enpt/model/metadata.py:364 in public method `read_common_meta`: D202: No blank lines allowed after function docstring (found 1) -enpt/model/metadata.py:268 in public method `get_earth_sun_distance`: +enpt/model/metadata.py:364 in public method `read_common_meta`: + D205: 1 blank line required between summary line and description (found 0) +enpt/model/metadata.py:364 in public method `read_common_meta`: + D400: First line should end with a period (not 'o') +enpt/model/metadata.py:397 in public method `get_earth_sun_distance`: D400: First line should end with a period (not ')') +enpt/model/metadata.py:419 in public method `read_metadata`: + D202: No blank lines allowed after function docstring (found 1) +enpt/model/metadata.py:419 in public method `read_metadata`: + D205: 1 blank line required between summary line and description (found 0) +enpt/model/metadata.py:419 in public method `read_metadata`: + D400: First line should end with a period (not 'y') enpt/model/srf.py:11 in public class `SRF`: D101: Missing docstring in public class enpt/model/srf.py:96 in public method `instrument`: @@ -36,13 +74,13 @@ enpt/processors/atmospheric_correction/atmospheric_correction.py:25 in public me D102: Missing docstring in public method enpt/processors/atmospheric_correction/atmospheric_correction.py:47 in public method `run_ac`: D102: Missing docstring in public method -enpt/execution/controller.py:49 in public method `read_L1B_data`: +enpt/execution/controller.py:48 in public method `read_L1B_data`: D205: 1 blank line required between summary line and description (found 0) -enpt/execution/controller.py:49 in public method `read_L1B_data`: +enpt/execution/controller.py:48 in public method `read_L1B_data`: D400: First line should end with a period (not ':') -enpt/execution/controller.py:82 in public method `run_geometry_processor`: +enpt/execution/controller.py:81 in public method `run_geometry_processor`: D102: Missing docstring in public method -enpt/execution/controller.py:89 in public method `write_output`: +enpt/execution/controller.py:88 in public method `write_output`: D102: Missing docstring in public method enpt/options/config.py:44 in public class `EnPTConfig`: D101: Missing docstring in public class diff --git a/tests/test_l1b_reader.py b/tests/test_l1b_reader.py index aba96f0..60efc2e 100644 --- a/tests/test_l1b_reader.py +++ b/tests/test_l1b_reader.py @@ -59,7 +59,6 @@ class Test_L1B_Reader(unittest.TestCase): print("") print("") - print("================================================") print("Create the L1B_Reader new instance") print("================================================") diff --git a/tests/test_radiometric_transform.py b/tests/test_radiometric_transform.py index a36cfe8..c344176 100644 --- a/tests/test_radiometric_transform.py +++ b/tests/test_radiometric_transform.py @@ -6,7 +6,6 @@ from unittest import TestCase, main from glob import glob import tempfile import zipfile -from datetime import datetime from enpt.processors.radiometric_transform import Radiometric_Transformer from enpt.options.config import EnPTConfig, config_for_testing @@ -33,8 +32,7 @@ class Test_Radiometric_Transformer(TestCase): root_dir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) # create EnPT Level 1 image - L1_obj = L1B_Reader(config=self.cfg)\ - .read_inputdata(root_dir, observation_time=datetime(2015, 12, 7, 10)) + L1_obj = L1B_Reader(config=self.cfg).read_inputdata(root_dir) # input assertions self.assertIsInstance(L1_obj, EnMAPL1Product_SensorGeo) -- GitLab From e4b913f36e1e4513fa632d19cfdeb69b8b441359 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Mon, 11 Jun 2018 16:59:52 +0200 Subject: [PATCH 05/11] Cleaned up a lot of deprecated code. Implemented automatic appending of gap fill image into controller module. --- enpt/execution/controller.py | 26 ++-- enpt/io/reader.py | 60 +------- enpt/model/images.py | 130 ++---------------- enpt/model/metadata.py | 116 ++-------------- enpt/options/config.py | 4 +- enpt/utils/path_generator.py | 5 + tests/__init__.py | 13 -- .../EnMAP_Level_1B/AlpineTest1_CWV2_SM0.zip | 3 - .../EnMAP_Level_1B/AlpineTest1_CWV2_SM1.zip | 3 - tests/data/Landsat8__500x500x2.bsq | 3 - tests/data/Landsat8__500x500x2.hdr | 16 --- tests/linting/pydocstyle.log | 88 ++++++------ tests/test_l1b_reader.py | 6 +- tests/test_radiometric_transform.py | 4 +- 14 files changed, 104 insertions(+), 373 deletions(-) delete mode 100644 tests/data/EnMAP_Level_1B/AlpineTest1_CWV2_SM0.zip delete mode 100644 tests/data/EnMAP_Level_1B/AlpineTest1_CWV2_SM1.zip delete mode 100644 tests/data/Landsat8__500x500x2.bsq delete mode 100644 tests/data/Landsat8__500x500x2.hdr diff --git a/enpt/execution/controller.py b/enpt/execution/controller.py index 6ddd5f5..10fd22f 100644 --- a/enpt/execution/controller.py +++ b/enpt/execution/controller.py @@ -32,7 +32,7 @@ class EnPT_Controller(object): warn_message="Implicitly cleaning up {!r}".format(self)) # read the L1B data - self.L1_obj = self.read_L1B_data(self.cfg.path_l1b_enmap_image) + self.L1_obj = self.read_L1B_data() def extract_zip_archive(self, path_zipfile: str) -> str: """Extract the given EnMAP image zip archive and return the L1B image root directory path. @@ -43,14 +43,18 @@ class EnPT_Controller(object): with zipfile.ZipFile(path_zipfile, "r") as zf: zf.extractall(self.tempDir) - return os.path.join(self.tempDir, os.path.basename(path_zipfile).split('.zip')[0]) + outdir = os.path.join(self.tempDir, os.path.basename(path_zipfile).split('.zip')[0]) - def read_L1B_data(self, path_enmap_image: str) -> EnMAPL1Product_SensorGeo: - """ + if not os.path.isdir(outdir): + raise NotADirectoryError(outdir) + + return outdir + + def read_L1B_data(self) -> EnMAPL1Product_SensorGeo: + """Read the provider L1B data given in config and return an EnMAP image object.""" + path_enmap_image = self.cfg.path_l1b_enmap_image + path_enmap_image_gapfill = self.cfg.path_l1b_enmap_image_gapfill - :param path_enmap_image: - :return: - """ # input validation if not os.path.isdir(path_enmap_image) and \ not (os.path.exists(path_enmap_image) and path_enmap_image.endswith('.zip')): @@ -60,12 +64,14 @@ class EnPT_Controller(object): # extract L1B image archive if needed if path_enmap_image.endswith('.zip'): path_enmap_image = self.extract_zip_archive(path_enmap_image) - if not os.path.isdir(path_enmap_image): - raise NotADirectoryError(path_enmap_image) + + # extract L1B gap fill image archive if needed + if path_enmap_image_gapfill and path_enmap_image_gapfill.endswith('.zip'): + path_enmap_image_gapfill = self.extract_zip_archive(path_enmap_image_gapfill) # run the reader RD = L1B_Reader(config=self.cfg) - L1_obj = RD.read_inputdata(path_enmap_image) + L1_obj = RD.read_inputdata(root_dir_main=path_enmap_image, root_dir_ext=path_enmap_image_gapfill) return L1_obj diff --git a/enpt/io/reader.py b/enpt/io/reader.py index 1765cb0..f2ce153 100644 --- a/enpt/io/reader.py +++ b/enpt/io/reader.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """Reader module for reading all kinds of EnMAP images.""" -# from datetime import datetime import logging import tempfile import zipfile @@ -9,7 +8,6 @@ import numpy as np from scipy.interpolate import interp1d from ..model.images import EnMAPL1Product_SensorGeo -# from ..model.metadata import EnMAP_Metadata_L1B_SensorGeo from ..options.config import EnPTConfig @@ -62,20 +60,22 @@ class L1B_Reader(object): l1b_main_obj = EnMAPL1Product_SensorGeo(root_dir_main, config=self.cfg, logger=self.logger, lon_lat_smpl=lon_lat_smpl) - # load data from the main object + # associate raster attributes with file links (raster data is read lazily / on demand) l1b_main_obj.vnir.data = l1b_main_obj.paths.vnir.data l1b_main_obj.vnir.mask_clouds = l1b_main_obj.paths.vnir.mask_clouds l1b_main_obj.vnir.deadpixelmap = l1b_main_obj.paths.vnir.deadpixelmap l1b_main_obj.swir.data = l1b_main_obj.paths.swir.data l1b_main_obj.swir.mask_clouds = l1b_main_obj.paths.swir.mask_clouds l1b_main_obj.swir.deadpixelmap = l1b_main_obj.paths.swir.deadpixelmap + + # compute radiance l1b_main_obj.DN2TOARadiance() # in case of a second file, we create new files that will be temporary save into a temporary directory - # and their path will be stored into a the paths of l1b_main_obj + # and their path will be stored into the paths of l1b_main_obj # NOTE: We do the following hypothesis: # - The dead pixel map will not change when acquiring 2 adjacent images. - if root_dir_ext is not None: + if root_dir_ext: l1b_ext_obj = EnMAPL1Product_SensorGeo(root_dir_ext, config=self.cfg, lon_lat_smpl=lon_lat_smpl) l1b_main_obj.append_new_image(l1b_ext_obj, n_line_ext) @@ -95,56 +95,6 @@ class L1B_Reader(object): # Return the l1b_main_obj return l1b_main_obj - # def read_inputdata_old(self, root_dir, observation_time: datetime, lon_lat_smpl: tuple=(15, 15), - # nsmile_coef: int=5, compute_snr: bool=True): - # # TODO move to init? --> This has been added in the init phase (will call the new read_inputdata method - # """Read L1B, DEM and spatial reference data. - # - # :param root_dir: Root directory of EnMAP Level-1B product - # :param observation_time: datetime of observation time (currently missing in metadata) - # :param lon_lat_smpl: number if sampling points in lon, lat fields - # :param nsmile_coef: number of polynomial coefficients for smile - # :param compute_snr: whether to compute SNR or not (default: True) - # :return: instance of EnMAPL1Product_MapGeo - # """ - # # get a new instance of EnMAPL1Product_MapGeo - # L1_obj = EnMAPL1Product_SensorGeo(root_dir, config=self.cfg) - # - # # read metadata - # L1_obj.meta = EnMAP_Metadata_L1B_SensorGeo(L1_obj.paths.metaxml, config=self.cfg, logger=L1_obj.logger) - # L1_obj.meta.read_metadata(observation_time=observation_time, lon_lat_smpl=lon_lat_smpl, - # nsmile_coef=nsmile_coef) - # - # # read VNIR data - # # call L1_obj.vnir.arr.setter which sets L1_obj.swir.arr to an instance of GeoArray class - # L1_obj.vnir.data = L1_obj.paths.vnir.data - # L1_obj.vnir.mask_clouds = L1_obj.paths.vnir.mask_clouds - # L1_obj.vnir.deadpixelmap = L1_obj.paths.vnir.deadpixelmap - # L1_obj.vnir.detector_meta = L1_obj.meta.vnir - # - # # read SWIR data - # # call L1_obj.swir.arr.setter which sets L1_obj.swir.arr to an instance of GeoArray class - # L1_obj.swir.data = L1_obj.paths.swir.data - # L1_obj.swir.mask_clouds = L1_obj.paths.swir.mask_clouds - # L1_obj.swir.deadpixelmap = L1_obj.paths.swir.deadpixelmap - # L1_obj.swir.detector_meta = L1_obj.meta.swir - # - # # compute radiance - # L1_obj.DN2TOARadiance() - # - # # compute SNR - # if compute_snr: - # with tempfile.TemporaryDirectory(dir=self.cfg.working_dir) as tmpDir,\ - # zipfile.ZipFile(self.cfg.path_l1b_snr_model, "r") as zf: - # zf.extractall(tmpDir) - # - # if L1_obj.meta.vnir.unitcode == 'TOARad': - # L1_obj.vnir.detector_meta.calc_snr_from_radiance(rad_data=L1_obj.vnir.data, dir_snr_models=tmpDir) - # if L1_obj.meta.swir.unitcode == 'TOARad': - # L1_obj.swir.detector_meta.calc_snr_from_radiance(rad_data=L1_obj.swir.data, dir_snr_models=tmpDir) - # - # return L1_obj - def validate_input(self): """Validate user inputs.""" pass diff --git a/enpt/model/images.py b/enpt/model/images.py index b34dc1a..fb117f0 100644 --- a/enpt/model/images.py +++ b/enpt/model/images.py @@ -5,12 +5,10 @@ import logging from types import SimpleNamespace import numpy as np from os import path, makedirs -# from shutil import copyfile from lxml import etree from glob import glob import utm from scipy.interpolate import interp2d -# from datetime import datetime # Use to generate preview import imageio @@ -18,17 +16,15 @@ from skimage import exposure from geoarray import GeoArray, NoDataMask, CloudMask -# Not used (thank to pycharm) -# from enpt.options.config import EnPTConfig -# from ..utils.path_generator import PathGenL1BProduct from ..utils.logging import EnPT_Logger from ..model.metadata import EnMAP_Metadata_L1B_SensorGeo, EnMAP_Metadata_L1B_Detector_SensorGeo from ..options.config import EnPTConfig from ..processors.dead_pixel_correction import Dead_Pixel_Corrector -############## -# BASE CLASSES -############## + +################ +# BASE CLASSES # +################ class _EnMAP_Image(object): @@ -441,12 +437,7 @@ class EnMAPL1Product_SensorGeo(object): if logger: self.logger = logger - # Define the path. get_paths will be removed (see comments below) - # self.paths = self.get_paths() return glob(os.path.join(self.root_dir, "*_header.xml"))[0] - # self.paths = SimpleNamespace() - # Read metadata here in order to get all information needed by to get paths. - # self.paths.metaxml = glob(path.join(root_dir, "*_header.xml"))[0] self.meta = EnMAP_Metadata_L1B_SensorGeo(glob(path.join(root_dir, "*_header.xml"))[0], config=self.cfg, logger=self.logger) self.meta.read_metadata(lon_lat_smpl) @@ -457,38 +448,9 @@ class EnMAPL1Product_SensorGeo(object): # Get the paths according information delivered in the metadata self.paths = self.get_paths(root_dir) - # self.paths.root_dir = root_dir - # - # # Get paths for vnir - # self.paths.vnir = SimpleNamespace() - # self.paths.vnir.root_dir = root_dir - # self.paths.vnir.metaxml = glob(path.join(root_dir, "*_header.xml"))[0] - # self.paths.vnir.data = path.join(root_dir, self.meta.vnir.data_filename) - # self.paths.vnir.mask_clouds = path.join(root_dir, self.meta.vnir.cloud_mask_filename) - # self.paths.vnir.deadpixelmap = path.join(root_dir, self.meta.vnir.dead_pixel_filename) - # self.paths.vnir.quicklook = path.join(root_dir, self.meta.vnir.quicklook_filename) - # - # # Get paths for swir - # self.paths.swir = SimpleNamespace() - # self.paths.swir.root_dir = root_dir - # self.paths.swir.metaxml = glob(path.join(root_dir, "*_header.xml"))[0] - # self.paths.swir.data = path.join(root_dir, self.meta.swir.data_filename) - # self.paths.swir.mask_clouds = path.join(root_dir, self.meta.swir.cloud_mask_filename) - # self.paths.swir.deadpixelmap = path.join(root_dir, self.meta.swir.dead_pixel_filename) - # self.paths.swir.quicklook = path.join(root_dir, self.meta.swir.quicklook_filename) self.detector_attrNames = ['vnir', 'swir'] - # Create a temporary tmp directory to store some data if needed - # self.tmpdir = tempfile.mkdtemp(dir=self.cfg.working_dir) - - # def __del__(self): - # """ - # Remove the (un)used temporary directory - # :return: None - # """ - # print("shutil.rmtree(self.tmpdir)") - def get_paths(self, root_dir): """ Get all file paths associated with the current instance of EnMAPL1Product_SensorGeo @@ -554,19 +516,6 @@ class EnMAPL1Product_SensorGeo(object): assert isinstance(string, str), "'log' can only be set to a string. Got %s." % type(string) self.logger.captured_stream = string - # def get_paths(self): - # """Get all file paths associated with the current instance of EnMAPL1Product_SensorGeo. - # - # :return: types.SimpleNamespace() - # """ - # paths = SimpleNamespace() - # paths.vnir = self.vnir.get_paths() - # paths.swir = self.swir.get_paths() - # paths.root_dir = paths.vnir.root_dir - # paths.metaxml = paths.vnir.metaxml - # - # return paths - # @classmethod # def from_L1B_provider_data(cls, path_enmap_image: str, config: EnPTConfig=None) -> EnMAPL1Product_SensorGeo: # """ @@ -657,60 +606,11 @@ class EnMAPL1Product_SensorGeo(object): xml.findall("ProductComponent/SWIRDetector/Data/Type/UnitCode")[0].text = self.meta.swir.unitcode xml.findall("ProductComponent/SWIRDetector/Data/Type/Unit")[0].text = self.meta.swir.unit xml_string = etree.tostring(xml, pretty_print=True, xml_declaration=True, encoding='UTF-8') - xml_file = open(product_dir + path.sep + path.basename(self.paths.metaxml), "w") - xml_file.write(xml_string.decode("utf-8")) - xml_file.close() + with open(product_dir + path.sep + path.basename(self.paths.metaxml), "w") as xml_file: + xml_file.write(xml_string.decode("utf-8")) return product_dir - # def save_old(self, outdir: str, suffix="") -> str: - # """Save this product to disk using almost the same format as for reading. - # - # :param outdir: Path to output directory - # :param suffix: Suffix to be appended to the output filename - # :return: Root path of written product - # """ - # product_dir = path.join( - # path.abspath(outdir), "{name}{suffix}".format( - # name=[ff for ff in self.paths.root_dir.split(path.sep) if ff != ''][-1], - # suffix=suffix) - # ) - # self.logger.info("Write product to: %s" % product_dir) - # makedirs(product_dir, exist_ok=True) - # - # for detector_name in self.detector_attrNames: - # detector = getattr(self, detector_name) - # detector_paths = getattr(self.paths, detector_name) - # - # for atts, fmt in ((("deadpixelmap", "mask_clouds"), "GTIff"), - # (("data",), "ENVI")): - # for att in atts: - # getattr(detector, att).save( - # path.join(product_dir, path.basename(getattr(detector_paths, att))), fmt=fmt) - # - # copyfile( - # src=detector_paths.quicklook, - # dst=path.join(product_dir, path.basename(detector_paths.quicklook)) - # ) - # - # xml = ElementTree.parse(self.paths.metaxml) - # for xml_name, real_name in (("detector1", "vnir"), ("detector2", "swir")): - # ele = xml.getroot().find(xml_name) - # - # # add unitcode - # new_ele = ElementTree.Element("unitcode") - # new_ele.text = getattr(self.meta, real_name).unitcode - # ele.append(new_ele) - # - # # add unit - # new_ele = ElementTree.Element("unit") - # new_ele.text = getattr(self.meta, real_name).unit - # ele.append(new_ele) - # - # xml.write(path.join(product_dir, path.basename(self.paths.metaxml))) - # - # return product_dir - def append_new_image(self, img2, n_lines: int=None): """ Check if a second image could pass with the first image. @@ -768,7 +668,7 @@ class EnMAPL1Product_SensorGeo(object): self.logger.warning("Set to the image number of line") n_lines = img2.meta.vnir.nrows - if n_lines < 50: # TODO: determine this values + if n_lines < 50: # TODO: determine these values self.logger.warning("A minimum of 50 lines is required, only %s were selected" % n_lines) self.logger.warning("Set the number of line to 50") n_lines = 50 @@ -779,31 +679,31 @@ class EnMAPL1Product_SensorGeo(object): ff = interp2d(x=[0, 1], y=[0, 1], z=[[img2.meta.vnir.lat_UL_UR_LL_LR[0], img2.meta.vnir.lat_UL_UR_LL_LR[1]], [img2.meta.vnir.lat_UL_UR_LL_LR[2], img2.meta.vnir.lat_UL_UR_LL_LR[3]]], kind='linear') - self.meta.vnir.lat_UL_UR_LL_LR[2] = ff(0, n_lines/img2.meta.vnir.nrows)[0] - self.meta.vnir.lat_UL_UR_LL_LR[3] = ff(1, n_lines/img2.meta.vnir.nrows)[0] + self.meta.vnir.lat_UL_UR_LL_LR[2] = np.array(ff(0, n_lines/img2.meta.vnir.nrows))[0] + self.meta.vnir.lat_UL_UR_LL_LR[3] = np.array(ff(1, n_lines/img2.meta.vnir.nrows))[0] lon_lat_smpl = (15, 15) self.meta.vnir.lats = self.meta.vnir.interpolate_corners(*self.meta.vnir.lat_UL_UR_LL_LR, *lon_lat_smpl) ff = interp2d(x=[0, 1], y=[0, 1], z=[[img2.meta.vnir.lon_UL_UR_LL_LR[0], img2.meta.vnir.lon_UL_UR_LL_LR[1]], [img2.meta.vnir.lon_UL_UR_LL_LR[2], img2.meta.vnir.lon_UL_UR_LL_LR[3]]], kind='linear') - self.meta.vnir.lon_UL_UR_LL_LR[2] = ff(0, n_lines/img2.meta.vnir.nrows)[0] - self.meta.vnir.lon_UL_UR_LL_LR[3] = ff(1, n_lines/img2.meta.vnir.nrows)[0] + self.meta.vnir.lon_UL_UR_LL_LR[2] = np.array(ff(0, n_lines/img2.meta.vnir.nrows))[0] + self.meta.vnir.lon_UL_UR_LL_LR[3] = np.array(ff(1, n_lines/img2.meta.vnir.nrows))[0] self.meta.vnir.lons = self.meta.vnir.interpolate_corners(*self.meta.vnir.lon_UL_UR_LL_LR, *lon_lat_smpl) # Create new corner coordinate - SWIR ff = interp2d(x=[0, 1], y=[0, 1], z=[[img2.meta.swir.lat_UL_UR_LL_LR[0], img2.meta.swir.lat_UL_UR_LL_LR[1]], [img2.meta.swir.lat_UL_UR_LL_LR[2], img2.meta.swir.lat_UL_UR_LL_LR[3]]], kind='linear') - self.meta.swir.lat_UL_UR_LL_LR[2] = ff(0, n_lines/img2.meta.swir.nrows)[0] - self.meta.swir.lat_UL_UR_LL_LR[3] = ff(1, n_lines/img2.meta.swir.nrows)[0] + self.meta.swir.lat_UL_UR_LL_LR[2] = np.array(ff(0, n_lines/img2.meta.swir.nrows))[0] + self.meta.swir.lat_UL_UR_LL_LR[3] = np.array(ff(1, n_lines/img2.meta.swir.nrows))[0] lon_lat_smpl = (15, 15) self.meta.swir.lats = self.meta.swir.interpolate_corners(*self.meta.swir.lat_UL_UR_LL_LR, *lon_lat_smpl) ff = interp2d(x=[0, 1], y=[0, 1], z=[[img2.meta.vnir.lon_UL_UR_LL_LR[0], img2.meta.vnir.lon_UL_UR_LL_LR[1]], [img2.meta.vnir.lon_UL_UR_LL_LR[2], img2.meta.vnir.lon_UL_UR_LL_LR[3]]], kind='linear') - self.meta.swir.lon_UL_UR_LL_LR[2] = ff(0, n_lines/img2.meta.swir.nrows)[0] - self.meta.swir.lon_UL_UR_LL_LR[3] = ff(1, n_lines/img2.meta.swir.nrows)[0] + self.meta.swir.lon_UL_UR_LL_LR[2] = np.array(ff(0, n_lines/img2.meta.swir.nrows))[0] + self.meta.swir.lon_UL_UR_LL_LR[3] = np.array(ff(1, n_lines/img2.meta.swir.nrows))[0] self.meta.swir.lons = self.meta.swir.interpolate_corners(*self.meta.swir.lon_UL_UR_LL_LR, *lon_lat_smpl) # append the vnir/swir image diff --git a/enpt/model/metadata.py b/enpt/model/metadata.py index 6592393..3ef2622 100644 --- a/enpt/model/metadata.py +++ b/enpt/model/metadata.py @@ -15,16 +15,7 @@ from ..options.config import EnPTConfig from .srf import SRF -# L1B_product_props = dict( -# xml_detector_label=dict( -# VNIR='detector1', -# SWIR='detector2'), -# fn_detector_suffix=dict( -# VNIR='D1', -# SWIR='D2') -# ) - -# Define a new L1B_product_props +# Define L1B_product_props L1B_product_props = dict( xml_detector_label=dict( VNIR='VNIR', @@ -79,11 +70,6 @@ class EnMAP_Metadata_L1B_Detector_SensorGeo(object): self.smile = None # type: np.ndarray # smile for each EnMAP image column self.l_min = None # type: np.ndarray self.l_max = None # type: np.ndarray - self.geom_view_zenith = None # type: float # viewing zenith angle - self.geom_view_azimuth = None # type: float # viewing azimuth angle - self.geom_sun_zenith = None # type: float # sun zenith angle - self.geom_sun_azimuth = None # type: float # sun azimuth angle - self.mu_sun = None # type: float # needed by SICOR for TOARad > TOARef conversion self.lat_UL_UR_LL_LR = None # type: list # latitude coordinates for UL, UR, LL, LR self.lon_UL_UR_LL_LR = None # type: list # longitude coordinates for UL, UR, LL, LR self.lats = None # type: np.ndarray # 2D array of latitude coordinates according to given lon/lat sampling @@ -120,10 +106,6 @@ class EnMAP_Metadata_L1B_Detector_SensorGeo(object): # read some basic information concerning the detector self.nrows = np.int(xml.findall("ProductComponent/%s/Data/Size/NRows" % lbl)[0].text) self.ncols = np.int(xml.findall("ProductComponent/%s/Data/Size/NCols" % lbl)[0].text) - self.geom_view_zenith = np.float(xml.findall("GeneralInfo/Geometry/Observation/ZenithAngle")[0].text) - self.geom_view_azimuth = np.float(xml.findall("GeneralInfo/Geometry/Observation/AzimuthAngle")[0].text) - self.geom_sun_zenith = np.float(xml.findall("GeneralInfo/Geometry/Illumination/ZenithAngle")[0].text) - self.geom_sun_azimuth = np.float(xml.findall("GeneralInfo/Geometry/Illumination/AzimuthAngle")[0].text) self.unitcode = xml.findall("ProductComponent/%s/Data/Type/UnitCode" % lbl)[0].text self.unit = xml.findall("ProductComponent/%s/Data/Type/Unit" % lbl)[0].text @@ -167,60 +149,6 @@ class EnMAP_Metadata_L1B_Detector_SensorGeo(object): self.lats = self.interpolate_corners(*self.lat_UL_UR_LL_LR, *lon_lat_smpl) self.lons = self.interpolate_corners(*self.lon_UL_UR_LL_LR, *lon_lat_smpl) - # def read_metadata_old(self, path_xml, lon_lat_smpl, nsmile_coef): - # """Read the metadata of a specific EnMAP detector in sensor geometry. - # - # :param path_xml: file path of the metadata XML file - # :param lon_lat_smpl: number if sampling points in lon, lat fields - # :param nsmile_coef: number of polynomial coefficients for smile - # """ - # xml = ElementTree.parse(path_xml).getroot() - # lbl = self.detector_label - # self.logger.info("Reading metadata for %s detector..." % self.detector_name) - - # self.fwhm = np.array(xml.findall("%s/fwhm" % lbl)[0].text.replace("\n", "").split(), dtype=np.float) - # self.wvl_center = np.array( - # xml.findall("%s/centre_wavelength" % lbl)[0].text.replace("\n", "").split(), dtype=np.float) - # self.srf = SRF.from_cwl_fwhm(self.wvl_center, self.fwhm) - # self.solar_irrad = self.calc_solar_irradiance_CWL_FWHM_per_band() - # self.nwvl = len(self.wvl_center) - # self.nrows = np.int(xml.findall("%s/rows" % lbl)[0].text) - # self.ncols = np.int(xml.findall("%s/columns" % lbl)[0].text) - # self.smile_coef = np.array(xml.findall("%s/smile" % lbl)[0].text.replace("\n", "").split(), dtype=np.float) \ - # .reshape((-1, nsmile_coef + 1))[:, 1:] - # self.nsmile_coef = nsmile_coef - # self.smile = self.calc_smile() - # self.l_min = np.array(xml.findall("%s/L_min" % lbl)[0].text.split(), dtype=np.float) - # self.l_max = np.array(xml.findall("%s/L_max" % lbl)[0].text.split(), dtype=np.float) - # self.geom_view_zenith = np.float( - # xml.findall("%s/observation_geometry/zenith_angle" % lbl)[0].text.split()[0]) - # self.geom_view_azimuth = np.float( - # xml.findall("%s/observation_geometry/azimuth_angle" % lbl)[0].text.split()[0]) - # self.geom_sun_zenith = np.float( - # xml.findall("%s/illumination_geometry/zenith_angle" % lbl)[0].text.split()[0]) - # self.geom_sun_azimuth = np.float( - # xml.findall("%s/illumination_geometry/azimuth_angle" % lbl)[0].text.split()[0]) - # self.lat_UL_UR_LL_LR = \ - # [float(xml.findall("%s/geometry/bounding_box/%s_northing" % (lbl, corner))[0].text.split()[0]) - # for corner in ("UL", "UR", "LL", "LR")] - # self.lon_UL_UR_LL_LR = \ - # [float(xml.findall("%s/geometry/bounding_box/%s_easting" % (lbl, corner))[0].text.split()[0]) - # for corner in ("UL", "UR", "LL", "LR")] - # self.lats = self.interpolate_corners(*self.lat_UL_UR_LL_LR, *lon_lat_smpl) - # self.lons = self.interpolate_corners(*self.lon_UL_UR_LL_LR, *lon_lat_smpl) - # - # try: - # self.unitcode = xml.findall("%s/unitcode" % lbl)[0].text - # # '" ".join(xml.findall("%s/radiance_unit" % lbl)[0].text.split()) - # self.unit = xml.findall("%s/unit" % lbl)[0].text - # except IndexError: - # self.unitcode = 'DN' - # self.unit = 'none' - # except Exception: - # raise - # - # self.snr = None - def calc_smile(self): """Compute smile for each EnMAP column. @@ -330,13 +258,13 @@ class EnMAP_Metadata_L1B_SensorGeo(object): Attributes: - logger(logging.Logger): None or logging instance - observation_datetime(datetime.datetime): datetime of observation time (currently missing in metadata) - - geom_view_zenith: tbd - - geom_view_azimuth: tbd - - geom_sun_zenith: tbd - - geom_sun_azimuth: tbd + - geom_view_zenith: viewing zenith angle + - geom_view_azimuth: viewing azimuth angle + - geom_sun_zenith: sun zenith angle + - geom_sun_azimuth: sun azimuth angle + - mu_sun: needed by SICOR for TOARad > TOARef conversion - vnir(EnMAP_Metadata_VNIR_SensorGeo) - swir(EnMAP_Metadata_SWIR_SensorGeo) - """ def __init__(self, path_metaxml, config: EnPTConfig, logger=None): @@ -351,10 +279,11 @@ class EnMAP_Metadata_L1B_SensorGeo(object): # defaults - Common self.observation_datetime = None # type: datetime # Date and Time of image observation - self.geom_view_zenith = None # type: float # tbd - self.geom_view_azimuth = None # type: float # tbd - self.geom_sun_zenith = None # type: float # tbd - self.geom_sun_azimuth = None # type: float # tbd + self.geom_view_zenith = None # type: float # viewing zenith angle + self.geom_view_azimuth = None # type: float # viewing azimuth angle + self.geom_sun_zenith = None # type: float # sun zenith angle + self.geom_sun_azimuth = None # type: float # sun azimuth angle + self.mu_sun = None # type: float # needed by SICOR for TOARad > TOARef conversion self.earthSunDist = None # type: float # earth-sun distance # TODO doc correct? self.vnir = None # type: EnMAP_Metadata_L1B_Detector_SensorGeo # metadata of VNIR only self.swir = None # type: EnMAP_Metadata_L1B_Detector_SensorGeo # metadata of SWIR only @@ -384,15 +313,7 @@ class EnMAP_Metadata_L1B_SensorGeo(object): self.geom_view_azimuth = np.float(xml.findall("GeneralInfo/Geometry/Observation/AzimuthAngle")[0].text) self.geom_sun_zenith = np.float(xml.findall("GeneralInfo/Geometry/Illumination/ZenithAngle")[0].text) self.geom_sun_azimuth = np.float(xml.findall("GeneralInfo/Geometry/Illumination/AzimuthAngle")[0].text) - - # def read_common_meta_old(self, observation_time: datetime=None): - # """Read the metadata belonging to both, the VNIR and SWIR detector of the EnMAP L1B product in sensor geometry. - # - # :param observation_time: date and time of image observation (datetime.datetime) - # """ - # # FIXME observation time is currently missing in the XML -- DONE - # self.observation_datetime = observation_time - # self.earthSunDist = self.get_earth_sun_distance(self.observation_datetime) + self.mu_sun = np.cos(np.deg2rad(self.geom_sun_zenith)) def get_earth_sun_distance(self, acqDate: datetime): """Get earth sun distance (requires file of pre calculated earth sun distance per day) @@ -433,16 +354,3 @@ class EnMAP_Metadata_L1B_SensorGeo(object): # define and read the SWIR metadata self.swir = EnMAP_Metadata_L1B_Detector_SensorGeo('SWIR', config=self.cfg, logger=self.logger) self.swir.read_metadata(self._path_xml, lon_lat_smpl) - - # def read_metadata_old(self, observation_time: datetime, lon_lat_smpl, nsmile_coef): - # """Read the metadata of the whole EnMAP L1B product in sensor geometry. - # - # :param observation_time: date and time of image observation (datetime.datetime) - # :param lon_lat_smpl: number if sampling points in lon, lat fields - # :param nsmile_coef: number of polynomial coefficients for smile - # """ - # self.read_common_meta(observation_time) - # self.vnir = EnMAP_Metadata_L1B_Detector_SensorGeo('VNIR', config=self.cfg, logger=self.logger) - # self.vnir.read_metadata(self._path_xml, lon_lat_smpl=lon_lat_smpl, nsmile_coef=nsmile_coef) - # self.swir = EnMAP_Metadata_L1B_Detector_SensorGeo('SWIR', config=self.cfg, logger=self.logger) - # self.swir.read_metadata(self._path_xml, lon_lat_smpl=lon_lat_smpl, nsmile_coef=nsmile_coef) diff --git a/enpt/options/config.py b/enpt/options/config.py index 3bd91d7..c01d002 100644 --- a/enpt/options/config.py +++ b/enpt/options/config.py @@ -35,7 +35,9 @@ path_options_default = os.path.join(path_enptlib, 'options', 'options_default.js config_for_testing = dict( path_l1b_enmap_image=os.path.abspath( - os.path.join(path_enptlib, '..', 'tests', 'data', 'EnMAP_Level_1B', 'AlpineTest1_CWV2_SM0.zip')), + os.path.join(path_enptlib, '..', 'tests', 'data', 'EnMAP_Level_1B_new', 'AlpineTest1_CWV2_SM0.zip')), + path_l1b_enmap_image_gapfill=os.path.abspath( + os.path.join(path_enptlib, '..', 'tests', 'data', 'EnMAP_Level_1B_new', 'AlpineTest2_CWV2_SM0.zip')), log_level='DEBUG', output_dir=os.path.join(path_enptlib, '..', 'tests', 'data', 'test_outputs') ) diff --git a/enpt/utils/path_generator.py b/enpt/utils/path_generator.py index 2f74904..d85c8b4 100644 --- a/enpt/utils/path_generator.py +++ b/enpt/utils/path_generator.py @@ -8,8 +8,13 @@ from xml.etree import ElementTree from ..model.metadata import L1B_product_props +# NOTE: +# paths belonging to providers L1B product are included in the *_header.xml file and read within metadata reader + + class PathGenL1BProduct(object): """Path generator class for generating file pathes corresponding to the EnMAP L1B product.""" + # TODO update this class def __init__(self, root_dir: str, detector_name: str): """Get an instance of the EnPT L1B image path generator. diff --git a/tests/__init__.py b/tests/__init__.py index 7359418..40a96af 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,14 +1 @@ # -*- coding: utf-8 -*- - -# import os -# -# enptRepo_rootpath = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) -# -# config_for_testing = dict( -# path_l1b_enmap_image=os.path.join(enptRepo_rootpath, 'tests', 'data', 'EnMAP_Level_1B_v2', 'Alpine1_CWV2_SM0'), -# path_l1b_enmap_image_ext=os.path.join(enptRepo_rootpath, 'tests', 'data', 'EnMAP_Level_1B_v2', -# 'Alpine2_CWV2_SM0'), -# path_l1b_enmap_image_ext2=os.path.join(enptRepo_rootpath, 'tests', 'data', 'EnMAP_Level_1B_v2', -# 'Alpine3_CWV2_SM0'), -# output_dir=os.path.join(enptRepo_rootpath, 'tests', 'data', 'test_outputs') -# ) diff --git a/tests/data/EnMAP_Level_1B/AlpineTest1_CWV2_SM0.zip b/tests/data/EnMAP_Level_1B/AlpineTest1_CWV2_SM0.zip deleted file mode 100644 index a4d4ace..0000000 --- a/tests/data/EnMAP_Level_1B/AlpineTest1_CWV2_SM0.zip +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f700eea124d70e97fbc38c7dfaad79ea65a08549ca484202abf90efc42c8d802 -size 23472036 diff --git a/tests/data/EnMAP_Level_1B/AlpineTest1_CWV2_SM1.zip b/tests/data/EnMAP_Level_1B/AlpineTest1_CWV2_SM1.zip deleted file mode 100644 index e3159ed..0000000 --- a/tests/data/EnMAP_Level_1B/AlpineTest1_CWV2_SM1.zip +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e31195239f427142cfa2692791771d4a966882714284e435e819db8a8335c1e2 -size 23470704 diff --git a/tests/data/Landsat8__500x500x2.bsq b/tests/data/Landsat8__500x500x2.bsq deleted file mode 100644 index 75b8435..0000000 --- a/tests/data/Landsat8__500x500x2.bsq +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:886bbc478c8f56999d41ba0e17a0070582129806a1bbd652fbea6aa745b4ce20 -size 4000000 diff --git a/tests/data/Landsat8__500x500x2.hdr b/tests/data/Landsat8__500x500x2.hdr deleted file mode 100644 index 7cd2064..0000000 --- a/tests/data/Landsat8__500x500x2.hdr +++ /dev/null @@ -1,16 +0,0 @@ -ENVI -description = { -/home/gfz-fe/scheffler/python/EnPT/tests/data/Landsat8__500x500x2.bsq} -samples = 500 -lines = 500 -bands = 2 -header offset = 0 -file type = ENVI Standard -data type = 5 -interleave = bsq -byte order = 0 -map info = {UTM, 1, 1, 291000, 4452000, 5, 5, 43, North,WGS-84} -coordinate system string = {PROJCS["WGS_1984_UTM_Zone_43N",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",75],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",0],UNIT["Meter",1]]} -band names = { -Band 1, -Band 2} diff --git a/tests/linting/pydocstyle.log b/tests/linting/pydocstyle.log index 3af9382..c98a566 100644 --- a/tests/linting/pydocstyle.log +++ b/tests/linting/pydocstyle.log @@ -1,50 +1,52 @@ enpt/version.py:1 at module level: D100: Missing docstring in public module -enpt/io/reader.py:42 in public method `read_inputdata`: +enpt/io/reader.py:40 in public method `read_inputdata`: D205: 1 blank line required between summary line and description (found 0) -enpt/io/reader.py:42 in public method `read_inputdata`: +enpt/io/reader.py:40 in public method `read_inputdata`: D400: First line should end with a period (not ']') -enpt/model/images.py:344 in public method `get_paths`: +enpt/model/images.py:340 in public method `get_paths`: D205: 1 blank line required between summary line and description (found 0) -enpt/model/images.py:344 in public method `get_paths`: +enpt/model/images.py:340 in public method `get_paths`: D400: First line should end with a period (not 'o') -enpt/model/images.py:390 in public method `generate_quicklook`: +enpt/model/images.py:386 in public method `generate_quicklook`: D205: 1 blank line required between summary line and description (found 0) -enpt/model/images.py:390 in public method `generate_quicklook`: +enpt/model/images.py:386 in public method `generate_quicklook`: D400: First line should end with a period (not 'e') -enpt/model/images.py:492 in public method `get_paths`: +enpt/model/images.py:454 in public method `get_paths`: D205: 1 blank line required between summary line and description (found 0) -enpt/model/images.py:492 in public method `get_paths`: +enpt/model/images.py:454 in public method `get_paths`: D400: First line should end with a period (not 'o') -enpt/model/images.py:616 in public method `run_AC`: +enpt/model/images.py:565 in public method `run_AC`: D102: Missing docstring in public method -enpt/model/images.py:623 in public method `save`: +enpt/model/images.py:572 in public method `save`: D205: 1 blank line required between summary line and description (found 0) -enpt/model/images.py:623 in public method `save`: +enpt/model/images.py:572 in public method `save`: D400: First line should end with a period (not 't') -enpt/model/images.py:714 in public method `append_new_image`: +enpt/model/images.py:614 in public method `append_new_image`: D205: 1 blank line required between summary line and description (found 0) -enpt/model/metadata.py:97 in public method `read_metadata`: +enpt/model/metadata.py:83 in public method `read_metadata`: D205: 1 blank line required between summary line and description (found 0) -enpt/model/metadata.py:97 in public method `read_metadata`: +enpt/model/metadata.py:83 in public method `read_metadata`: D400: First line should end with a period (not 'y') -enpt/model/metadata.py:290 in public method `interpolate_corners`: +enpt/model/metadata.py:218 in public method `interpolate_corners`: D205: 1 blank line required between summary line and description (found 0) -enpt/model/metadata.py:306 in public method `calc_solar_irradiance_CWL_FWHM_per_band`: +enpt/model/metadata.py:234 in public method `calc_solar_irradiance_CWL_FWHM_per_band`: D102: Missing docstring in public method -enpt/model/metadata.py:364 in public method `read_common_meta`: +enpt/model/metadata.py:255 in public class `EnMAP_Metadata_L1B_SensorGeo`: + D413: Missing blank line after last section ('Attributes') +enpt/model/metadata.py:293 in public method `read_common_meta`: D202: No blank lines allowed after function docstring (found 1) -enpt/model/metadata.py:364 in public method `read_common_meta`: +enpt/model/metadata.py:293 in public method `read_common_meta`: D205: 1 blank line required between summary line and description (found 0) -enpt/model/metadata.py:364 in public method `read_common_meta`: +enpt/model/metadata.py:293 in public method `read_common_meta`: D400: First line should end with a period (not 'o') -enpt/model/metadata.py:397 in public method `get_earth_sun_distance`: +enpt/model/metadata.py:318 in public method `get_earth_sun_distance`: D400: First line should end with a period (not ')') -enpt/model/metadata.py:419 in public method `read_metadata`: +enpt/model/metadata.py:340 in public method `read_metadata`: D202: No blank lines allowed after function docstring (found 1) -enpt/model/metadata.py:419 in public method `read_metadata`: +enpt/model/metadata.py:340 in public method `read_metadata`: D205: 1 blank line required between summary line and description (found 0) -enpt/model/metadata.py:419 in public method `read_metadata`: +enpt/model/metadata.py:340 in public method `read_metadata`: D400: First line should end with a period (not 'y') enpt/model/srf.py:11 in public class `SRF`: D101: Missing docstring in public class @@ -56,7 +58,9 @@ enpt/model/srf.py:114 in public method `__getitem__`: D105: Missing docstring in magic method enpt/model/srf.py:117 in public method `__iter__`: D105: Missing docstring in magic method -enpt/utils/path_generator.py:53 in public function `get_path_ac_options`: +enpt/utils/path_generator.py:15 in public class `PathGenL1BProduct`: + D204: 1 blank line required after class docstring (found 0) +enpt/utils/path_generator.py:58 in public function `get_path_ac_options`: D401: First line should be in imperative mood ('Return', not 'Returns') enpt/utils/logging.py:83 in public method `__getstate__`: D105: Missing docstring in magic method @@ -74,41 +78,37 @@ enpt/processors/atmospheric_correction/atmospheric_correction.py:25 in public me D102: Missing docstring in public method enpt/processors/atmospheric_correction/atmospheric_correction.py:47 in public method `run_ac`: D102: Missing docstring in public method -enpt/execution/controller.py:48 in public method `read_L1B_data`: - D205: 1 blank line required between summary line and description (found 0) -enpt/execution/controller.py:48 in public method `read_L1B_data`: - D400: First line should end with a period (not ':') -enpt/execution/controller.py:81 in public method `run_geometry_processor`: +enpt/execution/controller.py:87 in public method `run_geometry_processor`: D102: Missing docstring in public method -enpt/execution/controller.py:88 in public method `write_output`: +enpt/execution/controller.py:94 in public method `write_output`: D102: Missing docstring in public method -enpt/options/config.py:44 in public class `EnPTConfig`: +enpt/options/config.py:46 in public class `EnPTConfig`: D101: Missing docstring in public class -enpt/options/config.py:45 in public method `__init__`: +enpt/options/config.py:47 in public method `__init__`: D202: No blank lines allowed after function docstring (found 1) -enpt/options/config.py:125 in public method `absPath`: +enpt/options/config.py:127 in public method `absPath`: D102: Missing docstring in public method -enpt/options/config.py:128 in public method `get_parameter`: +enpt/options/config.py:130 in public method `get_parameter`: D102: Missing docstring in public method -enpt/options/config.py:191 in public method `to_dict`: +enpt/options/config.py:193 in public method `to_dict`: D202: No blank lines allowed after function docstring (found 1) -enpt/options/config.py:205 in public method `to_jsonable_dict`: +enpt/options/config.py:207 in public method `to_jsonable_dict`: D102: Missing docstring in public method -enpt/options/config.py:216 in public method `__repr__`: +enpt/options/config.py:218 in public method `__repr__`: D105: Missing docstring in magic method -enpt/options/config.py:220 in public function `json_to_python`: +enpt/options/config.py:222 in public function `json_to_python`: D103: Missing docstring in public function -enpt/options/config.py:253 in public function `python_to_json`: +enpt/options/config.py:255 in public function `python_to_json`: D103: Missing docstring in public function -enpt/options/config.py:275 in public class `EnPTValidator`: +enpt/options/config.py:277 in public class `EnPTValidator`: D101: Missing docstring in public class -enpt/options/config.py:276 in public method `__init__`: +enpt/options/config.py:278 in public method `__init__`: D205: 1 blank line required between summary line and description (found 0) -enpt/options/config.py:276 in public method `__init__`: +enpt/options/config.py:278 in public method `__init__`: D400: First line should end with a period (not 'r') -enpt/options/config.py:284 in public method `validate`: +enpt/options/config.py:286 in public method `validate`: D102: Missing docstring in public method -enpt/options/config.py:289 in public function `get_options`: +enpt/options/config.py:291 in public function `get_options`: D202: No blank lines allowed after function docstring (found 1) enpt/options/__init__.py:1 at module level: D104: Missing docstring in public package diff --git a/tests/test_l1b_reader.py b/tests/test_l1b_reader.py index 60efc2e..68506fd 100644 --- a/tests/test_l1b_reader.py +++ b/tests/test_l1b_reader.py @@ -9,13 +9,10 @@ Tests for `l1b_reader` module. """ import unittest -from glob import glob import os from os import path -# import sys import tempfile import zipfile -# from datetime import datetime import shutil from enpt.options.config import EnPTConfig, config_for_testing @@ -25,8 +22,9 @@ class Test_L1B_Reader(unittest.TestCase): """Tests for L1B_Reader class..""" def setUp(self): - self.pathList_testimages = glob(os.path.join(os.path.dirname(__file__), "data", "EnMAP_Level_1B_new", "*.zip")) self.config = EnPTConfig(**config_for_testing) + self.pathList_testimages = [self.config.path_l1b_enmap_image, + self.config.path_l1b_enmap_image_gapfill] self.tmpdir = tempfile.mkdtemp(dir=self.config.working_dir) def tearDown(self): diff --git a/tests/test_radiometric_transform.py b/tests/test_radiometric_transform.py index c344176..020985c 100644 --- a/tests/test_radiometric_transform.py +++ b/tests/test_radiometric_transform.py @@ -3,7 +3,6 @@ import os from unittest import TestCase, main -from glob import glob import tempfile import zipfile @@ -16,7 +15,8 @@ class Test_Radiometric_Transformer(TestCase): def setUp(self): """Set up the needed test data""" self.cfg = EnPTConfig(**config_for_testing) - self.pathList_testimages = glob(os.path.join(os.path.dirname(__file__), "data", "EnMAP_Level_1B", "*.zip")) + self.pathList_testimages = [self.cfg.path_l1b_enmap_image, + self.cfg.path_l1b_enmap_image_gapfill] self.RT = Radiometric_Transformer(config=self.cfg) def test_transform_TOARad2TOARef(self): -- GitLab From d82a135a7728b346ef7b7d34f4f9d9f72289a107 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Mon, 11 Jun 2018 17:17:27 +0200 Subject: [PATCH 06/11] Added parameter 'n_lines_to_append' to config. --- enpt/execution/controller.py | 3 ++- enpt/io/reader.py | 2 +- enpt/options/config.py | 1 + enpt/options/options_default.json | 7 +++++-- enpt/options/options_schema.py | 2 ++ 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/enpt/execution/controller.py b/enpt/execution/controller.py index 10fd22f..bb1d8a3 100644 --- a/enpt/execution/controller.py +++ b/enpt/execution/controller.py @@ -71,7 +71,8 @@ class EnPT_Controller(object): # run the reader RD = L1B_Reader(config=self.cfg) - L1_obj = RD.read_inputdata(root_dir_main=path_enmap_image, root_dir_ext=path_enmap_image_gapfill) + L1_obj = RD.read_inputdata(root_dir_main=path_enmap_image, root_dir_ext=path_enmap_image_gapfill, + n_line_ext=self.cfg.n_lines_to_append) return L1_obj diff --git a/enpt/io/reader.py b/enpt/io/reader.py index f2ce153..d61f2a4 100644 --- a/enpt/io/reader.py +++ b/enpt/io/reader.py @@ -49,7 +49,7 @@ class L1B_Reader(object): Read L1B EnMAP data. Extend the image by adding a second image [entire, partial] :param root_dir_main: Root directory of the main EnMAP Level-1B product :param root_dir_ext: Root directory of the extended EnMAP Level-1B product [optional] - :param n_line_ext: Number of line to be added to the main image [if None, use the whole image] + :param n_line_ext: Number of lines to be added to the main image [if None, use the whole image] :param lon_lat_smpl: number if sampling points in lon, lat fields :param compute_snr: whether to compute SNR or not (default: True) :return: instance of EnMAPL1Product_SensorGeo diff --git a/enpt/options/config.py b/enpt/options/config.py index c01d002..a7e6ece 100644 --- a/enpt/options/config.py +++ b/enpt/options/config.py @@ -79,6 +79,7 @@ class EnPTConfig(object): self.path_l1b_enmap_image_gapfill = self.absPath(gp('path_l1b_enmap_image_gapfill')) self.path_l1b_snr_model = self.absPath(gp('path_l1b_snr_model')) self.working_dir = self.absPath(gp('working_dir')) or None + self.n_lines_to_append = gp('n_lines_to_append') ################## # output options # diff --git a/enpt/options/options_default.json b/enpt/options/options_default.json index f772613..b76d6ed 100644 --- a/enpt/options/options_default.json +++ b/enpt/options/options_default.json @@ -4,9 +4,12 @@ "log_level": "INFO", /*the logging level to be used (choices: 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL')*/ "create_logfile": true, /*whether to write all log messages to a file (within output the directory)*/ "path_l1b_enmap_image": "", /*input path of the EnMAP L1B image to be processed (zip-archive or root directory)*/ - "path_l1b_enmap_image_gapfill": "", /*input path of an adjacent EnMAP L1B image to be used for gap-filling (zip-archive or root directory)*/ + "path_l1b_enmap_image_gapfill": "", /*input path of an adjacent EnMAP L1B image to be used for gap-filling + (zip-archive or root directory)*/ "path_l1b_snr_model": "./resources/EnMAP_Sensor/EnMAP_Level_1B_SNR.zip", /*input path of the EnMAP SNR model*/ - "working_dir": "" /*directory to be used for temporary files*/ + "working_dir": "", /*directory to be used for temporary files*/ + "n_lines_to_append": "None" /*Number of lines to be added to the main image [if None, use the whole path_l1b_enmap_image_gapfill]. + Requires path_l1b_enmap_image_gapfill to be set.*/ }, "output": { diff --git a/enpt/options/options_schema.py b/enpt/options/options_schema.py index 4e475c1..f0d19c2 100644 --- a/enpt/options/options_schema.py +++ b/enpt/options/options_schema.py @@ -11,6 +11,7 @@ enpt_schema_input = dict( path_l1b_enmap_image_gapfill=dict(type='string', required=False), path_l1b_snr_model=dict(type='string', required=False), working_dir=dict(type='string', required=False, nullable=True), + n_lines_to_append=dict(type='integer', required=False, nullable=True), )), output=dict( @@ -79,6 +80,7 @@ parameter_mapping = dict( path_l1b_enmap_image_gapfill=('general_opts', 'path_l1b_enmap_image_gapfill'), path_l1b_snr_model=('general_opts', 'path_l1b_snr_model'), working_dir=('general_opts', 'working_dir'), + n_lines_to_append=('general_opts', 'n_lines_to_append'), # output output_dir=('output', 'output_dir'), -- GitLab From 28a520e150baa44b4292f358d3c13acd7778fd3e Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Mon, 11 Jun 2018 17:31:50 +0200 Subject: [PATCH 07/11] Updated enpt_cli.py --- bin/enpt_cli.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/bin/enpt_cli.py b/bin/enpt_cli.py index 0c2b4c9..4370cc1 100644 --- a/bin/enpt_cli.py +++ b/bin/enpt_cli.py @@ -33,15 +33,35 @@ def get_enpt_argparser(): 'EnPT/blob/master/enpt/options/options_default.json') add('--CPUs', type=int, default=None, help='number of CPU cores to be used for processing (default: "None" -> use all available') - add('-im', '--path_l1b_enmap_image', default=None, + add('-im', '--path_l1b_enmap_image', type=str, default=None, help='input path of the EnMAP L1B image to be processed ' '(zip-archive or root directory; must be given if not contained in --json-config.)') - add('-imgap', '--path_l1b_enmap_image_gapfill', default=None, + add('-imgap', '--path_l1b_enmap_image_gapfill', type=str, default=None, help='input path of an adjacent EnMAP L1B image to be used for gap-filling (zip-archive or root directory)') - add('-od', '--output_dir', default=None, + add('-od', '--output_dir', type=str, default=None, help='output directory where processed data and log files are saved') - add('-wd', '--working_dir', default=None, + add('-wd', '--working_dir', type=str, default=None, help='directory to be used for temporary files') + add('-nla', '--n_lines_to_append', type=int, default=None, + help='number of lines to be added to the main image [if None, use the whole imgap]. Requires --imgap to be set') + add('--path_earthSunDist', type=str, default=None, + help='input path of the earth sun distance model') + add('--path_solar_irr', type=str, default=None, + help='input path of the solar irradiance model') + add('--scale_factor_toa_ref', type=int, default=None, + help='scale factor to be applied to TOA reflectance result') + add('--enable_keystone_correction', type=int, default=False) + add('--enable_vnir_swir_coreg', type=int, default=False) + add('--path_reference_image', type=str, default=None) + add('--sicor_cache_dir', type=str, default=None) + add('--auto_download_ecmwf', type=bool, default=False) + add('--enable_cloud_screening', type=bool, default=False) + add('--scale_factor_boa_ref', type=int, default=10000) + add('--run_smile_P', type=bool, default=False) + add('--run_deadpix_P', type=bool, default=True) + add('--deadpix_P_algorithm', type=str, default="spectral") + add('--deadpix_P_interp', type=str, default="linear") + add('--ortho_resampAlg', type=int, default=1) # link parser to run function parser.set_defaults(func=run_job) -- GitLab From 2e62d8cf9658a1543780b1837e608f91fe552802 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Mon, 11 Jun 2018 17:36:42 +0200 Subject: [PATCH 08/11] Renamed EnMAP test data. --- enpt/options/config.py | 4 ++-- .../AlpineTest1_CWV2_SM0_v2.zip} | 0 .../AlpineTest2_CWV2_SM0_v2.zip} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename tests/data/{EnMAP_Level_1B_new/AlpineTest1_CWV2_SM0.zip => EnMAP_Level_1B/AlpineTest1_CWV2_SM0_v2.zip} (100%) rename tests/data/{EnMAP_Level_1B_new/AlpineTest2_CWV2_SM0.zip => EnMAP_Level_1B/AlpineTest2_CWV2_SM0_v2.zip} (100%) diff --git a/enpt/options/config.py b/enpt/options/config.py index a7e6ece..2377533 100644 --- a/enpt/options/config.py +++ b/enpt/options/config.py @@ -35,9 +35,9 @@ path_options_default = os.path.join(path_enptlib, 'options', 'options_default.js config_for_testing = dict( path_l1b_enmap_image=os.path.abspath( - os.path.join(path_enptlib, '..', 'tests', 'data', 'EnMAP_Level_1B_new', 'AlpineTest1_CWV2_SM0.zip')), + os.path.join(path_enptlib, '..', 'tests', 'data', 'EnMAP_Level_1B', 'AlpineTest1_CWV2_SM0_v2.zip')), path_l1b_enmap_image_gapfill=os.path.abspath( - os.path.join(path_enptlib, '..', 'tests', 'data', 'EnMAP_Level_1B_new', 'AlpineTest2_CWV2_SM0.zip')), + os.path.join(path_enptlib, '..', 'tests', 'data', 'EnMAP_Level_1B', 'AlpineTest2_CWV2_SM0_v2.zip')), log_level='DEBUG', output_dir=os.path.join(path_enptlib, '..', 'tests', 'data', 'test_outputs') ) diff --git a/tests/data/EnMAP_Level_1B_new/AlpineTest1_CWV2_SM0.zip b/tests/data/EnMAP_Level_1B/AlpineTest1_CWV2_SM0_v2.zip similarity index 100% rename from tests/data/EnMAP_Level_1B_new/AlpineTest1_CWV2_SM0.zip rename to tests/data/EnMAP_Level_1B/AlpineTest1_CWV2_SM0_v2.zip diff --git a/tests/data/EnMAP_Level_1B_new/AlpineTest2_CWV2_SM0.zip b/tests/data/EnMAP_Level_1B/AlpineTest2_CWV2_SM0_v2.zip similarity index 100% rename from tests/data/EnMAP_Level_1B_new/AlpineTest2_CWV2_SM0.zip rename to tests/data/EnMAP_Level_1B/AlpineTest2_CWV2_SM0_v2.zip -- GitLab From 605932757292707d47ef3e7c0c69847fed96d6f1 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Mon, 11 Jun 2018 18:36:02 +0200 Subject: [PATCH 09/11] Some updates due to updated SICOR API. Added parameter 'disable_progress_bars' to config. Renamed test data. Docker image now clones SICOR master branch again. --- bin/enpt_cli.py | 2 ++ enpt/model/images.py | 2 +- enpt/model/metadata.py | 8 ++++---- enpt/options/config.py | 9 ++++++--- enpt/options/options_default.json | 5 +++-- enpt/options/options_schema.py | 2 ++ .../atmospheric_correction/atmospheric_correction.py | 4 ++-- .../dead_pixel_correction/dead_pixel_correction.py | 6 ++++-- ...ineTest1_CWV2_SM0_v2.zip => AlpineTest1_CWV2_SM0.zip} | 0 ...ineTest2_CWV2_SM0_v2.zip => AlpineTest2_CWV2_SM0.zip} | 0 tests/gitlab_CI_docker/build_enpt_testsuite_image.sh | 6 +++--- 11 files changed, 27 insertions(+), 17 deletions(-) rename tests/data/EnMAP_Level_1B/{AlpineTest1_CWV2_SM0_v2.zip => AlpineTest1_CWV2_SM0.zip} (100%) rename tests/data/EnMAP_Level_1B/{AlpineTest2_CWV2_SM0_v2.zip => AlpineTest2_CWV2_SM0.zip} (100%) diff --git a/bin/enpt_cli.py b/bin/enpt_cli.py index 4370cc1..7e8b723 100644 --- a/bin/enpt_cli.py +++ b/bin/enpt_cli.py @@ -44,6 +44,8 @@ def get_enpt_argparser(): help='directory to be used for temporary files') add('-nla', '--n_lines_to_append', type=int, default=None, help='number of lines to be added to the main image [if None, use the whole imgap]. Requires --imgap to be set') + add('-dpb', '--disable_progress_bars', type=bool, default=False, + help='whether to disable all progress bars during processing') add('--path_earthSunDist', type=str, default=None, help='input path of the earth sun distance model') add('--path_solar_irr', type=str, default=None, diff --git a/enpt/model/images.py b/enpt/model/images.py index fb117f0..215bb1b 100644 --- a/enpt/model/images.py +++ b/enpt/model/images.py @@ -360,7 +360,7 @@ class EnMAP_Detector_SensorGeo(_EnMAP_Image): Dead_Pixel_Corrector(algorithm=self.cfg.deadpix_P_algorithm, interp=self.cfg.deadpix_P_interp, logger=self.logger)\ - .correct(self.data, self.deadpixelmap) + .correct(self.data, self.deadpixelmap, progress=False if self.cfg.disable_progress_bars else True) def DN2TOARadiance(self): """Convert DNs to TOA radiance. diff --git a/enpt/model/metadata.py b/enpt/model/metadata.py index 3ef2622..300ba71 100644 --- a/enpt/model/metadata.py +++ b/enpt/model/metadata.py @@ -257,7 +257,7 @@ class EnMAP_Metadata_L1B_SensorGeo(object): Attributes: - logger(logging.Logger): None or logging instance - - observation_datetime(datetime.datetime): datetime of observation time (currently missing in metadata) + - observation_datetime(datetime.datetime): datetime of observation time - geom_view_zenith: viewing zenith angle - geom_view_azimuth: viewing azimuth angle - geom_sun_zenith: sun zenith angle @@ -302,11 +302,11 @@ class EnMAP_Metadata_L1B_SensorGeo(object): xml = ElementTree.parse(path_xml).getroot() # read the acquisition time - self.observation_datetime = xml.findall("GeneralInfo/ProductInfo/ProductStartTime")[0].text + self.observation_datetime = \ + datetime.strptime(xml.findall("GeneralInfo/ProductInfo/ProductStartTime")[0].text, '%Y-%m-%dT%H:%M:%S.%fZ') # get the distance earth sun from the acquisition date - self.earthSunDist = \ - self.get_earth_sun_distance(datetime.strptime(self.observation_datetime, '%Y-%m-%dT%H:%M:%S.%fZ')) + self.earthSunDist = self.get_earth_sun_distance(self.observation_datetime) # read Geometry (observation/illumination) angle self.geom_view_zenith = np.float(xml.findall("GeneralInfo/Geometry/Observation/ZenithAngle")[0].text) diff --git a/enpt/options/config.py b/enpt/options/config.py index 2377533..87a5ce8 100644 --- a/enpt/options/config.py +++ b/enpt/options/config.py @@ -35,11 +35,13 @@ path_options_default = os.path.join(path_enptlib, 'options', 'options_default.js config_for_testing = dict( path_l1b_enmap_image=os.path.abspath( - os.path.join(path_enptlib, '..', 'tests', 'data', 'EnMAP_Level_1B', 'AlpineTest1_CWV2_SM0_v2.zip')), + os.path.join(path_enptlib, '..', 'tests', 'data', 'EnMAP_Level_1B', 'AlpineTest1_CWV2_SM0.zip')), path_l1b_enmap_image_gapfill=os.path.abspath( - os.path.join(path_enptlib, '..', 'tests', 'data', 'EnMAP_Level_1B', 'AlpineTest2_CWV2_SM0_v2.zip')), + os.path.join(path_enptlib, '..', 'tests', 'data', 'EnMAP_Level_1B', 'AlpineTest2_CWV2_SM0.zip')), log_level='DEBUG', - output_dir=os.path.join(path_enptlib, '..', 'tests', 'data', 'test_outputs') + output_dir=os.path.join(path_enptlib, '..', 'tests', 'data', 'test_outputs'), + n_lines_to_append=50, + disable_progress_bars=True ) @@ -80,6 +82,7 @@ class EnPTConfig(object): self.path_l1b_snr_model = self.absPath(gp('path_l1b_snr_model')) self.working_dir = self.absPath(gp('working_dir')) or None self.n_lines_to_append = gp('n_lines_to_append') + self.disable_progress_bars = gp('disable_progress_bars') ################## # output options # diff --git a/enpt/options/options_default.json b/enpt/options/options_default.json index b76d6ed..c11f83a 100644 --- a/enpt/options/options_default.json +++ b/enpt/options/options_default.json @@ -8,8 +8,9 @@ (zip-archive or root directory)*/ "path_l1b_snr_model": "./resources/EnMAP_Sensor/EnMAP_Level_1B_SNR.zip", /*input path of the EnMAP SNR model*/ "working_dir": "", /*directory to be used for temporary files*/ - "n_lines_to_append": "None" /*Number of lines to be added to the main image [if None, use the whole path_l1b_enmap_image_gapfill]. - Requires path_l1b_enmap_image_gapfill to be set.*/ + "n_lines_to_append": "None", /*Number of lines to be added to the main image [if None, use the whole path_l1b_enmap_image_gapfill]. + Requires path_l1b_enmap_image_gapfill to be set.*/ + "disable_progress_bars": false /*whether to disable all progress bars during processing*/ }, "output": { diff --git a/enpt/options/options_schema.py b/enpt/options/options_schema.py index f0d19c2..4c225c6 100644 --- a/enpt/options/options_schema.py +++ b/enpt/options/options_schema.py @@ -12,6 +12,7 @@ enpt_schema_input = dict( path_l1b_snr_model=dict(type='string', required=False), working_dir=dict(type='string', required=False, nullable=True), n_lines_to_append=dict(type='integer', required=False, nullable=True), + disable_progress_bars=dict(type='boolean', required=False, nullable=True), )), output=dict( @@ -81,6 +82,7 @@ parameter_mapping = dict( path_l1b_snr_model=('general_opts', 'path_l1b_snr_model'), working_dir=('general_opts', 'working_dir'), n_lines_to_append=('general_opts', 'n_lines_to_append'), + disable_progress_bars=('general_opts', 'disable_progress_bars'), # output output_dir=('output', 'output_dir'), diff --git a/enpt/processors/atmospheric_correction/atmospheric_correction.py b/enpt/processors/atmospheric_correction/atmospheric_correction.py index d263a5c..3f0f7b5 100644 --- a/enpt/processors/atmospheric_correction/atmospheric_correction.py +++ b/enpt/processors/atmospheric_correction/atmospheric_correction.py @@ -21,8 +21,7 @@ class AtmosphericCorrector(object): """Create an instance of AtmosphericCorrector.""" self.cfg = config - @staticmethod - def get_ac_options(buffer_dir): + def get_ac_options(self, buffer_dir): path_opts = get_path_ac_options() try: @@ -37,6 +36,7 @@ class AtmosphericCorrector(object): tmp='%0f,', tau_a='%.2f,', vza='%.0f,') + vv["disable"] = self.cfg.disable_progress_bars options["ECMWF"]["path_db"] = "./ecmwf" return options diff --git a/enpt/processors/dead_pixel_correction/dead_pixel_correction.py b/enpt/processors/dead_pixel_correction/dead_pixel_correction.py index 29d5ca1..01ce41a 100644 --- a/enpt/processors/dead_pixel_correction/dead_pixel_correction.py +++ b/enpt/processors/dead_pixel_correction/dead_pixel_correction.py @@ -37,13 +37,15 @@ class Dead_Pixel_Corrector(object): '(%s, %s). Received %s.' % (image2correct.shape, image2correct.bands, image2correct.columns, deadcolumn_map.shape)) - def correct(self, image2correct: Union[np.ndarray, GeoArray], deadcolumn_map: Union[np.ndarray, GeoArray]): + def correct(self, image2correct: Union[np.ndarray, GeoArray], deadcolumn_map: Union[np.ndarray, GeoArray], + progress=False): """ NOTE: dead columns in the first or the last band are unmodified. :param image2correct: :param deadcolumn_map: + :param progress: whether to show progress bars :return: """ image2correct = GeoArray(image2correct) if not isinstance(image2correct, GeoArray) else image2correct @@ -98,7 +100,7 @@ class Dead_Pixel_Corrector(object): # spectral interpolation # ######################### - for band, column in tqdm(np.argwhere(deadcolumn_map), disable=False): # set disable=True to mute progress + for band, column in tqdm(np.argwhere(deadcolumn_map), disable=False if progress else True): if band in band_nbrs_spatial_interp: continue # already interpolated spatially above diff --git a/tests/data/EnMAP_Level_1B/AlpineTest1_CWV2_SM0_v2.zip b/tests/data/EnMAP_Level_1B/AlpineTest1_CWV2_SM0.zip similarity index 100% rename from tests/data/EnMAP_Level_1B/AlpineTest1_CWV2_SM0_v2.zip rename to tests/data/EnMAP_Level_1B/AlpineTest1_CWV2_SM0.zip diff --git a/tests/data/EnMAP_Level_1B/AlpineTest2_CWV2_SM0_v2.zip b/tests/data/EnMAP_Level_1B/AlpineTest2_CWV2_SM0.zip similarity index 100% rename from tests/data/EnMAP_Level_1B/AlpineTest2_CWV2_SM0_v2.zip rename to tests/data/EnMAP_Level_1B/AlpineTest2_CWV2_SM0.zip diff --git a/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh b/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh index 4acaa48..f6a46a6 100755 --- a/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh +++ b/tests/gitlab_CI_docker/build_enpt_testsuite_image.sh @@ -2,13 +2,13 @@ context_dir="./context" dockerfile="enpt_ci.docker" -tag="enpt_ci:0.4.0b4" +tag="enpt_ci:0.4.0" gitlab_runner="enpt_gitlab_CI_runner" # get sicor project rm -rf context/sicor -# git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git ./context/sicor -git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git --branch feature/improve_enmap --single-branch ./context/sicor +git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git ./context/sicor +# git clone https://gitext.gfz-potsdam.de/EnMAP/sicor.git --branch feature/improve_enmap --single-branch ./context/sicor # download sicor cache (fastens SICOR CI tests a lot, but cache needs to be updated manually using a local sicor repo: # 1. clone a fresh copy of sicor or delete sicor/sicor/aerosol_0_ch4_34d3778719cc87188787de09bb8f870d16050078.pkl.zip -- GitLab From a88081092ddeb1cd9a87626610a77adce4f14ead Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Tue, 12 Jun 2018 16:03:30 +0200 Subject: [PATCH 10/11] Update due to updated SICOR API. --- .../processors/atmospheric_correction/atmospheric_correction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enpt/processors/atmospheric_correction/atmospheric_correction.py b/enpt/processors/atmospheric_correction/atmospheric_correction.py index 3f0f7b5..90c4ef9 100644 --- a/enpt/processors/atmospheric_correction/atmospheric_correction.py +++ b/enpt/processors/atmospheric_correction/atmospheric_correction.py @@ -36,7 +36,7 @@ class AtmosphericCorrector(object): tmp='%0f,', tau_a='%.2f,', vza='%.0f,') - vv["disable"] = self.cfg.disable_progress_bars + vv["disable_progress_bars"] = self.cfg.disable_progress_bars options["ECMWF"]["path_db"] = "./ecmwf" return options -- GitLab From 85cfc746d0c04b81bdb855eba4da228610b9f3f1 Mon Sep 17 00:00:00 2001 From: Daniel Scheffler Date: Tue, 12 Jun 2018 16:24:02 +0200 Subject: [PATCH 11/11] Bugfix. --- .../radiometric_transform.py | 2 +- tests/linting/pydocstyle.log | 42 +++++++++---------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/enpt/processors/radiometric_transform/radiometric_transform.py b/enpt/processors/radiometric_transform/radiometric_transform.py index a09ce72..8f1e57b 100644 --- a/enpt/processors/radiometric_transform/radiometric_transform.py +++ b/enpt/processors/radiometric_transform/radiometric_transform.py @@ -40,7 +40,7 @@ class Radiometric_Transformer(object): # compute TOA reflectance constant = \ self.cfg.scale_factor_toa_ref * math.pi * enmap_ImageL1.meta.earthSunDist ** 2 / \ - (math.cos(math.radians(detector.detector_meta.geom_sun_zenith))) + (math.cos(math.radians(enmap_ImageL1.meta.geom_sun_zenith))) solIrr = np.array([detector.detector_meta.solar_irrad[band] for band in detector.detector_meta.srf.bands])\ .reshape(1, 1, detector.data.bands) toaRef = (constant * detector.data[:] / solIrr).astype(np.int16) diff --git a/tests/linting/pydocstyle.log b/tests/linting/pydocstyle.log index c98a566..69a5550 100644 --- a/tests/linting/pydocstyle.log +++ b/tests/linting/pydocstyle.log @@ -74,49 +74,49 @@ enpt/utils/logging.py:144 in public method `__exit__`: D105: Missing docstring in magic method enpt/processors/dead_pixel_correction/dead_pixel_correction.py:15 in public class `Dead_Pixel_Corrector`: D101: Missing docstring in public class -enpt/processors/atmospheric_correction/atmospheric_correction.py:25 in public method `get_ac_options`: +enpt/processors/atmospheric_correction/atmospheric_correction.py:24 in public method `get_ac_options`: D102: Missing docstring in public method enpt/processors/atmospheric_correction/atmospheric_correction.py:47 in public method `run_ac`: D102: Missing docstring in public method -enpt/execution/controller.py:87 in public method `run_geometry_processor`: +enpt/execution/controller.py:88 in public method `run_geometry_processor`: D102: Missing docstring in public method -enpt/execution/controller.py:94 in public method `write_output`: +enpt/execution/controller.py:95 in public method `write_output`: D102: Missing docstring in public method -enpt/options/config.py:46 in public class `EnPTConfig`: +enpt/options/config.py:48 in public class `EnPTConfig`: D101: Missing docstring in public class -enpt/options/config.py:47 in public method `__init__`: +enpt/options/config.py:49 in public method `__init__`: D202: No blank lines allowed after function docstring (found 1) -enpt/options/config.py:127 in public method `absPath`: +enpt/options/config.py:131 in public method `absPath`: D102: Missing docstring in public method -enpt/options/config.py:130 in public method `get_parameter`: +enpt/options/config.py:134 in public method `get_parameter`: D102: Missing docstring in public method -enpt/options/config.py:193 in public method `to_dict`: +enpt/options/config.py:197 in public method `to_dict`: D202: No blank lines allowed after function docstring (found 1) -enpt/options/config.py:207 in public method `to_jsonable_dict`: +enpt/options/config.py:211 in public method `to_jsonable_dict`: D102: Missing docstring in public method -enpt/options/config.py:218 in public method `__repr__`: +enpt/options/config.py:222 in public method `__repr__`: D105: Missing docstring in magic method -enpt/options/config.py:222 in public function `json_to_python`: +enpt/options/config.py:226 in public function `json_to_python`: D103: Missing docstring in public function -enpt/options/config.py:255 in public function `python_to_json`: +enpt/options/config.py:259 in public function `python_to_json`: D103: Missing docstring in public function -enpt/options/config.py:277 in public class `EnPTValidator`: +enpt/options/config.py:281 in public class `EnPTValidator`: D101: Missing docstring in public class -enpt/options/config.py:278 in public method `__init__`: +enpt/options/config.py:282 in public method `__init__`: D205: 1 blank line required between summary line and description (found 0) -enpt/options/config.py:278 in public method `__init__`: +enpt/options/config.py:282 in public method `__init__`: D400: First line should end with a period (not 'r') -enpt/options/config.py:286 in public method `validate`: +enpt/options/config.py:290 in public method `validate`: D102: Missing docstring in public method -enpt/options/config.py:291 in public function `get_options`: +enpt/options/config.py:295 in public function `get_options`: D202: No blank lines allowed after function docstring (found 1) enpt/options/__init__.py:1 at module level: D104: Missing docstring in public package -enpt/options/options_schema.py:115 in public function `get_updated_schema`: +enpt/options/options_schema.py:119 in public function `get_updated_schema`: D103: Missing docstring in public function -enpt/options/options_schema.py:116 in private nested function `deep_update`: +enpt/options/options_schema.py:120 in private nested function `deep_update`: D202: No blank lines allowed after function docstring (found 1) -enpt/options/options_schema.py:116 in private nested function `deep_update`: +enpt/options/options_schema.py:120 in private nested function `deep_update`: D400: First line should end with a period (not 'e') -enpt/options/options_schema.py:135 in public function `get_param_from_json_config`: +enpt/options/options_schema.py:139 in public function `get_param_from_json_config`: D103: Missing docstring in public function -- GitLab