Added options: coreg_max_shift_allowed, coreg_window_size,...

Added options: coreg_max_shift_allowed, coreg_window_size, ac_scale_factor_errors, path_custom_sicor_options, ac_fillnonclear_areas, ac_clear_area_labels, ac_max_ram_gb

gms_preprocessing/algorithms/L1B_P.py

@@ -529,13 +529,15 @@ class L1B_object(L1A_object):
         elif self.coreg_needed and self.spatRef_available:
             geoArr_ref = GeoArray(self.spatRef_scene.filePath)
             geoArr_shift = GeoArray(self.arr)
-            r_b4match, s_b4match = self.get_opt_bands4matching(target_cwlPos_nm=CFG.band_wavelength_for_matching,
+            r_b4match, s_b4match = self.get_opt_bands4matching(target_cwlPos_nm=CFG.coreg_band_wavelength_for_matching,
                                                                v=False)
             coreg_kwargs = dict(
                 r_b4match=r_b4match,
                 s_b4match=s_b4match,
                 align_grids=True,  # FIXME not needed here
                 match_gsd=True,  # FIXME not needed here
+                max_shift=CFG.coreg_max_shift_allowed,
+                window_size=CFG.coreg_window_size,
                 data_corners_ref=[[x, y] for x, y in self.spatRef_scene.polyUTM.convex_hull.exterior.coords],
                 data_corners_tgt=[transform_any_prj(EPSG2WKT(4326), self.meta_odict['coordinate system string'], x, y)
                                   for x, y in HLP_F.reorder_CornerLonLat(self.trueDataCornerLonLat)],

gms_preprocessing/algorithms/L1C_P.py

@@ -927,8 +927,8 @@ class AtmCorr(object):
                 ac_bandNs = [bandN for bandN in inObj.arr.bandnames if bandN in self.results.data_ac.keys()]
 
                 ac_errors = np.dstack((self.results.data_errors[bandN] for bandN in ac_bandNs))
-                ac_errors *= CFG.scale_factor_errors_ac  # scale using scale factor (output is float16)
-                out_dtype = np.int8 if CFG.scale_factor_errors_ac <= 255 else np.int16
+                ac_errors *= CFG.ac_scale_factor_errors  # scale using scale factor (output is float16)
+                out_dtype = np.int8 if CFG.ac_scale_factor_errors <= 255 else np.int16
                 ac_errors[nodata] = get_outFillZeroSaturated(out_dtype)[0]
                 ac_errors = ac_errors.astype(out_dtype)
                 inObj.ac_errors = ac_errors  # setter generates a GeoArray with the same bandnames like inObj.arr

gms_preprocessing/model/gms_object.py

@@ -313,7 +313,7 @@ class GMS_object(Dataset):
         """
 
         if not self._ac_options:
-            path_ac_options = PG.get_path_ac_options(self.GMS_identifier)
+            path_ac_options = CFG.path_custom_sicor_options or PG.get_path_ac_options(self.GMS_identifier)
 
             if path_ac_options and os.path.exists(path_ac_options):
                 # don't validate because options contain pathes that do not exist on another server:
@@ -329,6 +329,12 @@ class GMS_object(Dataset):
                 if 'uncertainties' in opt_dict:
                     opt_dict['uncertainties']['snr_model'] = PG.get_path_snr_model(self.GMS_identifier)
 
+                # apply custom configuration
+                opt_dict["logger"]['level'] = CFG.log_level
+                opt_dict["ram"]['upper_limit'] = CFG.ac_max_ram_gb
+                opt_dict["ram"]['unit'] = 'GB'
+                opt_dict["AC"]['fill_nonclear_areas'] = CFG.ac_fillnonclear_areas
+                opt_dict["AC"]['clear_area_labels'] = CFG.ac_clear_area_labels
                 # opt_dict['AC']['n_cores'] = CFG.CPUs if CFG.allow_subMultiprocessing else 1
 
                 self._ac_options = opt_dict

gms_preprocessing/options/config.py

@@ -182,6 +182,7 @@ class JobConfig(object):
         self.path_tempdir = self.absP(self.DB_config_table['path_tempdir'])
         self.path_ac_tables = self.absP(self.DB_config_table['path_ac_tables'])
         self.path_SNR_models = self.absP(self.DB_config_table['path_SNR_models'])
+        self.path_custom_sicor_options = gp('path_custom_sicor_options', json_paths['path_custom_sicor_options'])
         self.path_dem_proc_srtm_90m = self.absP(self.DB_config_table['path_dem_proc_srtm_90m'])
 
         if not self.is_test:
@@ -283,8 +284,12 @@ class JobConfig(object):
             gp('spatial_ref_plusminus_days', json_processors['L1B']['spatial_ref_plusminus_days'])
         self.spatial_ref_plusminus_years = \
             gp('spatial_ref_plusminus_years', json_processors['L1B']['spatial_ref_plusminus_years'])
-        self.band_wavelength_for_matching = \
-            gp('band_wavelength_for_matching', json_processors['L1B']['band_wavelength_for_matching'])
+        self.coreg_band_wavelength_for_matching = \
+            gp('coreg_band_wavelength_for_matching', json_processors['L1B']['coreg_band_wavelength_for_matching'])
+        self.coreg_max_shift_allowed = \
+            gp('coreg_max_shift_allowed', json_processors['L1B']['coreg_max_shift_allowed'])
+        self.coreg_window_size = \
+            gp('coreg_window_size', json_processors['L1B']['coreg_window_size'])
 
         # L1C
         self.exec_L1CP = gp('exec_L1CP', [
@@ -295,10 +300,16 @@ class JobConfig(object):
             gp('cloud_masking_algorithm', json_processors['L1C']['cloud_masking_algorithm'])
         self.export_L1C_obj_dumps = \
             gp('export_L1C_obj_dumps', json_processors['L1C']['export_L1C_obj_dumps'])
-        self.scale_factor_errors_ac = \
-            gp('scale_factor_errors_ac', json_processors['L1C']['scale_factor_errors_ac'])
         self.auto_download_ecmwf = \
             gp('auto_download_ecmwf', json_processors['L1C']['auto_download_ecmwf'])
+        self.ac_fillnonclear_areas = \
+            gp('ac_fillnonclear_areas', json_processors['L1C']['ac_fillnonclear_areas'])
+        self.ac_clear_area_labels = \
+            gp('ac_clear_area_labels', json_processors['L1C']['ac_clear_area_labels'])
+        self.ac_scale_factor_errors = \
+            gp('ac_scale_factor_errors', json_processors['L1C']['ac_scale_factor_errors'])
+        self.ac_max_ram_gb = \
+            gp('ac_max_ram_gb', json_processors['L1C']['ac_max_ram_gb'])
 
         # L2A
         self.exec_L2AP = gp('exec_L2AP', [

gms_preprocessing/options/options_default.json

@@ -33,6 +33,7 @@
         "path_earthSunDist": "",
         "path_solar_irr": "",
         "path_cloud_classif": "",
+        "path_custom_sicor_options": "",
         "path_ECMWF_db": ""
     },
 
@@ -56,7 +57,6 @@
             "delete_output": true,
             "SZA_SAA_calculation_accurracy": "coarse",  /*"coarse" or "fine"*/
             "export_VZA_SZA_SAA_RAA_stats": true
-
         },
 
         "L1B": {  /*Level 1B processing: calculation of geometric shift*/
@@ -69,7 +69,9 @@
             "spatial_ref_max_cloudcov": 20,  /*maximum cloud cover of reference scene in percent*/
             "spatial_ref_plusminus_days": 30,  /*maximum time interval between target and reference scene in days*/
             "spatial_ref_plusminus_years": 10,  /*maximum time interval between target and reference scene in years*/
-            "band_wavelength_for_matching": 550  /*optimal wavelength [nm] of the spectral bands used for matching*/
+            "coreg_band_wavelength_for_matching": 550,  /*optimal wavelength [nm] of the spectral bands used for matching*/
+            "coreg_max_shift_allowed": 5,  /*maximum shift distance in reference image pixel units (default: 5 px)*/
+            "coreg_window_size": [256, 256]
         },
 
         "L1C": {  /*Level 1C processing: atmospheric correction*/
@@ -85,8 +87,13 @@
                 "Sentinel-2B": "SICOR"
             },  /*'FMASK', 'Classical Bayesian', 'SICOR'*/
             "export_L1C_obj_dumps": false,
-            "scale_factor_errors_ac": 255,
-            "auto_download_ecmwf": false
+            "auto_download_ecmwf": false,
+            "ac_fillnonclear_areas": false,  /*whether to fill non-clear areas with TOA Reflectance values or not*/
+            "ac_clear_area_labels": ["Clear", "Snow", "Water", "Shadow"],
+                /*Defines to which area labels SICOR is applied. Full list:
+                ["Clear", "Snow", "Water", "Shadow", "Cirrus", "Cloud"] => whole image is atmosperically corrected*/
+            "ac_scale_factor_errors": 255,
+            "ac_max_ram_gb": 20  /*maximum amount of RAM to be allocated for atmospheric correction [gigabytes]*/
         },
 
         "L2A": {  /*Level 2A processing: geometric homogenization*/

gms_preprocessing/options/options_schema.py

@@ -35,6 +35,7 @@ gms_schema_input = dict(
             path_earthSunDist=dict(type='string', required=False),
             path_solar_irr=dict(type='string', required=False),
             path_cloud_classif=dict(type='string', required=False),
+            path_custom_sicor_options=dict(type='string', required=False, nullable=True),
             path_ECMWF_db=dict(type='string', required=False),
         )),
     processors=dict(
@@ -68,7 +69,10 @@ gms_schema_input = dict(
                 spatial_ref_max_cloudcov=dict(type='float', required=False, min=0, max=100),
                 spatial_ref_plusminus_days=dict(type='integer', required=False),
                 spatial_ref_plusminus_years=dict(type='integer', required=False),
-                band_wavelength_for_matching=dict(type='integer', required=False, min=350, max=2500),
+                coreg_band_wavelength_for_matching=dict(type='integer', required=False, min=350, max=2500),
+                coreg_max_shift_allowed=dict(type='float', required=False, min=0),
+                coreg_window_size=dict(type='list', required=False, minlength=0, maxlength=2,
+                                       schema=dict(type='integer', required=False, min=8))
                 )),
             L1C=dict(type='dict', required=False, schema=dict(
                 run_processor=dict(type='boolean', required=False),
@@ -85,8 +89,11 @@ gms_schema_input = dict(
                                                                                 'SICOR']),
                     }),
                 export_L1C_obj_dumps=dict(type='boolean', required=False),
-                scale_factor_errors_ac=dict(type='integer', required=False),
                 auto_download_ecmwf=dict(type='boolean', required=False),
+                ac_fillnonclear_areas=dict(type='boolean', required=False),
+                ac_clear_area_labels=dict(type='list', required=False), # TODO
+                ac_scale_factor_errors=dict(type='integer', required=False),
+                ac_max_ram_gb=dict(type='integer', required=False),
                 )),
             L2A=dict(type='dict', required=False, schema=dict(
                 run_processor=dict(type='boolean', required=False),