Skip to content
GitLab
Commit a3a7cec3 authored by Daniel Scheffler's avatar Daniel Scheffler
Browse files

Changed workflow to get SICOR options an to pass paths of tables and... 

Changed workflow to get SICOR options an to pass paths of tables and persistence files after sicor issue #6 has been fixed.
parent cbb49edf
Pipeline #1434 passed with stage
in 7 minutes and 30 seconds
Hide whitespace changes
Inline Side-by-side
gms_preprocessing/database/sicor_options/l8_options.json deleted 100644 → 0
View file @ cbb49edf
{
"ram": {
"upper_limit": 20,
"unit": "GB"
},
"ozo_to_DU": 71524.3,
"ozone_fallback":500.0, /* in [DU] */
"dT/dh":-0.006, /* in [K/m]*/
"S2Image": {
"sliceY": "slice(None, None, None)",
"sliceX": "slice(None, None, None)",
"unit": "reflectance"
},
"ECMWF": {
"total_AOT": "fc_total_AOT_550nm",
"path_db": "/home/gts2/data/aux_data/ECMWF/",
"target_resolution": 30.0,
"max_delta_day": 10,
"variables_aerosol": [
"fc_total_AOT_550nm",
"fc_sulphate_AOT_550nm",
"fc_black_carbon_AOT_550nm",
"fc_dust_AOT_550nm",
"fc_organic_matter_AOT_550nm",
"fc_sea_salt_AOT_550nm"
],
"variable_tcwv":"fc_TCWV",
"var2type": {
"fc_organic_matter_AOT_550nm": "aerosol_2",
"fc_sulphate_AOT_550nm": "aerosol_2",
"fc_sea_salt_AOT_550nm": "aerosol_1",
"fc_black_carbon_AOT_550nm": "aerosol_0",
"fc_dust_AOT_550nm": "aerosol_3"
}
},
"logger": {
"pprint": {
"indent": 3,
"compact": false,
"width": 100
},
"logg_options": true,
"datefmt": "%Y%m%d-%H:%M:%S",
"level": "INFO",
"format": "%(asctime)s:%(levelname)s::%(message)s",
"name": "SICOR"
},
"DEM": {
"fn": "/home/gts2/data/aux_data/dem/GLOBAL_SRTM_90m.h5",
"target_resolution": 30.0,
"return_zeros_if_missing": true
},
"RTFO": {
"aerosol_1": {
"flag": 10,
"table_path": "/table_aerosol/type_1",
"dim_scat": [
"tau_a"
],
"atm_tables_fn": "/usr/local/gts2/linear_atm_functions_ncwv_5_npre_4_ncoz_2_ntmp_2_wvl_350.0_2550.0_1.00_pca.h5",
"dim_atm": [
"spr",
"coz",
"cwv",
"tmp"
],
"only_toa": true
},
"aerosol_0": {
"flag": 10,
"table_path": "/table_aerosol/type_0",
"dim_scat": [
"tau_a"
],
"atm_tables_fn": "/usr/local/gts2/linear_atm_functions_ncwv_5_npre_4_ncoz_2_ntmp_2_wvl_350.0_2550.0_1.00_pca.h5",
"dim_atm": [
"spr",
"coz",
"cwv",
"tmp"
],
"only_toa": true
},
"cirrus": {
"flag": 20,
"table_path": "/table_cirrus",
"dim_scat": [
"tau_c",
"eff_rad_cirris"
],
"atm_tables_fn": "/usr/local/gts2/linear_atm_functions_ncwv_5_npre_4_ncoz_2_ntmp_2_wvl_350.0_2550.0_1.00_pca.h5",
"dim_atm": [
"spr",
"coz",
"cwv",
"tmp"
],
"only_toa": true
},
"aerosol_2": {
"flag": 10,
"table_path": "/table_aerosol/type_2",
"dim_scat": [
"tau_a"
],
"atm_tables_fn": "/usr/local/gts2/linear_atm_functions_ncwv_5_npre_4_ncoz_2_ntmp_2_wvl_350.0_2550.0_1.00_pca.h5",
"dim_atm": [
"spr",
"coz",
"cwv",
"tmp"
],
"only_toa": true
},
"aerosol_3": {
"flag": 10,
"table_path": "/table_aerosol/type_3",
"dim_scat": [
"tau_a"
],
"atm_tables_fn": "/usr/local/gts2/linear_atm_functions_ncwv_5_npre_4_ncoz_2_ntmp_2_wvl_350.0_2550.0_1.00_pca.h5",
"dim_atm": [
"spr",
"coz",
"cwv",
"tmp"
],
"only_toa": true
}
},
"report": {
"reporting": false, /* either true or false */
"RGB": {
"output_size": 5490,
"hist_chop_off_fraction": 0.05,
"rgb_bands": ["B01", "B02", "B03"]
},
"HTML": false,
"figs": {
"DEM": {},
"RGB": {}
},
"figs_clouds": {
"DEM": {},
"RGB": {}
},
"report_path": "/home/gts2/logs/L2A/processing_reports/[TYPE]/",
"JPG": true,
"dpi": 200,
"n_cols": 2
},
"cld_mask": {
"target_resolution": 30.0,
"nodata_value_mask":255,
"persistence_file": "/home/gts2/data/aux_data/cld_mask/cld_mask_S2_classi_20170411_v20170411_16:08:52.h5",
"processing_tiles": 4,
"novelty_detector": "/home/gts2/data/aux_data/cld_mask/noclear_novelty_detector_channel2_difference9_0_index10_1_channel12_index1_8.retrain.pkl",
"majority_mask_filter":[
{"block_replace": true,"block_replace_params": [[60,4]],"majority_width": 0},
{"block_replace": true,"block_replace_params": [[50,3]],"majority_width": 0},
{"block_replace": true,"block_replace_params": [[50, 10], [30, 10]],"majority_width": 10}
]
},
"AC": {
"aerosol_type_estimation":"maximum_ECMWF_type", /*should be:maximum_ECMWF_type*/
"max_pixel_processing": 30140100,
"fill_nonclear_areas": false,
"dd_statistics":10,
"n_smpl": {
"spr": 3,
"cwv": 3,
"tau_a": 3,
"rho": 12
},
"min_clear_fraction": 0.1,
"clear_area_labels": ["Clear", "Snow", "Water", "Shadow"],
"default_aerosol_type": "aerosol_2",
"override_aerosol_type": "auto",
"parameter_bounds": "image",
"n_cores": 1,
"bands": [
"B01",
"B02",
"B03",
"B04",
"B05",
"B06",
"B07"
],
"target_resolution": 30.0
},
"water_vapor": {
"type":"ECMWF"
},
"SRF": "L8",
"uncertainties":{
"snr_model": 100.0,
"model_linear": [[1,1,1,1]],
"model_vars": ["tau","spr","cwv","I"],
"default_errors": {
"dspr": 0.5,
"dcwv": 1.0,
"dtau": 0.05},
"model_quadratic": [
[0,0,0,0],
[0,0,0,0],
[0,0,0,0],
[0,0,0,1]
]
},
"output": [{"type":"none"}]
}
gms_preprocessing/database/sicor_options/s2_options.json deleted 100644 → 0
View file @ cbb49edf
{
"ram": {
"upper_limit": 20,
"unit": "GB"
},
"ozo_to_DU": 71524.3,
"ozone_fallback":500.0, /* in [DU] */
"dT/dh":-0.006, /* in [K/m]*/
"S2Image": {
"S2_MSI_granule_path": "None",
"sliceY": "slice(None, None, None)",
"sliceX": "slice(None, None, None)",
"unit": "reflectance",
"aux_fields": {
"spr": [
20.0
],
"cwv": [
60.0,
20.0
],
"ozo": "mean"
},
"target_resolution": "None",
"import_bands": "all",
"driver": "gdal_JP2OpenJPEG" /*gdal_JP2KAK or gdal_JP2OpenJPEG*/
},
"ECMWF": {
"total_AOT": "fc_total_AOT_550nm",
"path_db": "./",
"target_resolution": 20.0,
"max_delta_day": 10,
"variables_aerosol": [
"fc_total_AOT_550nm",
"fc_sulphate_AOT_550nm",
"fc_black_carbon_AOT_550nm",
"fc_dust_AOT_550nm",
"fc_organic_matter_AOT_550nm",
"fc_sea_salt_AOT_550nm"
],
"variable_tcwv":"fc_TCWV",
"var2type": {
"fc_organic_matter_AOT_550nm": "aerosol_2",
"fc_sulphate_AOT_550nm": "aerosol_2",
"fc_sea_salt_AOT_550nm": "aerosol_1",
"fc_black_carbon_AOT_550nm": "aerosol_0",
"fc_dust_AOT_550nm": "aerosol_3"
}
},
"base_output_types": ["L2A","rgb_jpeg","metadata"],
"output": [
{
"output_bands": {
"10.0": [
"B02",
"B03",
"B04",
"B08"
],
"60.0": [
"B01"
],
"20.0": [
"B05",
"B06",
"B07",
"B8A",
"B11",
"B12"
]
},
"mask_ss": [
10.0,
20.0,
60.0
],
"nodata_value_data":65535,
"max_value_uncert":0.1,
"nodata_value_mask":255,
"mask_geo_band": "B11",
"out_dir": "./",
"options_loseless": {
"JP2KAK": [
"QUALITY=100"
],
"JP2OpenJPEG": [
"QUALITY=100",
"REVERSIBLE=YES"
]
},
"n_cores": 1,
"max_time_minutes": 20,
"type": "L2A",
"options_mask": {
"JP2KAK": [
"QUALITY=100"
],
"JP2OpenJPEG": [
"QUALITY=100",
"REVERSIBLE=YES",
"WRITE_METADATA=YES"
]
},
"gdal_driver": "JP2OpenJPEG", /*e.g. JP2KAK or JP2OpenJPEG*/
"options_lossy": {
"JP2KAK": [
"QUALITY=30"
],
"JP2OpenJPEG": [
"QUALITY=30",
"REVERSIBLE=NO"
]
}
},
{
"format": "json",
"type": "metadata"
},
{
"format": "xml",
"type": "metadata"
},
{
"type": "rgb_jpeg",
"output_size": 10980,
"hist_chop_off_fraction":0.05,
"quality":60,
"rgb_bands": ["B11", "B08", "B03"]
}
],
"logger": {
"pprint": {
"indent": 3,
"compact": false,
"width": 100
},
"logg_options": true,
"datefmt": "%Y%m%d-%H:%M:%S",
"level": "INFO",
"format": "%(asctime)s:%(levelname)s::%(message)s",
"name": "SICOR"
},
"DEM": {
"fn": "/home/gts2/data/aux_data/dem/GLOBAL_SRTM_90m.h5",
"target_resolution": 20.0,
"return_zeros_if_missing": true
},
"RTFO": {
"aerosol_1": {
"flag": 10,
"table_path": "/table_aerosol/type_1",
"dim_scat": [
"tau_a"
],
"atm_tables_fn": "/usr/local/gts2/linear_atm_functions_ncwv_5_npre_4_ncoz_2_ntmp_2_wvl_350.0_2550.0_1.00_pca.h5",
"dim_atm": [
"spr",
"coz",
"cwv",
"tmp"
],
"only_toa": true
},
"aerosol_0": {
"flag": 10,
"table_path": "/table_aerosol/type_0",
"dim_scat": [
"tau_a"
],
"atm_tables_fn": "/usr/local/gts2/linear_atm_functions_ncwv_5_npre_4_ncoz_2_ntmp_2_wvl_350.0_2550.0_1.00_pca.h5",
"dim_atm": [
"spr",
"coz",
"cwv",
"tmp"
],
"only_toa": true
},
"cirrus": {
"flag": 20,
"table_path": "/table_cirrus",
"dim_scat": [
"tau_c",
"eff_rad_cirris"
],
"atm_tables_fn": "/usr/local/gts2/linear_atm_functions_ncwv_5_npre_4_ncoz_2_ntmp_2_wvl_350.0_2550.0_1.00_pca.h5",
"dim_atm": [
"spr",
"coz",
"cwv",
"tmp"
],
"only_toa": true
},
"aerosol_2": {
"flag": 10,
"table_path": "/table_aerosol/type_2",
"dim_scat": [
"tau_a"
],
"atm_tables_fn": "/usr/local/gts2/linear_atm_functions_ncwv_5_npre_4_ncoz_2_ntmp_2_wvl_350.0_2550.0_1.00_pca.h5",
"dim_atm": [
"spr",
"coz",
"cwv",
"tmp"
],
"only_toa": true
},
"aerosol_3": {
"flag": 10,
"table_path": "/table_aerosol/type_3",
"dim_scat": [
"tau_a"
],
"atm_tables_fn": "/usr/local/gts2/linear_atm_functions_ncwv_5_npre_4_ncoz_2_ntmp_2_wvl_350.0_2550.0_1.00_pca.h5",
"dim_atm": [
"spr",
"coz",
"cwv",
"tmp"
],
"only_toa": true
}
},
"report": {
"reporting": false, /* either true or false */
"RGB": {
"output_size": 5490,
"hist_chop_off_fraction": 0.05
},
"HTML": false,
"figs": {
"MSK_rgb": {},
"DEM": {},
"CWV": {},
"AOT": {},
"RGB": {},
"SPR": {}
},
"figs_clouds": {
"MSK_rgb": {},
"DEM": {},
"RGB": {}
},
"report_path": "/home/gts2/logs/L2A/processing_reports/[TYPE]/",
"JPG": true,
"dpi": 200,
"n_cols": 2
},
"cld_mask": {
"target_resolution": 20.0,
"nodata_value_mask":255,
"persistence_file": "/home/gts2/data/aux_data/cld_mask/cld_mask_S2_classi_20170412_v20170412_11:43:14.h5",
"processing_tiles": 4,
"novelty_detector": "/home/gts2/data/aux_data/cld_mask/noclear_novelty_detector_channel2_difference9_0_index10_1_channel12_index1_8.retrain.pkl",
"majority_mask_filter":[
{"block_replace": true,"block_replace_params": [[60,2]],"majority_width": 0},
{"block_replace": true,"block_replace_params": [[50,3]],"majority_width": 0},
{"block_replace": true,"block_replace_params": [[50, 10], [30, 10]],"majority_width": 10}
]
},
"AC": {
"aerosol_type_estimation":"maximum_ECMWF_type", /*should be:maximum_ECMWF_type*/
"max_pixel_processing": 30140100,
"fill_nonclear_areas": false,
"n_smpl": {
"spr": 3,
"cwv": 3,
"tau_a": 3,
"rho": 12
},
"min_clear_fraction": 0.1,
"clear_area_labels": ["Clear", "Snow", "Water", "Shadow"],
"default_aerosol_type": "aerosol_2",
"override_aerosol_type": "auto",
"parameter_bounds": "image",
"dd_statistics":10,
"n_cores": 1,
"target_resolution": 20.0,
"bands": [
"B01",
"B02",
"B03",
"B04",
"B05",
"B06",
"B07",
"B08",
"B8A",
"B11",
"B12"
]
},
"water_vapor": {
"type":"retrieval",
"int_order": 3, /* number of standardeviation which are allowwd as min,max filter for the water vapor retreival */
"nanmean": 20,
"n_smpl": {
"spr": 5,
"cwv": 10,
"tau_a": 5,
"rho": 20
},
"sigma": 20,
"rho_bounds": [
0.1,
1.0
], /* water vapor retreival is only performed for surface reflectances within this lower and upper bound */
"n_std": 2,
"bands": [
"B8A",
"B09"
], /* band names to use for water vapor retreivel, this is instrument specfic */
"target_resolution": 60 /* spatial sampling in meter at which the retreival shall be done -> will we sampled to native spatial sampling */
},
"SRF": "S2A",
"uncertainties":{
"snr_model": "/home/gts2/data/aux_data/S2A/S2A_SNR_model.csv",
"model_linear": [[1,1,1,1]],
"model_vars": ["tau","spr","cwv","I"],
"default_errors": {
"dspr": 0.5,
"dcwv": 1.0,
"dtau": 0.05},
"model_quadratic": [
[0,0,0,0],
[0,0,0,0],
[0,0,0,0],
[0,0,0,1]
]
},
"aot_retrieval": {
"n_smpl": {
"spr": 2,
"tau_a": 5,
"rho": 5
},
"dark_nbins": 200,
"dark_fraction": "0.01",
"dark_bands": [
"B01",
"B02"
],
"dark_n_target": 500,
"bands": [
"B01",
"B02",
"B03",
"B04"
]
}
}
gms_preprocessing/misc/path_generator.py
View file @ a3a7cec3
......@@ -302,10 +302,10 @@ def get_path_ac_options(GMS_identifier):
ac_options_file_dic = {
'AVNIR-2': None,
'TM4': 'l8_options.json',
'TM5': 'l8_options.json',
'TM7': 'l8_options.json', # AC uses Landsat-8 options for L7 but reads only a subset of the options
'LDCM': 'l8_options.json',
'TM4': 'sicor_l8_user_options.json',
'TM5': 'sicor_l8_user_options.json',
'TM7': 'sicor_l8_user_options.json', # AC uses Landsat-8 options for L7 but reads only a subset of the options
'LDCM': 'sicor_l8_user_options.json',
'SPOT1a': None,
'SPOT1b': None,
'SPOT2a': None,
......@@ -318,8 +318,8 @@ def get_path_ac_options(GMS_identifier):
'SPOT5b': None,
'RE5': None,