diff --git a/GDE_TOOLS_create_industrial_cells.py b/GDE_TOOLS_create_industrial_cells.py index 405cbb60596303f5a2e048ee7b7278a01001dcc8..fbe09ad0090e1f2a8a7a3dba73365971c5cb8d96 100644 --- a/GDE_TOOLS_create_industrial_cells.py +++ b/GDE_TOOLS_create_industrial_cells.py @@ -1222,6 +1222,7 @@ def generate_country_industrial_cells( in_crs="EPSG:4326", consistency_tol_dist=0.05, consistency_tol_area=0.05, + verbose=False, ): """This function reads the input aggregated exposure model of the country (location defined by aggr_model_pathname and country), identifies the unique points present in this input @@ -1316,6 +1317,8 @@ def generate_country_industrial_cells( range, which is calculated as ([max-min]/mean) of all cells. Default: 0.05. Only needed if consistency_checks is True. + verbose (bool): if True, print statements will be executed to describe progress in the + calculations. Default = False. Returns: cells_adj_bound_gdf (GeoDataFrame): GeoPandas GeoDataFrame with the cells defined around @@ -1394,6 +1397,8 @@ def generate_country_industrial_cells( # Retrieve unique points in the exposure file, determined with a specific precision # (points_gdf is a GeoPandas GeoDataFrame, ids_aggr is an array of strings with length # equal to the number of rows of aggr_mod_df): + if verbose: + print(" Building cells from input points...") points_gdf, ids_aggr = retrieve_unique_points( aggr_mod_df, col_lon, col_lat, id_str, precision=precision_points, in_crs=in_crs ) @@ -1423,6 +1428,8 @@ def generate_country_industrial_cells( # Adjust all the coordinates of the corners of the cells (coords_dict) by taking the # average value of all instances of that coordinate that "should be the same", as identified # in coords_uq: + if verbose: + print(" Adjusting cells' geometries...") coords_dict_adj = adjust_coords(coords_dict, coords_uq) # coords_dict_adj is a dictionary # Generate final output with adjusted cell geometries (cells_adj_gdf is a @@ -1436,7 +1443,11 @@ def generate_country_industrial_cells( big_dist_diff = "Not_Checked" big_area_diff = "Not_Checked" else: + if verbose: + print(" Running consistency checks...") # Consistency check 1: the output geometries should not overlap + if verbose: + print(" Consistency check #1 of 4...") num_overlaps = 999 # Initialise variable for the while loop to run at least once while num_overlaps > 0: intsect_gdf = overlap_by_full_geom_intersection(cells_adj_gdf, "id_1", "id_2") @@ -1461,6 +1472,8 @@ def generate_country_industrial_cells( overlap_found = "False" # Consistency check 2: + if verbose: + print(" Consistency check #2 of 4...") gaps_found = True # Initialise variable for the while loop to run at least once while gaps_found: # Expand the cells by 25% of their dimensions in all directions: @@ -1499,6 +1512,8 @@ def generate_country_industrial_cells( gap_found = str(gaps_found) # Consistency check 3: maximum distance between original points and final centroids: + if verbose: + print(" Consistency check #3 of 4...") max_dist_centr = get_distance_centroids(cells_adj_gdf, col_lon, col_lat) # Compare the maximum distance against the tolerance: if max_dist_centr > min(width_EW, width_NS) * consistency_tol_dist: @@ -1507,6 +1522,8 @@ def generate_country_industrial_cells( big_dist_diff = "False" # Consistency check 4: stability/variability of area of resulting cells: + if verbose: + print(" Consistency check #4 of 4...") rel_area_range = get_relative_area_range(cells_gdf) # Compare the relative area range ([max-min]/mean) against the tolerance: if rel_area_range > consistency_tol_area: @@ -1515,6 +1532,8 @@ def generate_country_industrial_cells( big_area_diff = "False" # Intersect cells with admnistrative boundary of country: + if verbose: + print(" Intersecting cells with admnistrative boundary of country...") cells_adj_bound_gdf = gpd.overlay(cells_adj_gdf, bounds_gdf, how="intersection") # Eliminate columns that are not useful: if "ID_0" in cells_adj_bound_gdf.columns: @@ -1536,6 +1555,9 @@ def generate_country_industrial_cells( cells_adj_bound_gdf["lat_s"].values[row] = geometry_row.bounds[1] cells_adj_bound_gdf["lat_n"].values[row] = geometry_row.bounds[3] + if verbose: + print(" generate_country_industrial_cells has finished generating the cells") + return ( cells_adj_bound_gdf, aggr_mod_df, @@ -1545,3 +1567,147 @@ def generate_country_industrial_cells( big_area_diff, country_id, ) + + +def export_modified_aggregated_model_file(aggr_mod_df, out_pathname, out_filename, separ=","): + """This function writes the aggr_mod_df DataFrame to a CSV file with name out_filename, + under the path out_pathname. If the file already exists, it will overwrite it. If the path + does not exist, it will create it. + + Args: + aggr_mod_df (DataFrame): Pandas DataFrame to be exported. + out_pathname (str): Path to where the output CSV will be written. + out_filename (str): File name of the output CSV file. + separ: Separator to use in the CSV file (default=','). + """ + + # Create out_pathname if it does not exist: + if not os.path.exists(out_pathname): + os.makedirs(out_pathname) + + aggr_mod_df.to_csv(os.path.join(out_pathname, out_filename), sep=separ) + + +def export_cells_to_geodatafile( + country_name, country_id, cells_gdf, out_pathname, out_filename, out_driver="ESRI Shapefile" +): + """This function creates a geospatial data file of format given by out_driver, with the IDs + and geometries of cells_gdf. Additional fields are written with the country name and ID. If + the file already exists, it will overwrite it. If the path does not exist, it will create + it. + + Args: + country_name (str): Name of country (as in the files of the aggregated model). + country_id (int): ID of the country as per the geospatial data files of the input + aggregated model. + cells_gdf (GeoDataFrame): GeoPandas GeoDataFrame with the cells geometry. This function + assumes that it contains at least the following columns: + - id: ID of the cell, given by id_str and an incremental + integer. + - geometry: (Shapely) polygons of the output cells. + out_pathname (str): Path to where the geospatial data file will be written. + out_filename (str): File name of the geospatial data file. + out_driver (str): "ESRI Shapefile", "GeoJSON", "GPKG", etc. Format of the output + geospatial data file. + """ + + # Check that in_gdf has the needed columns and terminate otherwise: + if ("id" not in cells_gdf.columns) or ("geometry" not in cells_gdf.columns): + print("ERROR!! One or more of id, geometry missing as columns of cells_gdf") + return + + # Create out_pathname if it does not exist: + if not os.path.exists(out_pathname): + os.makedirs(out_pathname) + + # Start a new GeoDataFrame with the desired columns/features of cells_gdf: + shp_gdf = gpd.GeoDataFrame({}, geometry=cells_gdf["geometry"]) + shp_gdf["ID_99"] = deepcopy(cells_gdf["id"]) + shp_gdf["NAME_99"] = deepcopy(cells_gdf["id"]) + + # Add values of admin unit level 0: + id_0 = np.array([country_id for i in range(shp_gdf.shape[0])]) + name_0 = np.array([country_name for i in range(shp_gdf.shape[0])]) + shp_gdf["ID_0"] = id_0 + shp_gdf["NAME_0"] = name_0 + + # Write to ShapeFile or other format (as per out_driver): + shp_gdf.to_file(os.path.join(out_pathname, out_filename), driver=out_driver) + + +def which_countries_on_a_grid( + metadata_filepath, + sheet_name="IND", + col_name="Variables", + var_name="Resolution", + target="30 arc seconds", + export=False, + out_pathname="", + out_filename="", +): + """This function retrieves the list of countries for which the industrial exposure is + defined on a 30-arcsec grid in the SERA exposure model. This information is provided in the + EFEHR repository within an Excel spreadsheet called + "European_Exposure_Model_Data_Assumptions.xlsx". The default values of all input parameters + refer to the structure of this file. + + Args: + metadata_filepath (str): Full path to the source metadata file, including file name and + extension. It needs to be an Excel spreadsheet (.xls or .xlsx). + sheet_name (str): Name of the sheet of metadata_filepath to read. Default: "IND". + col_name (str): Name of the column of sheet_name that contains the parameter names. + Default: "Variables". + var_name (str): Name of the parameter to read. Default: "Resolution". + target (str): Target value of var_name (i.e. the code will seek for cases in which the + value of var_name is target. Default: "30 arc seconds". + export (bool): If True, the resulting list of countries will be exported to a CSV file + under the path out_pathname and the file name out_filename. + Default: False. + out_pathname (str): Path where to export the output CSV to. Only needed if export is + True. Default: "". + out_filename (str): Name of the file where to write the list of countries. Only needed + if export is True. Default: "". + + Returns: + countries (list of str): List of countries for which the var_name takes the value given + by target (the countries for which the industrial exposure is + defined on a 30-arcsec grid in the SERA exposure model. + """ + + # Check metadata_filepath exists and points at an Excel spreadsheet: + if not os.path.isfile(metadata_filepath): + print("ERROR in which_countries_on_a_grid: input file not found") + return [] + if "xls" not in metadata_filepath.split(".")[-1]: + print("ERROR in which_countries_on_a_grid: input file is not an Excel spreadsheet") + return [] + + # Read the SERA metadata file: + metadata = pd.read_excel(metadata_filepath, sheet_name=sheet_name) + + # Identify the row in which the variable var_name is given: + which_row = np.where(metadata[col_name].values == var_name)[0] + + if len(which_row) != 1: # This should not occur + print("ERROR READING %s: ROW NOT FOUND." % (metadata_filepath)) + return [] + + # Retrieve the row in which the variable var_name is given: + whole_row = metadata.iloc[which_row[0], :] + # Columns in which var_name takes the value target: + which_cols = np.where(whole_row == target)[0] + # The names of those columns are the countries for which var_name takes the value target: + countries = list(whole_row.index[which_cols]) + + # Write a csv file with the list of countries: + if export: + # Create out_pathname if it does not exist: + if not os.path.exists(out_pathname): + os.makedirs(out_pathname) + # Write to file: + out_filepath = os.path.join(out_pathname, out_filename) + f = open(out_filepath, "w") + f.write(",".join(countries)) + f.close() + + return countries diff --git a/GDE_TOOLS_read_config_file.py b/GDE_TOOLS_read_config_file.py index d36902a3089392b7f531962f809e2050032328d4..9b80db00dce6abc4c46b4f3406c577e9bead36e5 100644 --- a/GDE_TOOLS_read_config_file.py +++ b/GDE_TOOLS_read_config_file.py @@ -31,317 +31,536 @@ out (i.e. not every float is tested to check whether it is a float or not). """ import os -import configparser # Documentation: https://docs.python.org/3/library/configparser.html +import configparser # Documentation: https://docs.python.org/3/library/configparser.html + def read_config_parameters(config_file_path, section_names_needed): - config = configparser.ConfigParser() - config.read(config_file_path) - out_config= {} - for section_name in section_names_needed: - if section_name not in config: - print('ERROR!! '+section_name+' NOT FOUND IN CONFIGURATION FILE. THE CODE CANNOT RUN!!') - else: - check_parameters(config, section_name) - out_config[section_name]= config[section_name] - return out_config + config = configparser.ConfigParser() + config.read(config_file_path) + out_config = {} + for section_name in section_names_needed: + if section_name not in config: + print( + "ERROR!! " + + section_name + + " NOT FOUND IN CONFIGURATION FILE. THE CODE CANNOT RUN!!" + ) + else: + check_parameters(config, section_name) + out_config[section_name] = config[section_name] + return out_config def check_parameters(config, section_name): - """ - section_name: name of the section in the configuration file (name that goes in []) - """ - if section_name=='File Paths': - kinds_of_paths= ['out_path', 'sera_models_path', 'sera_models_OQ_path', 'sera_boundaries_path', 'ghs_path', 'gpw_path', 'sat_path', 'completeness_path'] - for path_kind in kinds_of_paths: - if not config.has_option('File Paths', path_kind): - raise IOError('ERROR!! '+path_kind+' PARAMETER MISSING FROM CONFIG FILE!!') - else: - if not os.path.isdir(config['File Paths'][path_kind]): - raise IOError('ERROR!! '+path_kind+': THE PATH SPECIFIED IN THE CONFIG FILE DOES NOT EXIST') - elif section_name=='Available Results': - if not config.has_option('Available Results', 'results_available_for'): - raise IOError('ERROR!! results_available_for PARAMETER MISSING FROM CONFIG FILE!!') - else: - num_results_cases= len(config['Available Results']['results_available_for'].split(', ')) - if not config.has_option('Available Results', 'result_paths'): - raise IOError('ERROR!! result_paths PARAMETER MISSING FROM CONFIG FILE!!') - else: - num_result_paths= len(config['Available Results']['result_paths'].split(', ')) - if num_results_cases!=num_result_paths: - raise IOError('ERROR!! results_available_for AND result_paths SHOULD HAVE THE SAME NUMBER OF ELEMENTS IN THE CONFIG FILE!!') - elif section_name=='OBM Database': - if not config.has_option('OBM Database', 'db_obm_name'): - raise IOError('ERROR!! db_obm_name PARAMETER MISSING FROM CONFIG FILE!!') - if not config.has_option('OBM Database', 'db_obm_username'): - raise IOError('ERROR!! db_obm_username PARAMETER MISSING FROM CONFIG FILE!!') - if not config.has_option('OBM Database', 'db_obm_schema_name'): - raise IOError('ERROR!! db_obm_schema_name PARAMETER MISSING FROM CONFIG FILE!!') - if not config.has_option('OBM Database', 'db_obm_table_name'): - raise IOError('ERROR!! db_obm_table_name PARAMETER MISSING FROM CONFIG FILE!!') - elif section_name=='Tiles Database': - if not config.has_option('Tiles Database', 'db_tiles_name'): - raise IOError('ERROR!! db_tiles_name PARAMETER MISSING FROM CONFIG FILE!!') - if not config.has_option('Tiles Database', 'db_tiles_username'): - raise IOError('ERROR!! db_tiles_username PARAMETER MISSING FROM CONFIG FILE!!') - if not config.has_option('Tiles Database', 'db_tiles_schema_name'): - raise IOError('ERROR!! db_tiles_schema_name PARAMETER MISSING FROM CONFIG FILE!!') - if not config.has_option('Tiles Database', 'db_tiles_table_name'): - raise IOError('ERROR!! db_tiles_table_name PARAMETER MISSING FROM CONFIG FILE!!') - elif section_name=='Admin Units Database': - if not config.has_option('Admin Units Database', 'db_admin_name'): - raise IOError('ERROR!! db_admin_name PARAMETER MISSING FROM CONFIG FILE!!') - if not config.has_option('Admin Units Database', 'db_admin_username'): - raise IOError('ERROR!! db_admin_username PARAMETER MISSING FROM CONFIG FILE!!') - if not config.has_option('Admin Units Database', 'db_admin_schema_name'): - raise IOError('ERROR!! db_admin_schema_name PARAMETER MISSING FROM CONFIG FILE!!') - if not config.has_option('Admin Units Database', 'db_admin_table_name'): - raise IOError('ERROR!! db_admin_table_name PARAMETER MISSING FROM CONFIG FILE!!') - elif section_name=='Cells to Process': - check_of_parameters_to_define_list_of_cell_IDs(config, section_name) # Check parameters related to defining the way in which the list of cells is defined - elif section_name=='Ocuppancy String Groups': - num_occup_cases= check_of_occupancy_cases_parameter(config, section_name) - if config.has_option('Ocuppancy String Groups', 'occupancy_classifications'): - # It should have it, it is mandatory - num_classif_subgroups= len(config['Ocuppancy String Groups']['occupancy_classifications'].split(', ')) - else: - raise IOError('ERROR!! OCCUPANCY CLASSIFICATIONS ARE NEEDED IN THE CONFIG FILE!!') - if (config.has_option('Ocuppancy String Groups', 'occupancy_cases') and config.has_option('Ocuppancy String Groups', 'occupancy_classifications')): - if num_occup_cases!=num_classif_subgroups: - raise IOError('ERROR!! LISTS OF OCCUPANCY STRINGS NEED TO BE PROVIDED IN occupancy_classifications FOR EACH CASE IN occupancy_cases!!') - elif section_name=='OBM_buildings_per_cell': - check_of_sera_disaggregation_to_consider_parameter(config, section_name) - elif section_name=='GDE_gather_SERA_and_OBM': - check_of_sera_disaggregation_to_consider_parameter(config, section_name) - _= check_of_occupancy_cases_parameter(config, section_name) - if not config.has_option('GDE_gather_SERA_and_OBM', 'version_of_SERA_metadata'): - raise IOError('ERROR!! version_of_SERA_metadata PARAMETER MISSING FROM CONFIG FILE!!') - if not config.has_option('GDE_gather_SERA_and_OBM', 'print_screen_during_run'): - raise IOError('ERROR!! print_screen_during_run PARAMETER MISSING FROM CONFIG FILE!!') - else: - check_if_param_is_True_or_False(config['GDE_gather_SERA_and_OBM']['print_screen_during_run'], 'print_screen_during_run') - elif section_name=='GDE_plot_maps': - _= check_of_occupancy_cases_parameter(config, section_name) - if not config.has_option('GDE_plot_maps', 'location_var'): - raise IOError('ERROR!! location_var PARAMETER MISSING FROM CONFIG FILE!!') - if not config.has_option('GDE_plot_maps', 'parameters'): - raise IOError('ERROR!! parameters (e.g., number, num_dwells, night, structural) PARAMETER MISSING FROM CONFIG FILE!!') - elif section_name=='GDE_check_consistency': - if not config.has_option('GDE_check_consistency', 'location_var'): - raise IOError('ERROR!! location_var PARAMETER MISSING FROM CONFIG FILE!!') - elif section_name=='OBM_assign_cell_ids_and_adm_ids_to_footprints': - if not config.has_option('OBM_assign_cell_ids_and_adm_ids_to_footprints', 'method_to_use'): - raise IOError('ERROR!! method_to_use PARAMETER MISSING FROM CONFIG FILE!!') - else: - if config['OBM_assign_cell_ids_and_adm_ids_to_footprints']['method_to_use']!='just_centroid': - raise IOError('ERROR!! method_to_use: THE ONLY CURRENTLY SUPPORTED VALUE IS just_centroid. PLEASE CHECK CONFIG FILE!!') - if not config.has_option('OBM_assign_cell_ids_and_adm_ids_to_footprints', 'obm_ids_to_process'): - raise IOError('ERROR!! obm_ids_to_process PARAMETER MISSING FROM CONFIG FILE!!') - else: - # To do: check that obm_ids_to_process is either "None" or a list of OBM IDs - pass - if not config.has_option('OBM_assign_cell_ids_and_adm_ids_to_footprints', 'assign_cell_IDs'): - raise IOError('ERROR!! assign_cell_IDs PARAMETER MISSING FROM CONFIG FILE!!') - else: - check_if_param_is_True_or_False(config['OBM_assign_cell_ids_and_adm_ids_to_footprints']['assign_cell_IDs'], 'assign_cell_IDs') - if not config.has_option('OBM_assign_cell_ids_and_adm_ids_to_footprints', 'assign_admin_IDs'): - raise IOError('ERROR!! assign_admin_IDs PARAMETER MISSING FROM CONFIG FILE!!') - else: - check_if_param_is_True_or_False(config['OBM_assign_cell_ids_and_adm_ids_to_footprints']['assign_admin_IDs'], 'assign_admin_IDs') - if not config.has_option('OBM_assign_cell_ids_and_adm_ids_to_footprints', 'occupancy_case'): - raise IOError('ERROR!! occupancy_case PARAMETER MISSING FROM CONFIG FILE!!') - if not config.has_option('OBM_assign_cell_ids_and_adm_ids_to_footprints', 'adm_level'): - raise IOError('ERROR!! adm_level PARAMETER MISSING FROM CONFIG FILE!!') - elif section_name=='SERA_create_HDF5_metadata': - if not config.has_option('SERA_create_HDF5_metadata', 'version_of_SERA_metadata'): - raise IOError('ERROR!! version_of_SERA_metadata PARAMETER MISSING FROM CONFIG FILE!!') - elif section_name=='SERA_create_OQ_input_files': - if not config.has_option('SERA_create_OQ_input_files', 'version_of_SERA_metadata'): - raise IOError('ERROR!! version_of_SERA_metadata PARAMETER MISSING FROM CONFIG FILE!!') - check_of_sera_disaggregation_to_consider_parameter(config, section_name) - _= check_of_occupancy_cases_parameter(config, section_name) - if not config.has_option('SERA_create_OQ_input_files', 'countries'): - raise IOError('ERROR!! countries PARAMETER MISSING FROM CONFIG FILE!!') - elif section_name=='SERA_create_outputs_QGIS_for_checking': - if not config.has_option('SERA_create_outputs_QGIS_for_checking', 'country'): - raise IOError('ERROR!! country PARAMETER MISSING FROM CONFIG FILE!!') - elif section_name=='SERA_create_visual_output_of_grid_model_full_files': - check_of_sera_disaggregation_to_consider_parameter(config, section_name) - _= check_of_occupancy_cases_parameter(config, section_name) - if not config.has_option('SERA_create_visual_output_of_grid_model_full_files', 'max_num_cells_per_stage'): - raise IOError('ERROR!! max_num_cells_per_stage PARAMETER MISSING FROM CONFIG FILE!!') - else: - if not config[section_name]['max_num_cells_per_stage'].isnumeric(): - raise IOError('ERROR!! max_num_cells_per_stage NEEDS TO BE A POSITIVE INTEGER!!') - elif section_name=='SERA_distributing_exposure_to_cells': - check_of_sera_disaggregation_to_consider_parameter(config, section_name) - if not config.has_option('SERA_distributing_exposure_to_cells', 'countries'): - raise IOError('ERROR!! countries PARAMETER MISSING FROM CONFIG FILE!!') - if not config.has_option('SERA_distributing_exposure_to_cells', 'columns_to_distribute'): - raise IOError('ERROR!! columns_to_distribute PARAMETER MISSING FROM CONFIG FILE!!') - if not config.has_option('SERA_distributing_exposure_to_cells', 'write_hdf5_bdg_classes_param'): - raise IOError('ERROR!! write_hdf5_bdg_classes_param PARAMETER MISSING FROM CONFIG FILE!!') - else: - check_if_param_is_True_or_False(config['SERA_distributing_exposure_to_cells']['write_hdf5_bdg_classes_param'], 'write_hdf5_bdg_classes_param') - elif section_name=='SERA_exploration_investigate_full_CSV_files': - if not config.has_option('SERA_exploration_investigate_full_CSV_files', 'tolerance_ratio'): - raise IOError('ERROR!! tolerance_ratio PARAMETER MISSING FROM CONFIG FILE!!') - else: - try: - float(config[section_name]['tolerance_ratio']) - except: - raise IOError('ERROR!! tolerance_ratio PARAMETER NEEDS TO BE A REAL NUMBER') - if not config.has_option('SERA_exploration_investigate_full_CSV_files', 'tolerance_diff'): - raise IOError('ERROR!! tolerance_diff PARAMETER MISSING FROM CONFIG FILE!!') - else: - try: - float(config[section_name]['tolerance_diff']) - except: - raise IOError('ERROR!! tolerance_diff PARAMETER NEEDS TO BE A REAL NUMBER') - elif section_name=='SERA_exploration_investigate_macro_taxonomies': - if not config.has_option('SERA_exploration_investigate_macro_taxonomies', 'country'): - raise IOError('ERROR!! country PARAMETER MISSING FROM CONFIG FILE!!') - _= check_of_occupancy_cases_parameter(config, section_name) - elif section_name=='SERA_exploration_summarise_columns_in_full_CSV_files': - _= check_of_occupancy_cases_parameter(config, section_name) - elif section_name=='SERA_mapping_admin_units_to_cells': - if not config.has_option('SERA_mapping_admin_units_to_cells', 'countries'): - raise IOError('ERROR!! countries PARAMETER MISSING FROM CONFIG FILE!!') - elif section_name=='SERA_mapping_admin_units_to_cells_add_GHS': - if not config.has_option('SERA_mapping_admin_units_to_cells_add_GHS', 'ghs_input_filename'): - raise IOError('ERROR!! ghs_input_filename PARAMETER MISSING FROM CONFIG FILE!!') - elif section_name=='SERA_mapping_admin_units_to_cells_add_Sat': - if not config.has_option('SERA_mapping_admin_units_to_cells_add_Sat', 'apply_model'): - raise IOError('ERROR!! apply_model PARAMETER MISSING FROM CONFIG FILE!!') - else: - check_if_param_is_True_or_False(config['SERA_mapping_admin_units_to_cells_add_Sat']['apply_model'], 'apply_model') - elif section_name=='SERA_testing_compare_visual_output_vs_OQ_input_files': - if not config.has_option('SERA_testing_compare_visual_output_vs_OQ_input_files', 'country'): - raise IOError('ERROR!! country PARAMETER MISSING FROM CONFIG FILE!!') - check_of_sera_disaggregation_to_consider_parameter(config, section_name) - if not config.has_option('SERA_testing_compare_visual_output_vs_OQ_input_files', 'occupancy_case'): - raise IOError('ERROR!! occupancy_case PARAMETER MISSING FROM CONFIG FILE!!') - if not config.has_option('SERA_testing_compare_visual_output_vs_OQ_input_files', 'visual_output_filename'): - raise IOError('ERROR!! visual_output_filename PARAMETER MISSING FROM CONFIG FILE!!') - elif section_name=='SERA_testing_mapping_admin_units_to_cells_qualitycontrol': - if not config.has_option('SERA_testing_mapping_admin_units_to_cells_qualitycontrol', 'country'): - raise IOError('ERROR!! country PARAMETER MISSING FROM CONFIG FILE!!') - if not config.has_option('SERA_testing_mapping_admin_units_to_cells_qualitycontrol', 'tolerance'): - raise IOError('ERROR!! tolerance PARAMETER MISSING FROM CONFIG FILE!!') - else: - try: - float(config[section_name]['tolerance']) - except: - raise IOError('ERROR!! tolerance PARAMETER NEEDS TO BE A REAL NUMBER') - if not config.has_option('SERA_testing_mapping_admin_units_to_cells_qualitycontrol', 'number_of_cell_samples'): - raise IOError('ERROR!! number_of_cell_samples PARAMETER MISSING FROM CONFIG FILE!!') - else: - if not config[section_name]['number_of_cell_samples'].isnumeric(): - raise IOError('ERROR!! number_of_cell_samples NEEDS TO BE A POSITIVE INTEGER!!') - elif section_name=='SERA_testing_rebuilding_exposure_from_cells_alternative_01' or section_name=='SERA_testing_rebuilding_exposure_from_cells_alternative_02' or section_name=='SERA_testing_rebuilding_exposure_from_cells_alternative_03': - if not config.has_option('SERA_testing_rebuilding_exposure_from_cells_alternative_01', 'countries'): - raise IOError('ERROR!! countries PARAMETER MISSING FROM CONFIG FILE!!') - check_of_sera_disaggregation_to_consider_parameter(config, section_name) - _= check_of_occupancy_cases_parameter(config, section_name) - elif section_name=='GDE_check_tiles_vs_visual_CSVs': - _= check_of_occupancy_cases_parameter(config, section_name) - if not config.has_option('GDE_check_tiles_vs_visual_CSVs', 'path_GDE_tiles'): - raise IOError('ERROR!! path_GDE_tiles PARAMETER MISSING FROM CONFIG FILE!!') - if not config.has_option('GDE_check_tiles_vs_visual_CSVs', 'path_visual_csv'): - raise IOError('ERROR!! path_visual_csv PARAMETER MISSING FROM CONFIG FILE!!') - else: - if config['GDE_check_tiles_vs_visual_CSVs']['path_visual_csv'].split('.')[-1]!='csv': - raise IOError('ERROR!! path_visual_csv IN CONFIG FILE IS MEANT TO BE A .csv FILE!!') - if not config.has_option('GDE_check_tiles_vs_visual_CSVs', 'decimal_places_gral'): - raise IOError('ERROR!! decimal_places_gral PARAMETER MISSING FROM CONFIG FILE!!') - else: - if not config[section_name]['decimal_places_gral'].isnumeric(): - raise IOError('ERROR!! decimal_places_gral NEEDS TO BE A POSITIVE INTEGER!!') - if not config.has_option('GDE_check_tiles_vs_visual_CSVs', 'decimal_places_costs'): - raise IOError('ERROR!! decimal_places_costs PARAMETER MISSING FROM CONFIG FILE!!') - else: - if not config[section_name]['decimal_places_costs'].isnumeric(): - raise IOError('ERROR!! decimal_places_costs NEEDS TO BE A POSITIVE INTEGER!!') - else: - raise IOError('ERROR IN check_parameters(): '+section_name+' NOT RECOGNISED!!') - - + """ + section_name: name of the section in the configuration file (name that goes in []) + """ + if section_name == "File Paths": + kinds_of_paths = [ + "out_path", + "sera_models_path", + "sera_models_OQ_path", + "sera_boundaries_path", + "ghs_path", + "gpw_path", + "sat_path", + "completeness_path", + ] + for path_kind in kinds_of_paths: + if not config.has_option("File Paths", path_kind): + raise IOError("ERROR!! " + path_kind + " PARAMETER MISSING FROM CONFIG FILE!!") + else: + if not os.path.isdir(config["File Paths"][path_kind]): + raise IOError( + "ERROR!! " + + path_kind + + ": THE PATH SPECIFIED IN THE CONFIG FILE DOES NOT EXIST" + ) + if not config.has_option("File Paths", "boundaries_type"): + raise IOError("ERROR!! boundaries_type PARAMETER MISSING FROM CONFIG FILE!!") + elif section_name == "Available Results": + if not config.has_option("Available Results", "results_available_for"): + raise IOError("ERROR!! results_available_for PARAMETER MISSING FROM CONFIG FILE!!") + else: + num_results_cases = len( + config["Available Results"]["results_available_for"].split(", ") + ) + if not config.has_option("Available Results", "result_paths"): + raise IOError("ERROR!! result_paths PARAMETER MISSING FROM CONFIG FILE!!") + else: + num_result_paths = len(config["Available Results"]["result_paths"].split(", ")) + if num_results_cases != num_result_paths: + raise IOError( + "ERROR!! results_available_for AND result_paths SHOULD HAVE THE SAME NUMBER OF ELEMENTS IN THE CONFIG FILE!!" + ) + elif section_name == "OBM Database": + if not config.has_option("OBM Database", "db_obm_name"): + raise IOError("ERROR!! db_obm_name PARAMETER MISSING FROM CONFIG FILE!!") + if not config.has_option("OBM Database", "db_obm_username"): + raise IOError("ERROR!! db_obm_username PARAMETER MISSING FROM CONFIG FILE!!") + if not config.has_option("OBM Database", "db_obm_schema_name"): + raise IOError("ERROR!! db_obm_schema_name PARAMETER MISSING FROM CONFIG FILE!!") + if not config.has_option("OBM Database", "db_obm_table_name"): + raise IOError("ERROR!! db_obm_table_name PARAMETER MISSING FROM CONFIG FILE!!") + elif section_name == "Tiles Database": + if not config.has_option("Tiles Database", "db_tiles_name"): + raise IOError("ERROR!! db_tiles_name PARAMETER MISSING FROM CONFIG FILE!!") + if not config.has_option("Tiles Database", "db_tiles_username"): + raise IOError("ERROR!! db_tiles_username PARAMETER MISSING FROM CONFIG FILE!!") + if not config.has_option("Tiles Database", "db_tiles_schema_name"): + raise IOError("ERROR!! db_tiles_schema_name PARAMETER MISSING FROM CONFIG FILE!!") + if not config.has_option("Tiles Database", "db_tiles_table_name"): + raise IOError("ERROR!! db_tiles_table_name PARAMETER MISSING FROM CONFIG FILE!!") + elif section_name == "Admin Units Database": + if not config.has_option("Admin Units Database", "db_admin_name"): + raise IOError("ERROR!! db_admin_name PARAMETER MISSING FROM CONFIG FILE!!") + if not config.has_option("Admin Units Database", "db_admin_username"): + raise IOError("ERROR!! db_admin_username PARAMETER MISSING FROM CONFIG FILE!!") + if not config.has_option("Admin Units Database", "db_admin_schema_name"): + raise IOError("ERROR!! db_admin_schema_name PARAMETER MISSING FROM CONFIG FILE!!") + if not config.has_option("Admin Units Database", "db_admin_table_name"): + raise IOError("ERROR!! db_admin_table_name PARAMETER MISSING FROM CONFIG FILE!!") + elif section_name == "Cells to Process": + check_of_parameters_to_define_list_of_cell_IDs( + config, section_name + ) # Check parameters related to defining the way in which the list of cells is defined + elif section_name == "Ocuppancy String Groups": + num_occup_cases = check_of_occupancy_cases_parameter(config, section_name) + if config.has_option("Ocuppancy String Groups", "occupancy_classifications"): + # It should have it, it is mandatory + num_classif_subgroups = len( + config["Ocuppancy String Groups"]["occupancy_classifications"].split(", ") + ) + else: + raise IOError("ERROR!! OCCUPANCY CLASSIFICATIONS ARE NEEDED IN THE CONFIG FILE!!") + if config.has_option( + "Ocuppancy String Groups", "occupancy_cases" + ) and config.has_option("Ocuppancy String Groups", "occupancy_classifications"): + if num_occup_cases != num_classif_subgroups: + raise IOError( + "ERROR!! LISTS OF OCCUPANCY STRINGS NEED TO BE PROVIDED IN occupancy_classifications FOR EACH CASE IN occupancy_cases!!" + ) + elif section_name == "OBM_buildings_per_cell": + check_of_sera_disaggregation_to_consider_parameter(config, section_name) + elif section_name == "GDE_gather_SERA_and_OBM": + check_of_sera_disaggregation_to_consider_parameter(config, section_name) + _ = check_of_occupancy_cases_parameter(config, section_name) + if not config.has_option("GDE_gather_SERA_and_OBM", "version_of_SERA_metadata"): + raise IOError( + "ERROR!! version_of_SERA_metadata PARAMETER MISSING FROM CONFIG FILE!!" + ) + if not config.has_option("GDE_gather_SERA_and_OBM", "print_screen_during_run"): + raise IOError( + "ERROR!! print_screen_during_run PARAMETER MISSING FROM CONFIG FILE!!" + ) + else: + check_if_param_is_True_or_False( + config["GDE_gather_SERA_and_OBM"]["print_screen_during_run"], + "print_screen_during_run", + ) + elif section_name == "GDE_plot_maps": + _ = check_of_occupancy_cases_parameter(config, section_name) + if not config.has_option("GDE_plot_maps", "location_var"): + raise IOError("ERROR!! location_var PARAMETER MISSING FROM CONFIG FILE!!") + if not config.has_option("GDE_plot_maps", "parameters"): + raise IOError( + "ERROR!! parameters (e.g., number, num_dwells, night, structural) PARAMETER MISSING FROM CONFIG FILE!!" + ) + elif section_name == "GDE_check_consistency": + if not config.has_option("GDE_check_consistency", "location_var"): + raise IOError("ERROR!! location_var PARAMETER MISSING FROM CONFIG FILE!!") + elif section_name == "OBM_assign_cell_ids_and_adm_ids_to_footprints": + if not config.has_option( + "OBM_assign_cell_ids_and_adm_ids_to_footprints", "method_to_use" + ): + raise IOError("ERROR!! method_to_use PARAMETER MISSING FROM CONFIG FILE!!") + else: + if ( + config["OBM_assign_cell_ids_and_adm_ids_to_footprints"]["method_to_use"] + != "just_centroid" + ): + raise IOError( + "ERROR!! method_to_use: THE ONLY CURRENTLY SUPPORTED VALUE IS just_centroid. PLEASE CHECK CONFIG FILE!!" + ) + if not config.has_option( + "OBM_assign_cell_ids_and_adm_ids_to_footprints", "obm_ids_to_process" + ): + raise IOError("ERROR!! obm_ids_to_process PARAMETER MISSING FROM CONFIG FILE!!") + else: + # To do: check that obm_ids_to_process is either "None" or a list of OBM IDs + pass + if not config.has_option( + "OBM_assign_cell_ids_and_adm_ids_to_footprints", "assign_cell_IDs" + ): + raise IOError("ERROR!! assign_cell_IDs PARAMETER MISSING FROM CONFIG FILE!!") + else: + check_if_param_is_True_or_False( + config["OBM_assign_cell_ids_and_adm_ids_to_footprints"]["assign_cell_IDs"], + "assign_cell_IDs", + ) + if not config.has_option( + "OBM_assign_cell_ids_and_adm_ids_to_footprints", "assign_admin_IDs" + ): + raise IOError("ERROR!! assign_admin_IDs PARAMETER MISSING FROM CONFIG FILE!!") + else: + check_if_param_is_True_or_False( + config["OBM_assign_cell_ids_and_adm_ids_to_footprints"]["assign_admin_IDs"], + "assign_admin_IDs", + ) + if not config.has_option( + "OBM_assign_cell_ids_and_adm_ids_to_footprints", "occupancy_case" + ): + raise IOError("ERROR!! occupancy_case PARAMETER MISSING FROM CONFIG FILE!!") + if not config.has_option("OBM_assign_cell_ids_and_adm_ids_to_footprints", "adm_level"): + raise IOError("ERROR!! adm_level PARAMETER MISSING FROM CONFIG FILE!!") + elif section_name == "SERA_create_HDF5_metadata": + if not config.has_option("SERA_create_HDF5_metadata", "version_of_SERA_metadata"): + raise IOError( + "ERROR!! version_of_SERA_metadata PARAMETER MISSING FROM CONFIG FILE!!" + ) + elif section_name == "SERA_create_OQ_input_files": + if not config.has_option("SERA_create_OQ_input_files", "version_of_SERA_metadata"): + raise IOError( + "ERROR!! version_of_SERA_metadata PARAMETER MISSING FROM CONFIG FILE!!" + ) + check_of_sera_disaggregation_to_consider_parameter(config, section_name) + _ = check_of_occupancy_cases_parameter(config, section_name) + if not config.has_option("SERA_create_OQ_input_files", "countries"): + raise IOError("ERROR!! countries PARAMETER MISSING FROM CONFIG FILE!!") + elif section_name == "SERA_create_outputs_QGIS_for_checking": + if not config.has_option("SERA_create_outputs_QGIS_for_checking", "country"): + raise IOError("ERROR!! country PARAMETER MISSING FROM CONFIG FILE!!") + elif section_name == "SERA_create_visual_output_of_grid_model_full_files": + check_of_sera_disaggregation_to_consider_parameter(config, section_name) + _ = check_of_occupancy_cases_parameter(config, section_name) + if not config.has_option( + "SERA_create_visual_output_of_grid_model_full_files", "max_num_cells_per_stage" + ): + raise IOError( + "ERROR!! max_num_cells_per_stage PARAMETER MISSING FROM CONFIG FILE!!" + ) + else: + if not config[section_name]["max_num_cells_per_stage"].isnumeric(): + raise IOError( + "ERROR!! max_num_cells_per_stage NEEDS TO BE A POSITIVE INTEGER!!" + ) + elif section_name == "SERA_distributing_exposure_to_cells": + check_of_sera_disaggregation_to_consider_parameter(config, section_name) + if not config.has_option("SERA_distributing_exposure_to_cells", "countries"): + raise IOError("ERROR!! countries PARAMETER MISSING FROM CONFIG FILE!!") + if not config.has_option( + "SERA_distributing_exposure_to_cells", "columns_to_distribute" + ): + raise IOError("ERROR!! columns_to_distribute PARAMETER MISSING FROM CONFIG FILE!!") + if not config.has_option( + "SERA_distributing_exposure_to_cells", "write_hdf5_bdg_classes_param" + ): + raise IOError( + "ERROR!! write_hdf5_bdg_classes_param PARAMETER MISSING FROM CONFIG FILE!!" + ) + else: + check_if_param_is_True_or_False( + config["SERA_distributing_exposure_to_cells"]["write_hdf5_bdg_classes_param"], + "write_hdf5_bdg_classes_param", + ) + elif section_name == "SERA_exploration_investigate_full_CSV_files": + if not config.has_option( + "SERA_exploration_investigate_full_CSV_files", "tolerance_ratio" + ): + raise IOError("ERROR!! tolerance_ratio PARAMETER MISSING FROM CONFIG FILE!!") + else: + try: + float(config[section_name]["tolerance_ratio"]) + except: + raise IOError("ERROR!! tolerance_ratio PARAMETER NEEDS TO BE A REAL NUMBER") + if not config.has_option( + "SERA_exploration_investigate_full_CSV_files", "tolerance_diff" + ): + raise IOError("ERROR!! tolerance_diff PARAMETER MISSING FROM CONFIG FILE!!") + else: + try: + float(config[section_name]["tolerance_diff"]) + except: + raise IOError("ERROR!! tolerance_diff PARAMETER NEEDS TO BE A REAL NUMBER") + elif section_name == "SERA_exploration_investigate_macro_taxonomies": + if not config.has_option("SERA_exploration_investigate_macro_taxonomies", "country"): + raise IOError("ERROR!! country PARAMETER MISSING FROM CONFIG FILE!!") + _ = check_of_occupancy_cases_parameter(config, section_name) + elif section_name == "SERA_exploration_summarise_columns_in_full_CSV_files": + _ = check_of_occupancy_cases_parameter(config, section_name) + elif section_name == "SERA_mapping_admin_units_to_cells": + if not config.has_option("SERA_mapping_admin_units_to_cells", "countries"): + raise IOError("ERROR!! countries PARAMETER MISSING FROM CONFIG FILE!!") + elif section_name == "SERA_mapping_admin_units_to_cells_add_GHS": + if not config.has_option( + "SERA_mapping_admin_units_to_cells_add_GHS", "ghs_input_filename" + ): + raise IOError("ERROR!! ghs_input_filename PARAMETER MISSING FROM CONFIG FILE!!") + elif section_name == "SERA_mapping_admin_units_to_cells_add_Sat": + if not config.has_option("SERA_mapping_admin_units_to_cells_add_Sat", "apply_model"): + raise IOError("ERROR!! apply_model PARAMETER MISSING FROM CONFIG FILE!!") + else: + check_if_param_is_True_or_False( + config["SERA_mapping_admin_units_to_cells_add_Sat"]["apply_model"], + "apply_model", + ) + elif section_name == "SERA_testing_compare_visual_output_vs_OQ_input_files": + if not config.has_option( + "SERA_testing_compare_visual_output_vs_OQ_input_files", "country" + ): + raise IOError("ERROR!! country PARAMETER MISSING FROM CONFIG FILE!!") + check_of_sera_disaggregation_to_consider_parameter(config, section_name) + if not config.has_option( + "SERA_testing_compare_visual_output_vs_OQ_input_files", "occupancy_case" + ): + raise IOError("ERROR!! occupancy_case PARAMETER MISSING FROM CONFIG FILE!!") + if not config.has_option( + "SERA_testing_compare_visual_output_vs_OQ_input_files", "visual_output_filename" + ): + raise IOError("ERROR!! visual_output_filename PARAMETER MISSING FROM CONFIG FILE!!") + elif section_name == "SERA_testing_mapping_admin_units_to_cells_qualitycontrol": + if not config.has_option( + "SERA_testing_mapping_admin_units_to_cells_qualitycontrol", "country" + ): + raise IOError("ERROR!! country PARAMETER MISSING FROM CONFIG FILE!!") + if not config.has_option( + "SERA_testing_mapping_admin_units_to_cells_qualitycontrol", "tolerance" + ): + raise IOError("ERROR!! tolerance PARAMETER MISSING FROM CONFIG FILE!!") + else: + try: + float(config[section_name]["tolerance"]) + except: + raise IOError("ERROR!! tolerance PARAMETER NEEDS TO BE A REAL NUMBER") + if not config.has_option( + "SERA_testing_mapping_admin_units_to_cells_qualitycontrol", "number_of_cell_samples" + ): + raise IOError("ERROR!! number_of_cell_samples PARAMETER MISSING FROM CONFIG FILE!!") + else: + if not config[section_name]["number_of_cell_samples"].isnumeric(): + raise IOError("ERROR!! number_of_cell_samples NEEDS TO BE A POSITIVE INTEGER!!") + elif ( + section_name == "SERA_testing_rebuilding_exposure_from_cells_alternative_01" + or section_name == "SERA_testing_rebuilding_exposure_from_cells_alternative_02" + or section_name == "SERA_testing_rebuilding_exposure_from_cells_alternative_03" + ): + if not config.has_option( + "SERA_testing_rebuilding_exposure_from_cells_alternative_01", "countries" + ): + raise IOError("ERROR!! countries PARAMETER MISSING FROM CONFIG FILE!!") + check_of_sera_disaggregation_to_consider_parameter(config, section_name) + _ = check_of_occupancy_cases_parameter(config, section_name) + elif section_name == "GDE_check_tiles_vs_visual_CSVs": + _ = check_of_occupancy_cases_parameter(config, section_name) + if not config.has_option("GDE_check_tiles_vs_visual_CSVs", "path_GDE_tiles"): + raise IOError("ERROR!! path_GDE_tiles PARAMETER MISSING FROM CONFIG FILE!!") + if not config.has_option("GDE_check_tiles_vs_visual_CSVs", "path_visual_csv"): + raise IOError("ERROR!! path_visual_csv PARAMETER MISSING FROM CONFIG FILE!!") + else: + if ( + config["GDE_check_tiles_vs_visual_CSVs"]["path_visual_csv"].split(".")[-1] + != "csv" + ): + raise IOError( + "ERROR!! path_visual_csv IN CONFIG FILE IS MEANT TO BE A .csv FILE!!" + ) + if not config.has_option("GDE_check_tiles_vs_visual_CSVs", "decimal_places_gral"): + raise IOError("ERROR!! decimal_places_gral PARAMETER MISSING FROM CONFIG FILE!!") + else: + if not config[section_name]["decimal_places_gral"].isnumeric(): + raise IOError("ERROR!! decimal_places_gral NEEDS TO BE A POSITIVE INTEGER!!") + if not config.has_option("GDE_check_tiles_vs_visual_CSVs", "decimal_places_costs"): + raise IOError("ERROR!! decimal_places_costs PARAMETER MISSING FROM CONFIG FILE!!") + else: + if not config[section_name]["decimal_places_costs"].isnumeric(): + raise IOError("ERROR!! decimal_places_costs NEEDS TO BE A POSITIVE INTEGER!!") + elif section_name == "SERA_creating_industrial_cells": + if not config.has_option("SERA_creating_industrial_cells", "countries"): + raise IOError("ERROR!! countries PARAMETER MISSING FROM CONFIG FILE!!") + if not config.has_option("SERA_creating_industrial_cells", "col_lon"): + raise IOError("ERROR!! col_lon PARAMETER MISSING FROM CONFIG FILE!!") + if not config.has_option("SERA_creating_industrial_cells", "col_lat"): + raise IOError("ERROR!! col_lat PARAMETER MISSING FROM CONFIG FILE!!") + if not config.has_option("SERA_creating_industrial_cells", "width_EW"): + raise IOError("ERROR!! width_EW PARAMETER MISSING FROM CONFIG FILE!!") + else: + if not config[section_name]["width_EW"].isnumeric(): + raise IOError("ERROR!! width_EW NEEDS TO BE A POSITIVE INTEGER!!") + if not config.has_option("SERA_creating_industrial_cells", "width_NS"): + raise IOError("ERROR!! width_NS PARAMETER MISSING FROM CONFIG FILE!!") + else: + if not config[section_name]["width_NS"].isnumeric(): + raise IOError("ERROR!! width_NS NEEDS TO BE A POSITIVE INTEGER!!") + if not config.has_option("SERA_creating_industrial_cells", "precision_points"): + raise IOError("ERROR!! precision_points PARAMETER MISSING FROM CONFIG FILE!!") + else: + if not config[section_name]["precision_points"].isnumeric(): + raise IOError("ERROR!! precision_points NEEDS TO BE A POSITIVE INTEGER!!") + if not config.has_option("SERA_creating_industrial_cells", "id_str"): + raise IOError("ERROR!! id_str PARAMETER MISSING FROM CONFIG FILE!!") + if not config.has_option("SERA_creating_industrial_cells", "export_type"): + raise IOError("ERROR!! export_type PARAMETER MISSING FROM CONFIG FILE!!") + else: + if (config[section_name]["export_type"].lower() != "shp") and ( + config[section_name]["export_type"].lower() != "gpkg" + ): + raise IOError("ERROR!! export_type PARAMETER NOT RECOGNISED!!") + if not config.has_option("SERA_creating_industrial_cells", "in_crs"): + raise IOError("ERROR!! in_crs PARAMETER MISSING FROM CONFIG FILE!!") + if not config.has_option("SERA_creating_industrial_cells", "consistency_checks"): + raise IOError("ERROR!! consistency_checks PARAMETER MISSING FROM CONFIG FILE!!") + else: + check_if_param_is_True_or_False( + config["SERA_creating_industrial_cells"]["consistency_checks"], + "consistency_checks", + ) + if not config.has_option("SERA_creating_industrial_cells", "autoadjust"): + raise IOError("ERROR!! autoadjust PARAMETER MISSING FROM CONFIG FILE!!") + else: + check_if_param_is_True_or_False( + config["SERA_creating_industrial_cells"]["autoadjust"], "autoadjust" + ) + if not config.has_option("SERA_creating_industrial_cells", "verbose"): + raise IOError("ERROR!! verbose PARAMETER MISSING FROM CONFIG FILE!!") + else: + check_if_param_is_True_or_False( + config["SERA_creating_industrial_cells"]["verbose"], "verbose" + ) + if not config.has_option("SERA_creating_industrial_cells", "consistency_tol_dist"): + raise IOError("ERROR!! consistency_tol_dist PARAMETER MISSING FROM CONFIG FILE!!") + else: + try: + val = float(config[section_name]["consistency_tol_dist"]) + if val <= 0.0: + raise IOError( + "ERROR!! consistency_tol_dist PARAMETER NEEDS TO BE A POSITIVE REAL NUMBER" + ) + except: + raise IOError( + "ERROR!! consistency_tol_dist PARAMETER NEEDS TO BE A POSITIVE REAL NUMBER" + ) + if not config.has_option("SERA_creating_industrial_cells", "consistency_tol_area"): + raise IOError("ERROR!! consistency_tol_area PARAMETER MISSING FROM CONFIG FILE!!") + else: + try: + val = float(config[section_name]["consistency_tol_area"]) + if val <= 0.0: + raise IOError( + "ERROR!! consistency_tol_area PARAMETER NEEDS TO BE A POSITIVE REAL NUMBER" + ) + except: + raise IOError( + "ERROR!! consistency_tol_area PARAMETER NEEDS TO BE A POSITIVE REAL NUMBER" + ) + else: + raise IOError("ERROR IN check_parameters(): " + section_name + " NOT RECOGNISED!!") + + def check_if_param_is_True_or_False(parameter_string, parameter_name): - if parameter_string.lower()!='true' and parameter_string.lower()!='false': - raise IOError('ERROR!! '+parameter_name+' PARAMETER SHOULD BE True OR False') - - + if parameter_string.lower() != "true" and parameter_string.lower() != "false": + raise IOError("ERROR!! " + parameter_name + " PARAMETER SHOULD BE True OR False") + + def check_of_parameters_to_define_list_of_cell_IDs(config, section_name): - """ - This function checks that the config file contains under section_name the - parameters needed to define the list of cell IDs to process. - The supported methods are: - - country_in_DB: the IDs of the pieces of cells for which country= country_name in the database will be selected - - bbox: the cells within a bounding box will be selected as per bb_sw_lon, bb_sw_lat, bb_ne_lon and bb_ne_lat - - arbitrary: manually list the IDs of cells as per arbitr_list - - random_from_country: the code selects a number num_random_cells of random cells from country_name (useful for testing the code quickly) - """ - if config.has_option(section_name, 'method'): - # It should have it, it is mandatory - if not((config[section_name]['method']=='arbitrary') or - (config[section_name]['method']=='country_in_DB') or - (config[section_name]['method']=='bbox') or - (config[section_name]['method']=='random_from_country') or - (config[section_name]['method']=='admin_ID_in_country')): - raise IOError('ERROR!! METHOD FOR THE DEFINITION OF THE LIST OF CELLS TO ANALYSE NOT RECOGNISE. REVISE THE CONFIGURATION FILE.') - if config[section_name]['method']=='arbitrary': - if not config.has_option(section_name, 'arbitr_list'): - raise IOError('ERORR! arbitr_list NEEDED IN CONFIG FILE WHEN method IS arbitrary') - if (config[section_name]['method']=='country_in_DB') or (config[section_name]['method']=='random_from_country') or (config[section_name]['method']=='admin_ID_in_country'): - if not config.has_option(section_name, 'country_name'): - raise IOError('ERORR! country_name NEEDED IN CONFIG FILE WHEN method IS country_in_DB OR random_from_country OR admin_id_in_country') - if config[section_name]['method']=='admin_ID_in_country': - if not config.has_option(section_name, 'admin_id_in_country'): - raise IOError('ERORR! admin_id_in_country NEEDED IN CONFIG FILE WHEN method IS admin_id_in_country') - if config[section_name]['method']=='random_from_country': - if not config.has_option(section_name, 'num_random_cells'): - raise IOError('ERORR! num_random_cells NEEDED IN CONFIG FILE WHEN method IS random_from_country') - else: - # TO DO: Create check to make sure that num_random_cells is an integer - pass - if config[section_name]['method']=='bbox': - if not (config.has_option(section_name, 'bb_sw_lon') and config.has_option(section_name, 'bb_sw_lat') and config.has_option(section_name, 'bb_ne_lon') and config.has_option(section_name, 'bb_ne_lat')) : - raise IOError('ERORR! ONE OR MORE OF THE FOLLOWING PARAMETERS ARE NEEDED WHEN method IS bbox: bb_sw_lon, bb_sw_lat, bb_ne_lon, bb_ne_lat') - else: - # TO DO: Create check to make sure they are all floats and within the limits of latitudes and longitudes - pass - else: - raise IOError('ERROR!! METHOD TO DEFINE LIST OF CELLS IS NEEDED IN THE CONFIG FILE!!') + """ + This function checks that the config file contains under section_name the + parameters needed to define the list of cell IDs to process. + The supported methods are: + - country_in_DB: the IDs of the pieces of cells for which country= country_name in the database will be selected + - bbox: the cells within a bounding box will be selected as per bb_sw_lon, bb_sw_lat, bb_ne_lon and bb_ne_lat + - arbitrary: manually list the IDs of cells as per arbitr_list + - random_from_country: the code selects a number num_random_cells of random cells from country_name (useful for testing the code quickly) + """ + if config.has_option(section_name, "method"): + # It should have it, it is mandatory + if not ( + (config[section_name]["method"] == "arbitrary") + or (config[section_name]["method"] == "country_in_DB") + or (config[section_name]["method"] == "bbox") + or (config[section_name]["method"] == "random_from_country") + or (config[section_name]["method"] == "admin_ID_in_country") + ): + raise IOError( + "ERROR!! METHOD FOR THE DEFINITION OF THE LIST OF CELLS TO ANALYSE NOT RECOGNISE. REVISE THE CONFIGURATION FILE." + ) + if config[section_name]["method"] == "arbitrary": + if not config.has_option(section_name, "arbitr_list"): + raise IOError( + "ERORR! arbitr_list NEEDED IN CONFIG FILE WHEN method IS arbitrary" + ) + if ( + (config[section_name]["method"] == "country_in_DB") + or (config[section_name]["method"] == "random_from_country") + or (config[section_name]["method"] == "admin_ID_in_country") + ): + if not config.has_option(section_name, "country_name"): + raise IOError( + "ERORR! country_name NEEDED IN CONFIG FILE WHEN method IS country_in_DB OR random_from_country OR admin_id_in_country" + ) + if config[section_name]["method"] == "admin_ID_in_country": + if not config.has_option(section_name, "admin_id_in_country"): + raise IOError( + "ERORR! admin_id_in_country NEEDED IN CONFIG FILE WHEN method IS admin_id_in_country" + ) + if config[section_name]["method"] == "random_from_country": + if not config.has_option(section_name, "num_random_cells"): + raise IOError( + "ERORR! num_random_cells NEEDED IN CONFIG FILE WHEN method IS random_from_country" + ) + else: + # TO DO: Create check to make sure that num_random_cells is an integer + pass + if config[section_name]["method"] == "bbox": + if not ( + config.has_option(section_name, "bb_sw_lon") + and config.has_option(section_name, "bb_sw_lat") + and config.has_option(section_name, "bb_ne_lon") + and config.has_option(section_name, "bb_ne_lat") + ): + raise IOError( + "ERORR! ONE OR MORE OF THE FOLLOWING PARAMETERS ARE NEEDED WHEN method IS bbox: bb_sw_lon, bb_sw_lat, bb_ne_lon, bb_ne_lat" + ) + else: + # TO DO: Create check to make sure they are all floats and within the limits of latitudes and longitudes + pass + else: + raise IOError("ERROR!! METHOD TO DEFINE LIST OF CELLS IS NEEDED IN THE CONFIG FILE!!") def check_of_sera_disaggregation_to_consider_parameter(config, section_name): - """ - This function checks that the config file contains under section_name - the appropriate entry for sera_disaggregation_to_consider. - """ - if config.has_option(section_name, 'sera_disaggregation_to_consider'): - # It should have it, it is mandatory. Options are: area, gpw_2015_pop, ghs, sat_27f and sat_27f_model - if not((config[section_name]['sera_disaggregation_to_consider']=='area') or - (config[section_name]['sera_disaggregation_to_consider']=='gpw_2015_pop') or - (config[section_name]['sera_disaggregation_to_consider']=='ghs') or - (config[section_name]['sera_disaggregation_to_consider']=='sat_27f') or - (config[section_name]['sera_disaggregation_to_consider']=='sat_27f_model')): - raise IOError('ERROR!! THE VALUE OF sera_disaggregation_to_consider IN THE CONFIG FILE IS NOT RECOGNISED.') - else: - raise IOError('ERROR!! SERA DISAGGREGATION METHOD TO CONSIDER IS NEEDED IN THE CONFIG FILE!!') + """ + This function checks that the config file contains under section_name + the appropriate entry for sera_disaggregation_to_consider. + """ + if config.has_option(section_name, "sera_disaggregation_to_consider"): + # It should have it, it is mandatory. Options are: area, gpw_2015_pop, ghs, sat_27f and sat_27f_model + if not ( + (config[section_name]["sera_disaggregation_to_consider"] == "area") + or (config[section_name]["sera_disaggregation_to_consider"] == "gpw_2015_pop") + or (config[section_name]["sera_disaggregation_to_consider"] == "ghs") + or (config[section_name]["sera_disaggregation_to_consider"] == "sat_27f") + or (config[section_name]["sera_disaggregation_to_consider"] == "sat_27f_model") + ): + raise IOError( + "ERROR!! THE VALUE OF sera_disaggregation_to_consider IN THE CONFIG FILE IS NOT RECOGNISED." + ) + else: + raise IOError( + "ERROR!! SERA DISAGGREGATION METHOD TO CONSIDER IS NEEDED IN THE CONFIG FILE!!" + ) + - def check_of_occupancy_cases_parameter(config, section_name): - """ - This function checks that the config file contains under section_name - the appropriate entry for occupancy_cases. - """ - if config.has_option(section_name, 'occupancy_cases'): - # It should have it, it is mandatory - which_occupancy_cases= config[section_name]['occupancy_cases'].split(', ') - if 'Oth' in which_occupancy_cases: - which_occupancy_cases.remove('Oth') - num_occup_cases= len(which_occupancy_cases) - else: - raise IOError('ERROR!! OCCUPANCY CASES ARE NEEDED IN THE CONFIG FILE!!') - return num_occup_cases - \ No newline at end of file + """ + This function checks that the config file contains under section_name + the appropriate entry for occupancy_cases. + """ + if config.has_option(section_name, "occupancy_cases"): + # It should have it, it is mandatory + which_occupancy_cases = config[section_name]["occupancy_cases"].split(", ") + if "Oth" in which_occupancy_cases: + which_occupancy_cases.remove("Oth") + num_occup_cases = len(which_occupancy_cases) + else: + raise IOError("ERROR!! OCCUPANCY CASES ARE NEEDED IN THE CONFIG FILE!!") + return num_occup_cases diff --git a/GDE_config_file_TEMPLATE.ini b/GDE_config_file_TEMPLATE.ini index bb5a4c2182b75afcd5f9e24ba8bcca95f7bb0d39..5cfa3e0e5c60634fd814c9aaa66801eb2125f385 100644 --- a/GDE_config_file_TEMPLATE.ini +++ b/GDE_config_file_TEMPLATE.ini @@ -23,6 +23,8 @@ sera_models_path = WRITE_PATH sera_models_OQ_path = WRITE_PATH # Path to the shapefiles of admin units used in SERA: sera_boundaries_path = WRITE_PATH +# File type containing the boundaries ("shp"=Shapefile, "gpkg"=Geopackage): +boundaries_type = shp # Path to the GHS layer: ghs_path = WRITE_PATH # Path to the GPW data: @@ -274,3 +276,36 @@ occupancy_cases = Res, Com, Ind, Oth decimal_places_gral = 4 decimal_places_costs = 0 +[SERA_creating_industrial_cells] +# Countries to process (if many, separate with comma and space): +countries = Albania, Austria, Belgium, Bosnia_and_Herzegovina, Bulgaria, Croatia, Cyprus, Czechia, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Iceland, Ireland, Italy, Latvia, Lithuania, Luxembourg, Malta, Moldova, Montenegro, Netherlands, North_Macedonia, Norway, Poland, Portugal, Romania, Serbia, Slovakia, Slovenia, Spain, Sweden, Switzerland, United_Kingdom +# Names of the columns in the SERA model that contain longitudes and latitudes: +col_lon = LONGITUDE +col_lat = LATITUDE +# Width of the cell in the East-West direction, in arcseconds, >0: +width_EW = 30 +# Width of the cell in the North-south direction, in arcseconds, >0: +width_NS = 30 +# First part of the string used to generate IDs of the inidividual points (e.g. "IND"). +# Do not include country's ISO2 code: +id_str = IND +# Number of decimal places to be used to determine unique points present in the aggregated +# exposure model (4 is a reasonable value): +precision_points = 4 +# Run consistency checks (True) or not (False): +consistency_checks = True +# Autoadjust potential leftover overlaps and gaps (True) or not (False): +autoadjust = True +# Print statemens of progress while running: +verbose = False +# CRS of the SERA files: +in_crs = EPSG:4326 +# Tolerance to assess how large the maximum distance between the original points and the +# centroids of the generated cells is with respect to the width of the cells. Only needed +# if consistency_checks is True (e.g. 0.05 implies a 5% of the width as tolerance): +consistency_tol_dist = 0.05 +# Tolerance to assess how large the variability of the area of the generated cells is. Only +# needed if consistency_checks is True: +consistency_tol_area = 0.05 +# File type to export the created cells to ("shp"=Shapefile, "gpkg"=Geopackage): +export_type = shp diff --git a/SERA_creating_industrial_cells.py b/SERA_creating_industrial_cells.py new file mode 100644 index 0000000000000000000000000000000000000000..c1c960a425a2b081c1693272dfab901796eeda9c --- /dev/null +++ b/SERA_creating_industrial_cells.py @@ -0,0 +1,245 @@ +""" +Copyright (C) 2021 + Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + + +Global Dynamic Exposure Model +Helmholtz Centre Potsdam +GFZ German Research Centre for Geosciences +Section 2.6: Seismic Hazard and Risk Dynamics + +SERA_creating_industrial_cells +============================== +This code is used to handle the fact that industrial exposure may be defined in 30-arcsec cells +instead of administrative units in the SERA exposure model. The SERA model only provides the +centroids of those cells with a certain decimal precision. This code makes use of the tools +defined in GDE_TOOLS_create_industrial_cells.py, which create cells around the points given as +input and export these cells to a geodata file. The country/ies to process is defined in the +configuration file. The cells' North-South and East-West widths are given as input too. +""" + +import sys +import os +import GDE_TOOLS_create_industrial_cells as gdet_ind +import GDE_TOOLS_general as gdet_gral +import GDE_TOOLS_read_config_file as gdet_conf +import datetime + + +def run(config_dict): + #################################################### + # READ CONFIGURATION PARAMETERS + #################################################### + print("Processing configuration parameters...") + # Path for output: + out_path = config_dict["File Paths"]["out_path"] + # SERA models path: + sera_models_path = config_dict["File Paths"]["sera_models_path"] + # SERA boundaries shapefiles path: + sera_shp_path = config_dict["File Paths"]["sera_boundaries_path"] + bound_type = config_dict["File Paths"]["boundaries_type"] + # Countries to process: + countries = config_dict["SERA_creating_industrial_cells"]["countries"].split(", ") + # Name of columns with longitude and latitude in the SERA model: + col_lon = config_dict["SERA_creating_industrial_cells"]["col_lon"] + col_lat = config_dict["SERA_creating_industrial_cells"]["col_lat"] + # Widths of the cells to be generated: + width_EW = float(config_dict["SERA_creating_industrial_cells"]["width_EW"]) + width_NS = float(config_dict["SERA_creating_industrial_cells"]["width_NS"]) + # Input is given in arcseconds, transform to degrees: + width_EW = width_EW / (60.0 * 60.0) + width_NS = width_NS / (60.0 * 60.0) + # First part of the string used to generate IDs of the individual points (e.g. "IND"): + id_str = config_dict["SERA_creating_industrial_cells"]["id_str"] + # Number of decimal places to be used to determine unique points in the aggregated model: + precision_points = int(config_dict["SERA_creating_industrial_cells"]["precision_points"]) + # Run consistency checks (True) or not (False): + consistency_checks_str = config_dict["SERA_creating_industrial_cells"]["consistency_checks"] + if consistency_checks_str.lower() == "true": + consist_checks = True + elif consistency_checks_str.lower() == "false": + consist_checks = False + else: + print( + "ERROR!! IT SHOULD NOT GET TO THIS POINT, " + "AS THE PARAMETER IS CHECKED IN THE CONFIG FILE CHECKS" + ) + # Autoadjust potential leftover overlaps and gaps (True) or not (False): + autoadjust_str = config_dict["SERA_creating_industrial_cells"]["autoadjust"] + if autoadjust_str.lower() == "true": + autoadjust = True + elif autoadjust_str.lower() == "false": + autoadjust = False + else: + print( + "ERROR!! IT SHOULD NOT GET TO THIS POINT, " + "AS THE PARAMETER IS CHECKED IN THE CONFIG FILE CHECKS" + ) + # Print statemens of progress while running: + verbose_str = config_dict["SERA_creating_industrial_cells"]["verbose"] + if verbose_str.lower() == "true": + verbose = True + elif verbose_str.lower() == "false": + verbose = False + else: + print( + "ERROR!! IT SHOULD NOT GET TO THIS POINT, " + "AS THE PARAMETER IS CHECKED IN THE CONFIG FILE CHECKS" + ) + # Tolerances for the consistency checks: + if consist_checks: + # Tolerance for maximum distance between the original points and the centroids of the + # generated cells: + tol_dist = float(config_dict["SERA_creating_industrial_cells"]["consistency_tol_dist"]) + # Tolerance for variability of the area of the generated cells: + tol_area = float(config_dict["SERA_creating_industrial_cells"]["consistency_tol_area"]) + else: # Dummy values, they will not be used + tol_dist = 0.05 + tol_area = 0.05 + # CRS of the SERA files: + in_crs = config_dict["SERA_creating_industrial_cells"]["in_crs"] + # File type to export the created cells to (e.g. "shp"=Shapefile, "gpkg"=Geopackage): + export_type = config_dict["SERA_creating_industrial_cells"]["export_type"] + #################################################### + # START + #################################################### + # Auxiliary dictionary definining the precision to use to convert coordinates into strings: + dec_precision_EW = int("{:E}".format(width_EW).split("-")[1]) + dec_precision_NS = int("{:E}".format(width_NS).split("-")[1]) + precision_cells = {} + precision_cells["lon_w"] = "{:.%sf}" % (dec_precision_EW) + precision_cells["lat_s"] = "{:.%sf}" % (dec_precision_NS) + precision_cells["lon_e"] = "{:.%sf}" % (dec_precision_EW) + precision_cells["lat_n"] = "{:.%sf}" % (dec_precision_NS) + # Run country by country + if consist_checks: + log = ["country,overlap_found,gap_found,big_distance_found,big_area_variability_found"] + processing_times = ["country,processing_time"] + for k, country_name in enumerate(countries): + start_time = datetime.datetime.now() + print("\n") + print( + "Working on %s, country %s of %s\n" + % (country_name, str(k + 1), str(len(countries))) + ) + + # Generate cells for this country: + country_iso2 = gdet_gral.get_ISO2_code_for_country(country_name) + id_str_country = "%s_%s" % (country_iso2, id_str) + results = gdet_ind.generate_country_industrial_cells( + country_name, + col_lon, + col_lat, + width_EW, + width_NS, + id_str_country, + precision_points, + precision_cells, + sera_models_path, + sera_shp_path, + boundaries_type=bound_type, + consistency_checks=consist_checks, + autoadjust_overlap_gap=autoadjust, + in_crs=in_crs, + consistency_tol_dist=tol_dist, + consistency_tol_area=tol_area, + verbose=verbose, + ) + ( + cells, + aggr_model, + overlap_found, + gap_found, + big_dist_diff, + big_area_diff, + country_id, + ) = results + + # Print to screen and a log file the results of the consistency check if requested: + if consist_checks: + print("\n") + print(" Results for consistency checks:") + if overlap_found == "True": + overlap_str = "Problem found with cells overlapping. ERROR." + else: + overlap_str = "Cells do not overlap. OK." + if gap_found == "True": + gap_str = "Problem found with gaps in between cells. ERROR." + else: + gap_str = "Cells do not have gaps in between. OK." + if big_dist_diff == "True": + big_dist_str = "Distances between centroids and points too large. ERROR." + else: + big_dist_str = "Distances between centroids and points are OK." + if big_area_diff == "True": + big_area_str = "Variability of resulting cell areas too large. ERROR." + else: + big_area_str = "Variability of resulting cell areas is OK." + print(" %s" % (overlap_str)) + print(" %s" % (gap_str)) + print(" %s" % (big_dist_str)) + print(" %s" % (big_area_str)) + out_str = [country_name, overlap_found, gap_found, big_dist_diff, big_area_diff] + log.append(",".join(out_str)) + + # Export SERA aggregated exposure model with additional columns referring to cells: + if verbose: + print("\n") + print(" Updating aggregated exposure file...") + aggr_filename = "Exposure_Ind_%s.csv" % (country_name) + gdet_ind.export_modified_aggregated_model_file( + aggr_model, os.path.join(out_path, "Ind_Cells"), aggr_filename, separ="," + ) + + # Export geodata file of cells: + if verbose: + print("\n") + print(" Exporting cells geometry to file...") + cells_filename = "Adm99_%s.%s" % (country_name, export_type) + gdet_ind.export_cells_to_geodatafile( + country_name, country_id, cells, os.path.join(out_path, "Ind_Cells"), cells_filename + ) + + # Time it took to process this country: + end_time = datetime.datetime.now() + duration = (end_time - start_time).total_seconds() + processing_times.append("%s,%s" % (country_name, str(duration))) + + # Write output log of consistency checks: + if consist_checks: + gdet_gral.write_log_file( + log, os.path.join(out_path, "Ind_Cells", "log_consistency_checks.csv") + ) + + # Write output log of processing times: + gdet_gral.write_log_file( + processing_times, os.path.join(out_path, "Ind_Cells", "log_processing_times.csv") + ) + + print("\n") + print("Done!") + + +if __name__ == "__main__": + # This code needs to be run from the command line as python3 namefile.py configfile.ini. + # sys.argv retrieves all the commands entered in the command line; position [0] is this + # code, position [1] will be the config file name. + config_filename = sys.argv[1] + section_names_to_validate = ["File Paths", os.path.basename(__file__).split(".")[0]] + config_dict = gdet_conf.read_config_parameters( + os.path.join(os.getcwd(), config_filename), section_names_to_validate + ) + run(config_dict) diff --git a/docs/02_Execution.md b/docs/02_Execution.md index dd670dd74aaf7ea8b4d0a786c1f69ae3fdf3c324..05e58dd79d19b022545b7c76d1347f345a415984 100644 --- a/docs/02_Execution.md +++ b/docs/02_Execution.md @@ -14,17 +14,18 @@ The scripts are run from the command line as: The order in which the scripts in the present repository need to be run to produce the GDE model for a region of interest is: -1. Run `OBM_assign_cell_ids_and_adm_ids_to_footprints.py` -2. Run `SERA_create_HDF5_metadata.py`. -3. Run `SERA_mapping_admin_units_to_cells.py` -4. Run `SERA_mapping_admin_units_to_cells_add_GHS.py` (if GHS criterion desired) -5. Run `SERA_mapping_admin_units_to_cells_add_GPW.py` (if GPW criterion desired) -6. Run `SERA_mapping_admin_units_to_cells_add_Sat.py` (if Sat or Sat_mod criterion desired) -7. Run `SERA_distributing_exposure_to_cells.py` with the desired distribution method. -8. If the OpenQuake input files for the SERA model distributed onto a grid are desired (i.e. not GDE, just SERA), run `SERA_create_OQ_input_files.py` with the desired distribution method. -9. If a CSV summarising the number of buildings, dwellings, people and costs by cell according to the SERA model is desired (i.e. not GDE, just SERA), run `SERA_create_visual_output_of_grid_model_full_files.py` with the desired distribution method. -10. Run `OBM_buildings_per_cell.py` with the desired distribution method. -11. Run `GDE_gather_SERA_and_OBM.py` with the desired distribution method. The output is: +1. If the country/ies of interest have their industrial exposure defined on a 30-arcsec grid, run `SERA_creating_industrial_cells.py` +2. Run `OBM_assign_cell_ids_and_adm_ids_to_footprints.py` +3. Run `SERA_create_HDF5_metadata.py` +4. Run `SERA_mapping_admin_units_to_cells.py` +5. Run `SERA_mapping_admin_units_to_cells_add_GHS.py` (if GHS criterion desired) +6. Run `SERA_mapping_admin_units_to_cells_add_GPW.py` (if GPW criterion desired) +7. Run `SERA_mapping_admin_units_to_cells_add_Sat.py` (if Sat or Sat_mod criterion desired) +8. Run `SERA_distributing_exposure_to_cells.py` with the desired distribution method. +9. If the OpenQuake input files for the SERA model distributed onto a grid are desired (i.e. not GDE, just SERA), run `SERA_create_OQ_input_files.py` with the desired distribution method. +10. If a CSV summarising the number of buildings, dwellings, people and costs by cell according to the SERA model is desired (i.e. not GDE, just SERA), run `SERA_create_visual_output_of_grid_model_full_files.py` with the desired distribution method. +11. Run `OBM_buildings_per_cell.py` with the desired distribution method. +12. Run `GDE_gather_SERA_and_OBM.py` with the desired distribution method. The output is: - a series of CSV files that serve as input for damage/risk calculations to be run in OpenQuake (https://github.com/gem/oq-engine); - a CSV file that summarises results per cell and contains the geometry of the cells so that it can all be visualised with a GIS; - a CSV file that summarises results per adminstrative unit and contains the geometry of the administrative boundaries so that it can all be visualised with a GIS; @@ -32,19 +33,19 @@ The order in which the scripts in the present repository need to be run to produ ## Testing Scripts -- The scripts `SERA_testing_rebuilding_exposure_from_cells_alternative_01.py`, `SERA_testing_rebuilding_exposure_from_cells_alternative_02.py` and `SERA_testing_rebuilding_exposure_from_cells_alternative_03.py` can be run after step 7 above. They compare the SERA-on-a-grid model against the original files of the SERA model. +- The scripts `SERA_testing_rebuilding_exposure_from_cells_alternative_01.py`, `SERA_testing_rebuilding_exposure_from_cells_alternative_02.py` and `SERA_testing_rebuilding_exposure_from_cells_alternative_03.py` can be run after step 8 above. They compare the SERA-on-a-grid model against the original files of the SERA model. -- The script `SERA_testing_compare_visual_output_vs_OQ_input_files.py` can be run after step 9 above to compare the number of buildings, people and cost per cell reported in the OpenQuake input file (generated from the grid) and the visual output CSV. +- The script `SERA_testing_compare_visual_output_vs_OQ_input_files.py` can be run after step 10 above to compare the number of buildings, people and cost per cell reported in the OpenQuake input file (generated from the grid) and the visual output CSV. -- The script `SERA_create_outputs_QGIS_for_checking.py` can be run after step 6 above to create a summary of the parameters mapped (GHS, GPW, Sat, etc) in CSV format to be read with QGIS, enabling a visual check of the results. +- The script `SERA_create_outputs_QGIS_for_checking.py` can be run after step 7 above to create a summary of the parameters mapped (GHS, GPW, Sat, etc) in CSV format to be read with QGIS, enabling a visual check of the results. -- The script `SERA_testing_mapping_admin_units_to_cells_qualitycontrol.py` can be run after step 3 above to check the areas of the cells mapped for the administrative units for which step 3 was run. +- The script `SERA_testing_mapping_admin_units_to_cells_qualitycontrol.py` can be run after step 4 above to check the areas of the cells mapped for the administrative units for which step 3 was run. -- The script `GDE_check_consistency.py` can be run after step 11 above. It carries out different consistency checks on the resulting GDE model (see detailed description of this script). +- The script `GDE_check_consistency.py` can be run after step 12 above. It carries out different consistency checks on the resulting GDE model (see detailed description of this script). -- The script `GDE_check_OQ_input_files.py` can be run after step 11 above. It prints to screen some summary values of the files and checks that the asset ID values are all unique. +- The script `GDE_check_OQ_input_files.py` can be run after step 12 above. It prints to screen some summary values of the files and checks that the asset ID values are all unique. -- The script `GDE_check_tiles_vs_visual_CSVs.py` can be run after step 11 above. It reads the visual CSV output by cell and the corresponding GDE tiles HDF5 files and compares the number of buildings, cost and number of people in each cell according to each of the two. An output CSV file collects the discrepancies found, if any. +- The script `GDE_check_tiles_vs_visual_CSVs.py` can be run after step 12 above. It reads the visual CSV output by cell and the corresponding GDE tiles HDF5 files and compares the number of buildings, cost and number of people in each cell according to each of the two. An output CSV file collects the discrepancies found, if any. ## Other Scripts diff --git a/docs/04_Core_Scripts.md b/docs/04_Core_Scripts.md index c56c25610695ec9f2c20e5e0406c6e8087844d37..b464356ad562ace739f235775afb06f698c06570 100644 --- a/docs/04_Core_Scripts.md +++ b/docs/04_Core_Scripts.md @@ -3,6 +3,30 @@ For each core script, the enumerated configurable parameters are those that are specific to that script, i.e. defined in the configuration file under a subtitle that matches the name of the file. General parameters are not explained herein but in `03_Config_File.md` and `GDE_config_file_TEMPLATE.ini`. +# SERA_creating_industrial_cells.py + +## Configurable parameters: + +The parameters that need to be specified under the `SERA_creating_industrial_cells` section of the configuration file are: + +- countries = Countries to process. If more than one, separate with comma and space. +- col_lon, col_lat = Names of the columns in the SERA model that contain longitudes and latitudes. +- width_EW, width_NS = Widths (arcseconds) of the cells in which the industrial exposure is defined, in the east-west and north-south directions, respectively. +- id_str= First part of the string used to generate IDs of the inidividual points (e.g. "IND"). Do not include the country's ISO2 code (it gets added by the script automatically). +- precision_points = Number of decimal places to be used to determine unique points present in the input aggregated exposure model. +- consistency_checks = True or False (run consistency checks or not). +- autoadjust = After a first adjustment of the cells' geometries to fix any overlaps and/or gaps, a check is carried out if consistency_checks is True to determine if there are any potential leftover overlaps and/or gaps. If autoadjust is True, the script will adjust the geometry again until no further overlaps/gaps are found. If False, the script will not carry out this further adjustment. +- verbose = If True, a series of print statemens of progress are executed while running. +- in_crs = Coordinate reference system of the input and output. +- consistency_tol_dist = Tolerance to assess how large the maximum distance between the original points and the centroids of the generated cells is with respect to the width of the cells. Only needed if consistency_checks is True (e.g. 0.05 implies a 5% of the width as tolerance). +- consistency_tol_area = Tolerance to assess how large the variability of the area of the generated cells is. Only needed if consistency_checks is True. +- export_type = File type to export the created cells to ("shp"=Shapefile, "gpkg"=Geopackage). + +## What the code does: + +This code is used to handle the fact that industrial exposure may be defined in 30-arcsec cells instead of administrative units in the SERA exposure model. The SERA model only provides the centroids of those cells with a certain decimal precision. This code makes use of the tools defined in `GDE_TOOLS_create_industrial_cells.py`, which create cells around the points given as input and export these cells to a geodata file, adjusting the geometries so as to avoid overlaps and gaps between neighbouring cells. The country/ies to process is/are defined in the configuration file. Details on the algorithms used can be found in `08_Industrial_Cells.md.md`. + + # OBM_assign_cell_ids_and_adm_ids_to_footprints.py ## Configurable parameters: @@ -123,7 +147,7 @@ The parameters that need to be specified under the `SERA_distributing_exposure_t - sera_disaggregation_to_consider = area, gpw_2015_pop, ghs, sat_27f or sat_27f_model. Select the parameter to use to distribute the SERA model to the grid. - ignore_occupancy_cases = Res, Com, Ind or leave empty. The code reads the occupancy cases from the SERA models and only ignores them if specified here. - countries = Countries to process. If more than one, separate with comma and space. -- admin_ids_to_ignore = 1110101. Within those countries, do not process admin units specified under this parameter. This is useful for running parts of countries only, it can be empty or ignored too. +- admin_ids_to_ignore = Within those countries, do not process admin units specified under this parameter. This is useful for running parts of countries only, it can be empty or ignored too. - columns_to_distribute = buildings, dwell_per_bdg, area_per_dwelling_sqm, cost_per_area_usd, ppl_per_dwell - write_hdf5_bdg_classes_param = if True, write the HDF5 file of building classes parameters; if False, do not (`write_hdf5_bdg_classes parameter` for `gdet_sera.distribute_SERA_to_cells()` function). The building classes parameters depend only on the SERA model, not on the distribution method, so they do not need to be re-written each time. Setting this parameter to False saves a lot of processing time. @@ -170,7 +194,7 @@ The parameters that need to be specified under the `SERA_create_OQ_input_files` - sera_disaggregation_to_consider = area, gpw_2015_pop, ghs, sat_27f or sat_27f_model. Select the parameter to use to distribute the SERA model to the grid. - occupancy_cases = Res, Com, Ind. Occupancy cases to process. - countries = Countries to process. If more than one, separate with comma and space. -- admin_ids_to_ignore = 1110101. Within those countries, do not process admin units specified under this parameter. This is useful for running parts of countries only, it can be empty or ignored too. +- admin_ids_to_ignore = Within those countries, do not process admin units specified under this parameter. This is useful for running parts of countries only, it can be empty or ignored too. ## What the code does: diff --git a/docs/08_Industrial_Cells.md b/docs/08_Industrial_Cells.md new file mode 100644 index 0000000000000000000000000000000000000000..4efff78afc8c62a9de662acf29fbc80e67a7ccbf --- /dev/null +++ b/docs/08_Industrial_Cells.md @@ -0,0 +1,54 @@ +# SERA Industrial Exposure Defined in 30-arcsec Cells: The Tools in GDE_TOOLS_create_industrial_cells.py + +In broad terms, `GDE_TOOLS_create_industrial_cells.py` contains tools to: + +- read the points in which the SERA industrial exposure model is defined + + + +- generate 30-arcsec cell geometries around them + + + +- carry out a first adjustment of the coordinates of the cells based on grouping together all longitudes and then all latitudes that "should be" the same, as per a precision criterion + + + + + + + +- check that the resulting adjusted geometries are satisfactory, that is, that no potential overlaps/gaps between cells remain + + + +- carry out more computationally-demanding operations to adjust the geometries when overlaps/gaps are found +- intersect the adjusted cell geometries with the country boundaries +- update the SERA industrial exposure input files to indicate the corresponding IDs of the generated cells +- export the geometries of the generated cells + +The core function of the tools, which is called by `SERA_creating_industrial_cells.py` is `generate_country_industrial_cells()`. The following figures illustrate the functions that it calls and the process as a whole. + +After the input points are retrieved from the SERA input files, unique points are identified according to a certain input precision. The coordinates (longitude, latitude) that "should be" the same are identified by rounding the coordinates according to a certain number of decimal places and identifying unique values at that precision level. This is done using strings and dictionaries that store the original and new coordinates. The cells dataframe is finally updated to reflect the new adjusted coordinates and geometries. + + + + + +Steps indicated in purple and magenta in the figures that follow are only executed if the input parameters `consistency_checks` and `autoadjust_overlap_gap` are True, respectively. These steps assess whether there are any overlaps/gaps left in between neighbouring cells and adjust the geometries when this is the case. In the case of gaps, geometries are not adjusted if the cells are in diagonal with respect to one another and they participate of other cases of intersection, as this can lead to contradictory adjustments of the geometries in each step because one pair of cells that intersect each other are adjusted at a time. + +Cells are only trimmed as per country boundaries after the consistency checks. Otherwise, checks that look at the distance between the resulting cell centroids and the original points and the final aras of the cells would be meaningless, because the trimmed geometry cannot guarantee such consistencies with the original input points. + + + +Gaps are identified by generating an enlarged version of the cells (i.e. increasing their dimensions), searching for subsequent intersections (so as to know which cells are neighbours of which other cells), and then subtracting the original cell geometries to these intersections. This resulting geometry is analysed to decide whether a gap exists between the cells or not. + + + + + +Which coordinates need to be adjusted is determined by first identifying the relative position of one cell with respect to the other. + + + + diff --git a/docs/Images/industrial_30arcsec_approach_01.png b/docs/Images/industrial_30arcsec_approach_01.png new file mode 100644 index 0000000000000000000000000000000000000000..2ec41b25e4e5974cdd612542139b17dfbec48047 Binary files /dev/null and b/docs/Images/industrial_30arcsec_approach_01.png differ diff --git a/docs/Images/industrial_30arcsec_approach_02.png b/docs/Images/industrial_30arcsec_approach_02.png new file mode 100644 index 0000000000000000000000000000000000000000..6e24a6d72c6735eb51da33e65fe91bc33f1e5748 Binary files /dev/null and b/docs/Images/industrial_30arcsec_approach_02.png differ diff --git a/docs/Images/industrial_30arcsec_approach_03.png b/docs/Images/industrial_30arcsec_approach_03.png new file mode 100644 index 0000000000000000000000000000000000000000..cd10b5badb476422846d3455bda65c8fb1b11c71 Binary files /dev/null and b/docs/Images/industrial_30arcsec_approach_03.png differ diff --git a/docs/Images/industrial_30arcsec_approach_04.png b/docs/Images/industrial_30arcsec_approach_04.png new file mode 100644 index 0000000000000000000000000000000000000000..a154eca85dbd7f2e8de4b4ac9987ad2e6213817f Binary files /dev/null and b/docs/Images/industrial_30arcsec_approach_04.png differ diff --git a/docs/Images/industrial_30arcsec_approach_05.png b/docs/Images/industrial_30arcsec_approach_05.png new file mode 100644 index 0000000000000000000000000000000000000000..275b3ee7ab1dbc33c7bd3df6aca68555d2a795be Binary files /dev/null and b/docs/Images/industrial_30arcsec_approach_05.png differ diff --git a/docs/Images/industrial_30arcsec_approach_06.png b/docs/Images/industrial_30arcsec_approach_06.png new file mode 100644 index 0000000000000000000000000000000000000000..bb36cf669e4085acb37fb97476a974e02bea41ac Binary files /dev/null and b/docs/Images/industrial_30arcsec_approach_06.png differ diff --git a/docs/Images/industrial_30arcsec_approach_07.png b/docs/Images/industrial_30arcsec_approach_07.png new file mode 100644 index 0000000000000000000000000000000000000000..ef2322736339f7b16b7a0d6579592bb959758ce9 Binary files /dev/null and b/docs/Images/industrial_30arcsec_approach_07.png differ diff --git a/docs/Images/industrial_30arcsec_approach_08.png b/docs/Images/industrial_30arcsec_approach_08.png new file mode 100644 index 0000000000000000000000000000000000000000..7398a1e7cab1f92f4d113be1fa097b31bab44ed9 Binary files /dev/null and b/docs/Images/industrial_30arcsec_approach_08.png differ diff --git a/docs/Images/industrial_30arcsec_approach_09.png b/docs/Images/industrial_30arcsec_approach_09.png new file mode 100644 index 0000000000000000000000000000000000000000..c58db9ad67f1d5b1f9ab4d23327f3f73f8df6668 Binary files /dev/null and b/docs/Images/industrial_30arcsec_approach_09.png differ diff --git a/docs/Images/industrial_30arcsec_approach_10.png b/docs/Images/industrial_30arcsec_approach_10.png new file mode 100644 index 0000000000000000000000000000000000000000..4c25a6ce8ba6163ae2a0fa0ff30a251a59d24e7d Binary files /dev/null and b/docs/Images/industrial_30arcsec_approach_10.png differ diff --git a/docs/Images/industrial_30arcsec_approach_11.png b/docs/Images/industrial_30arcsec_approach_11.png new file mode 100644 index 0000000000000000000000000000000000000000..8fdfebda6346a9d202aeeb0b3babfad7db933f7d Binary files /dev/null and b/docs/Images/industrial_30arcsec_approach_11.png differ diff --git a/docs/Images/industrial_30arcsec_approach_12.png b/docs/Images/industrial_30arcsec_approach_12.png new file mode 100644 index 0000000000000000000000000000000000000000..1a7fe09674507d9d642ed580ce46104b9d685dcc Binary files /dev/null and b/docs/Images/industrial_30arcsec_approach_12.png differ diff --git a/docs/Images/industrial_30arcsec_approach_13.png b/docs/Images/industrial_30arcsec_approach_13.png new file mode 100644 index 0000000000000000000000000000000000000000..bf980278a2a54534a37be18aab641033794ce876 Binary files /dev/null and b/docs/Images/industrial_30arcsec_approach_13.png differ