Commit 1ea80a79 authored by Cecilia Nievas's avatar Cecilia Nievas
Browse files

Added functionality to read completeness from OBM Tiles database

parent 70249739
......@@ -710,4 +710,113 @@ def get_ghs_built_up_area_from_obm_tiles_opened_connection(grid_cell_id, open_cu
return ghs_built_up
def get_completeness_from_obm_tiles(grid_cell_id, db_hostname, db_name,
db_username, db_table):
"""This function retrieves the completeness value from the OBM tiles database for a
specified zoom level 18 quadkey.
Note that a password is required to access the database. If not provided as input, this
function will prompt the user to indicate it.
Args:
grid_cell_id (str): Quadkey of the zoom level 18 target tile.
db_hostname (str): Name of the server where the database is located.
db_name (str): Name of the database.
db_username (str): Username for access to the database.
db_table (str): Name of the table where this information is stored.
Returns:
completeness (int): Completeness value in the target tile. The possible values are:
0: unknown
1: complete
2: almost complete
3: incomplete
4: undecidable
5: water
6: empty
"""
# Connect to the database:
connection = psycopg2.connect('host=%s dbname=%s user=%s password=%s'
% (db_hostname, db_name, db_username, getpass.getpass()))
connection.set_session(autocommit=True)
# Open a cursor to perform database operations
cursor = connection.cursor()
# Query the database:
sql_query = "SELECT completeness FROM %s WHERE quadkey='%s';" \
% (db_table, grid_cell_id)
cursor.execute(sql_query)
exec_result = cursor.fetchall()
connection.close()
if len(exec_result) == 0: # If the quadkey is not found, assume completeness unknown
completeness = 0
elif len(exec_result) == 1:
completeness = exec_result[0][0]
else: # More than one entries found, this is an error
print("ERROR IN get_completeness_from_obm_tiles: "
"MORE THAN ONE ENTRY FOUND FOR CELL ID %s"
% (grid_cell_id))
completeness = -999.9
return completeness
def get_completeness_from_obm_tiles_for_several(grid_cell_ids, db_hostname, db_name,
db_username, db_table):
"""This function retrieves the completeness value from the OBM tiles database for a
specified zoom level 18 quadkey.
Note that a password is required to access the database. If not provided as input, this
function will prompt the user to indicate it.
Args:
grid_cell_ids (list of str): List of quadkeys of the zoom level 18 target tiles.
db_hostname (str): Name of the server where the database is located.
db_name (str): Name of the database.
db_username (str): Username for access to the database.
db_table (str): Name of the table where this information is stored.
Returns:
completeness (list of int): List of completeness values of the target tiles. The
possible values are:
0: unknown
1: complete
2: almost complete
3: incomplete
4: undecidable
5: water
6: empty
"""
# Connect to the database:
connection = psycopg2.connect('host=%s dbname=%s user=%s password=%s'
% (db_hostname, db_name, db_username, getpass.getpass()))
connection.set_session(autocommit=True)
# Open a cursor to perform database operations
cursor = connection.cursor()
completeness = []
for grid_cell_id in grid_cell_ids:
# Query the database:
sql_query = "SELECT completeness FROM %s WHERE quadkey='%s';" \
% (db_table, grid_cell_id)
cursor.execute(sql_query)
exec_result = cursor.fetchall()
if len(exec_result) == 0: # If the quadkey is not found, assume completeness unknown
completeness.append(0)
elif len(exec_result) == 1:
completeness.append(exec_result[0][0])
else: # More than one entries found, this is an error
print("ERROR IN get_completeness_from_obm_tiles: "
"MORE THAN ONE ENTRY FOUND FOR CELL ID %s"
% (grid_cell_id))
completeness.append(-999.9)
connection.close()
return completeness
......@@ -149,6 +149,10 @@ def check_parameters(config, section_name):
raise IOError(
"ERROR!! version_of_SERA_metadata PARAMETER MISSING FROM CONFIG FILE!!"
)
if not config.has_option("GDE_gather_SERA_and_OBM", "read_completeness"):
raise IOError(
"ERROR!! read_completeness PARAMETER MISSING FROM CONFIG FILE!!"
)
if not config.has_option("GDE_gather_SERA_and_OBM", "print_screen_during_run"):
raise IOError(
"ERROR!! print_screen_during_run PARAMETER MISSING FROM CONFIG FILE!!"
......@@ -296,25 +300,25 @@ def check_parameters(config, section_name):
"SERA_mapping_admin_units_to_cells_add_GHS_from_HDF5", "ghs_input_filename"
):
raise IOError("ERROR!! ghs_input_filename PARAMETER MISSING FROM CONFIG FILE!!")
elif section_name == "SERA_mapping_admin_units_to_cells_add_GHS_from_OBM_tiles":
elif section_name == "OBM tiles":
if not config.has_option(
"SERA_mapping_admin_units_to_cells_add_GHS_from_OBM_tiles", "obm_tiles_db_hostname"
"OBM tiles", "obm_tiles_db_hostname"
):
raise IOError("ERROR!! obm_tiles_db_hostname PARAMETER MISSING FROM CONFIG FILE!!")
if not config.has_option(
"SERA_mapping_admin_units_to_cells_add_GHS_from_OBM_tiles", "obm_tiles_db_name"
"OBM tiles", "obm_tiles_db_name"
):
raise IOError("ERROR!! obm_tiles_db_name PARAMETER MISSING FROM CONFIG FILE!!")
if not config.has_option(
"SERA_mapping_admin_units_to_cells_add_GHS_from_OBM_tiles", "obm_tiles_db_username"
"OBM tiles", "obm_tiles_db_username"
):
raise IOError("ERROR!! obm_tiles_db_username PARAMETER MISSING FROM CONFIG FILE!!")
if not config.has_option(
"SERA_mapping_admin_units_to_cells_add_GHS_from_OBM_tiles", "obm_tiles_db_table"
"OBM tiles", "obm_tiles_db_table"
):
raise IOError("ERROR!! obm_tiles_db_table PARAMETER MISSING FROM CONFIG FILE!!")
if not config.has_option(
"SERA_mapping_admin_units_to_cells_add_GHS_from_OBM_tiles", "obm_tiles_source_id"
"OBM tiles", "obm_tiles_source_built_up_id"
):
raise IOError("ERROR!! obm_tiles_source_id PARAMETER MISSING FROM CONFIG FILE!!")
elif section_name == "SERA_mapping_admin_units_to_cells_add_Sat":
......
......@@ -63,6 +63,15 @@ db_admin_username = DB_USERNAME
db_admin_schema_name = DB_SCHEMA_NAME
db_admin_table_name = DB_TABLE_NAME
[OBM tiles]
# See https://git.gfz-potsdam.de/dynamicexposure/server-components/deployments/databases#obm-tiles
obm_tiles_db_hostname = SERVER_NAME
obm_tiles_db_name = DB_NAME
obm_tiles_db_username = DB_USERNAME
obm_tiles_db_table = DB_TABLE_NAME
# Source ID for built-up area to retrieve:
obm_tiles_source_built_up_id = 1
[Cells to Process]
# PARAMETER TO DECIDE HOW THE LIST OF CELLS IS DEFINED
# Options are:
......@@ -102,6 +111,8 @@ version_of_SERA_metadata = YYYY_MM_DD
# SELECT CASE OF SERA DISAGGREGATION TO CONSIDER:
# Options are: area, gpw_2015_pop, ghs, sat_27f and sat_27f_model
sera_disaggregation_to_consider = sat_27f
# Read completeness from CSV or OBM tiles (options: "csv", "obm_tiles"):
read_completeness = obm_tiles
# Print messages while running the code (for debugging purposes):
print_screen_during_run = False
# Occupancy cases in which the OBM buildings will be classified ('Oth' is implicit in the method):
......@@ -202,13 +213,6 @@ occupancy_cases = Res, Com, Ind
# Countries to process (if many, separate with comma and space):
countries = Greece
[SERA_mapping_admin_units_to_cells_add_GHS_from_OBM_tiles]
obm_tiles_db_hostname = nozomi.gfz-potsdam.de
obm_tiles_db_name = obm_tiles
obm_tiles_db_username = consumer
obm_tiles_db_table = obm_built_area_assessments
obm_tiles_source_id = 1
[SERA_mapping_admin_units_to_cells_add_GHS_from_HDF5]
ghs_input_filename = FILENAME.hdf5
......
......@@ -65,6 +65,11 @@ def run_this_file(config_dict):
DB_name_grid= config_dict['Tiles Database']['db_tiles_name']
DB_schema_name_grid= config_dict['Tiles Database']['db_tiles_schema_name']
DB_table_name_grid= config_dict['Tiles Database']['db_tiles_table_name']
# Database from which the completeness values are read:
obm_tiles_db_hostname = config_dict['OBM tiles']['obm_tiles_db_hostname']
obm_tiles_db_name = config_dict['OBM tiles']['obm_tiles_db_name']
obm_tiles_db_username = config_dict['OBM tiles']['obm_tiles_db_username']
obm_tiles_db_table = config_dict['OBM tiles']['obm_tiles_db_table']
# Case of SERA disaggregation to consider (['area', 'gpw_2015_pop', 'ghs', 'sat_27f', 'sat_27f_model']):
sera_disaggregation_to_consider= config_dict['GDE_gather_SERA_and_OBM']['sera_disaggregation_to_consider']
# Select whether to print to screen intermediate values or not (useful for debugging):
......@@ -88,6 +93,8 @@ def run_this_file(config_dict):
min_grid_cell_id= np.min(grid_cells_list) - 1
# Version of SERA metadata to use:
version_of_SERA_metadata= config_dict['GDE_gather_SERA_and_OBM']['version_of_SERA_metadata']
# Read completeness from CSV or OBM tiles (options: "csv", "obm_tiles"):
read_completeness= config_dict['GDE_gather_SERA_and_OBM']['read_completeness']
####################################################
# START
####################################################
......@@ -107,7 +114,12 @@ def run_this_file(config_dict):
else:
print('ERROR!!! METHOD FOR DEFINING LIST OF CELL IDs UNKNOWN. ERROR!!!')
print('Retrieving completeness information for grid cells...')
compl_cells= gdet_gral.retrieve_completeness_for_cells_from_CSV(grid_cells_list, completeness_filepath) # array with completeness values (integers)
if read_completeness == "csv":
compl_cells= gdet_gral.retrieve_completeness_for_cells_from_CSV(grid_cells_list, completeness_filepath) # array with completeness values (integers)
elif read_completeness == "obm_tiles":
compl_cells= gdet_gral.get_completeness_from_obm_tiles_for_several(grid_cells_list, obm_tiles_db_hostname, obm_tiles_db_name, obm_tiles_db_username, obm_tiles_db_table)
else:
print("ERROR: 'read_completeness' VALUE IN CONFIGURATION FILE COULD NOT BE RECOGNISED!!!")
# To keep track of rows for writing the OpenQuake input files:
next_row_id= [0 for case in occupancy_cases]
# Initialise admin_summary_dict:
......@@ -174,15 +186,16 @@ def run_this_file(config_dict):
cell_summary_dict= gdet_gral.fill_in_cell_summary_dictionary_with_SERA_LeftOver_or_Total_bdgs_dwells_ppl(cell_summary_dict, case, 'SERA', sera_num_bdgs, sera_num_dwells, sera_num_ppl, sera_cost)
admin_summary_dict= gdet_gral.fill_in_admin_summary_dictionary_with_SERA_LeftOver_or_Total_bdgs_dwells_ppl(admin_summary_dict, case, 'SERA', sera_country_adminIDs, sera_country_adminIDs_levels, sera_num_bdgs_per_class_and_adm_unit, sera_num_dwells_per_class_and_adm_unit, sera_num_ppl_per_class_and_adm_unit, sera_cost_per_class_and_adm_unit, sera_shp_path, os.path.join(out_path, 'Europe_SERA_metadata_v_'+version_of_SERA_metadata+'.hdf5'), input_kind='per_class_and_adm_unit')
obm_with_classes_num_bdgs_per_class_and_adm_unit, obm_with_classes_num_dwells_per_class_and_adm_unit, obm_with_classes_num_ppl_per_class_and_adm_unit, obm_with_classes_cost_per_class_and_adm_unit, obm_list_bdg_classes_of_arrays, obm_list_country_adminIDs_of_arrays= gdet_gral.disaggregate_OBM_by_class_and_adm_unit_main(obm_dict, sera_country_adminIDs, sera_classes)
if compl_cells[i]==5 and len(obm_dict.keys())>0: # 5:'Irrelevant (Water)'
error_message= ' ERROR: completeness of cell ID '+str(grid_cell_id)+' is 5 but OBM buildings have been found --> TREATING AS INCOMPLETE CELL NOW.'
print('\n')
print(' '+ error_message)
log.append(error_message)
print('\n')
compl_cells[i]==3
cell_summary_dict['Completeness']= 999
if compl_cells[i]==1: # cell is complete, no additional OQ input files need to be written
for compl_value in [5, 6]: # 5: 'water', 6: 'empty'
if compl_cells[i] == compl_value and len(obm_dict.keys()) > 0: # 5:'Irrelevant (Water)'
error_message= ' ERROR: completeness of cell ID %s is %s but OBM buildings have been found --> TREATING AS INCOMPLETE CELL NOW.' % (str(grid_cell_id), str(compl_value))
print('\n')
print(' '+ error_message)
log.append(error_message)
print('\n')
compl_cells[i]==3
cell_summary_dict['Completeness']= 999
if compl_cells[i] == 1 or compl_cells[i] == 5 or compl_cells[i] == 6: # cell is complete (1) or water (5) or empty (6), no additional OQ input files need to be written
leftover_num_bdgs= 0.0
leftover_num_dwells= 0.0
leftover_num_ppl= 0.0
......
......@@ -50,12 +50,11 @@ def run_this_file(config_dict):
# Path for GHS HDF5 file:
ghs_path = config_dict['File Paths']['ghs_path']
# Database from which the GHS built-up values are read:
self_dict = config_dict['SERA_mapping_admin_units_to_cells_add_GHS_from_OBM_tiles']
obm_tiles_db_hostname = self_dict['obm_tiles_db_hostname']
obm_tiles_db_name = self_dict['obm_tiles_db_name']
obm_tiles_db_username = self_dict['obm_tiles_db_username']
obm_built_area_assessments = self_dict['obm_tiles_db_table']
obm_tiles_source_id = int(self_dict['obm_tiles_source_id'])
obm_tiles_db_hostname = config_dict['OBM tiles']['obm_tiles_db_hostname']
obm_tiles_db_name = config_dict['OBM tiles']['obm_tiles_db_name']
obm_tiles_db_username = config_dict['OBM tiles']['obm_tiles_db_username']
obm_tiles_db_table = config_dict['OBM tiles']['obm_tiles_db_table']
obm_tiles_source_id = int(config_dict['OBM tiles']['obm_tiles_source_built_up_id'])
# Tiles database:
DB_username_grid = config_dict['Tiles Database']['db_tiles_username']
DB_name_grid = config_dict['Tiles Database']['db_tiles_name']
......@@ -118,7 +117,7 @@ def run_this_file(config_dict):
ghs_built_up_i = gdet_psql.get_ghs_built_up_area_from_obm_tiles_opened_connection(
str(grid_cell_id),
cursor,
obm_built_area_assessments,
obm_tiles_db_table,
obm_tiles_source_id)
print(str(grid_cell_id)+": "+str(ghs_built_up_i))
# Write GHS built up area to database (distributed proportionally to the area of
......@@ -160,7 +159,7 @@ if __name__=='__main__':
section_names_to_validate= ['File Paths',
'Tiles Database',
'Cells to Process',
os.path.basename(__file__).split('.')[0]]
'OBM tiles']
config_dict= gdet_conf.read_config_parameters(os.path.join(os.getcwd(),
config_filename),
section_names_to_validate)
......
FILE,Self,File Paths,OBM Database,Tiles Database,Admin Units Database,Ocuppancy String Groups,Available Results,Cells to Process
GDE_check_consistency,Y,Y,Y,N,N,Y,Y,N
GDE_check_OQ_input_files,N,Y,N,N,N,N,Y,N
GDE_gather_SERA_and_OBM,Y,Y,N,Y,N,N,N,Y
GDE_plot_maps,Y,Y,N,N,N,N,Y,N
OBM_assign_cell_ids_and_adm_ids_to_footprints,Y,N,Y,N,Y,N,N,N
OBM_buildings_per_cell,Y,Y,Y,Y,N,Y,N,Y
SERA_create_HDF5_metadata,Y,Y,N,N,N,N,N,N
SERA_create_OQ_input_files,Y,Y,N,Y,N,N,N,N
SERA_create_outputs_QGIS_for_checking,Y,Y,N,Y,N,N,N,N
SERA_create_visual_output_of_grid_model_full_files,Y,Y,N,Y,N,N,N,Y
SERA_distributing_exposure_to_cells,Y,Y,N,Y,N,N,N,N
SERA_exploration_check_Ids_unequivocally_defined,N,Y,N,N,N,N,N,N
SERA_exploration_check_one_point_per_ID,N,Y,N,N,N,N,N,N
SERA_exploration_countries_read,N,Y,N,N,N,N,N,N
SERA_exploration_investigate_full_CSV_files,Y,Y,N,N,N,N,N,N
SERA_exploration_investigate_macro_taxonomies,Y,Y,N,N,N,N,N,N
SERA_exploration_longest_strings,N,Y,N,N,N,N,N,N
SERA_exploration_summarise_columns_in_full_CSV_files,Y,Y,N,N,N,N,N,N
SERA_mapping_admin_units_to_cells,Y,Y,N,Y,N,N,N,N
SERA_mapping_admin_units_to_cells_add_GHS_from_HDF5,Y,Y,N,Y,N,N,N,Y
SERA_mapping_admin_units_to_cells_add_GHS_from_OBM_tiles,Y,Y,N,Y,N,N,N,Y
SERA_mapping_admin_units_to_cells_add_GPW,N,Y,N,Y,N,N,N,Y
SERA_mapping_admin_units_to_cells_add_Sat,Y,Y,N,Y,N,N,N,Y
SERA_testing_compare_visual_output_vs_OQ_input_files,Y,Y,N,N,N,N,N,N
SERA_testing_mapping_admin_units_to_cells_qualitycontrol,Y,Y,N,Y,N,N,N,N
SERA_testing_rebuilding_exposure_from_cells_alternative_01,Y,Y,N,Y,N,N,N,N
SERA_testing_rebuilding_exposure_from_cells_alternative_02,Y,Y,N,N,N,N,N,N
SERA_testing_rebuilding_exposure_from_cells_alternative_03,Y,Y,N,N,N,N,N,N
GDE_check_tiles_vs_visual_CSVs,Y,Y,N,N,N,N,N,N
FILE,Self,File Paths,OBM Database,Tiles Database,Admin Units Database,OBM tiles,Ocuppancy String Groups,Available Results,Cells to Process
GDE_check_consistency,Y,Y,Y,N,N,N,Y,Y,N
GDE_check_OQ_input_files,N,Y,N,N,N,N,N,Y,N
GDE_gather_SERA_and_OBM,Y,Y,N,Y,N,Y,N,N,Y
GDE_plot_maps,Y,Y,N,N,N,N,N,Y,N
OBM_assign_cell_ids_and_adm_ids_to_footprints,Y,N,Y,N,Y,N,N,N,N
OBM_buildings_per_cell,Y,Y,Y,Y,N,N,Y,N,Y
SERA_create_HDF5_metadata,Y,Y,N,N,N,N,N,N,N
SERA_create_OQ_input_files,Y,Y,N,Y,N,N,N,N,N
SERA_create_outputs_QGIS_for_checking,Y,Y,N,Y,N,N,N,N,N
SERA_create_visual_output_of_grid_model_full_files,Y,Y,N,Y,N,N,N,N,Y
SERA_distributing_exposure_to_cells,Y,Y,N,Y,N,N,N,N,N
SERA_exploration_check_Ids_unequivocally_defined,N,Y,N,N,N,N,N,N,N
SERA_exploration_check_one_point_per_ID,N,Y,N,N,N,N,N,N,N
SERA_exploration_countries_read,N,Y,N,N,N,N,N,N,N
SERA_exploration_investigate_full_CSV_files,Y,Y,N,N,N,N,N,N,N
SERA_exploration_investigate_macro_taxonomies,Y,Y,N,N,N,N,N,N,N
SERA_exploration_longest_strings,N,Y,N,N,N,N,N,N,N
SERA_exploration_summarise_columns_in_full_CSV_files,Y,Y,N,N,N,N,N,N,N
SERA_mapping_admin_units_to_cells,Y,Y,N,Y,N,N,N,N,N
SERA_mapping_admin_units_to_cells_add_GHS_from_HDF5,Y,Y,N,Y,N,N,N,N,Y
SERA_mapping_admin_units_to_cells_add_GHS_from_OBM_tiles,N,Y,N,Y,N,Y,N,N,Y
SERA_mapping_admin_units_to_cells_add_GPW,N,Y,N,Y,N,N,N,N,Y
SERA_mapping_admin_units_to_cells_add_Sat,Y,Y,N,Y,N,N,N,N,Y
SERA_testing_compare_visual_output_vs_OQ_input_files,Y,Y,N,N,N,N,N,N,N
SERA_testing_mapping_admin_units_to_cells_qualitycontrol,Y,Y,N,Y,N,N,N,N,N
SERA_testing_rebuilding_exposure_from_cells_alternative_01,Y,Y,N,Y,N,N,N,N,N
SERA_testing_rebuilding_exposure_from_cells_alternative_02,Y,Y,N,N,N,N,N,N,N
SERA_testing_rebuilding_exposure_from_cells_alternative_03,Y,Y,N,N,N,N,N,N,N
GDE_check_tiles_vs_visual_CSVs,Y,Y,N,N,N,N,N,N,N
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment