Added functionality to read completeness from OBM Tiles database

GDE_TOOLS_psql.py

@@ -710,4 +710,113 @@ def get_ghs_built_up_area_from_obm_tiles_opened_connection(grid_cell_id, open_cu
     return ghs_built_up
 
 
+def get_completeness_from_obm_tiles(grid_cell_id, db_hostname, db_name,
+                                    db_username, db_table):
+    """This function retrieves the completeness value from the OBM tiles database for a
+    specified zoom level 18 quadkey.
 
+    Note that a password is required to access the database. If not provided as input, this
+    function will prompt the user to indicate it.
+
+    Args:
+        grid_cell_id (str): Quadkey of the zoom level 18 target tile.
+        db_hostname (str): Name of the server where the database is located.
+        db_name (str): Name of the database.
+        db_username (str): Username for access to the database.
+        db_table (str): Name of the table where this information is stored.
+    
+    Returns:
+        completeness (int): Completeness value in the target tile. The possible values are:
+            0: unknown
+            1: complete
+            2: almost complete
+            3: incomplete
+            4: undecidable
+            5: water
+            6: empty
+    """
+
+    # Connect to the database:
+    connection = psycopg2.connect('host=%s dbname=%s user=%s password=%s'
+                                  % (db_hostname, db_name, db_username, getpass.getpass()))
+    connection.set_session(autocommit=True)
+
+    # Open a cursor to perform database operations
+    cursor = connection.cursor()
+
+    # Query the database:
+    sql_query = "SELECT completeness FROM %s WHERE quadkey='%s';" \
+                % (db_table, grid_cell_id)
+    cursor.execute(sql_query)
+    exec_result = cursor.fetchall()
+    connection.close()
+
+    if len(exec_result) == 0:  # If the quadkey is not found, assume completeness unknown
+        completeness = 0
+    elif len(exec_result) == 1:
+        completeness = exec_result[0][0]
+    else:  # More than one entries found, this is an error
+        print("ERROR IN get_completeness_from_obm_tiles: "
+              "MORE THAN ONE ENTRY FOUND FOR CELL ID %s"
+              % (grid_cell_id))
+        completeness = -999.9
+
+    return completeness
+
+
+def get_completeness_from_obm_tiles_for_several(grid_cell_ids, db_hostname, db_name,
+                                                db_username, db_table):
+    """This function retrieves the completeness value from the OBM tiles database for a
+    specified zoom level 18 quadkey.
+
+    Note that a password is required to access the database. If not provided as input, this
+    function will prompt the user to indicate it.
+
+    Args:
+        grid_cell_ids (list of str): List of quadkeys of the zoom level 18 target tiles.
+        db_hostname (str): Name of the server where the database is located.
+        db_name (str): Name of the database.
+        db_username (str): Username for access to the database.
+        db_table (str): Name of the table where this information is stored.
+    
+    Returns:
+        completeness (list of int): List of completeness values of the target tiles. The
+        possible values are:
+            0: unknown
+            1: complete
+            2: almost complete
+            3: incomplete
+            4: undecidable
+            5: water
+            6: empty
+    """
+
+    # Connect to the database:
+    connection = psycopg2.connect('host=%s dbname=%s user=%s password=%s'
+                                  % (db_hostname, db_name, db_username, getpass.getpass()))
+    connection.set_session(autocommit=True)
+
+    # Open a cursor to perform database operations
+    cursor = connection.cursor()
+    
+    completeness = []
+    for grid_cell_id in grid_cell_ids:
+        # Query the database:
+        sql_query = "SELECT completeness FROM %s WHERE quadkey='%s';" \
+                    % (db_table, grid_cell_id)
+        cursor.execute(sql_query)
+        exec_result = cursor.fetchall()
+
+        if len(exec_result) == 0:  # If the quadkey is not found, assume completeness unknown
+            completeness.append(0)
+        elif len(exec_result) == 1:
+            completeness.append(exec_result[0][0])
+        else:  # More than one entries found, this is an error
+            print("ERROR IN get_completeness_from_obm_tiles: "
+                  "MORE THAN ONE ENTRY FOUND FOR CELL ID %s"
+                  % (grid_cell_id))
+            completeness.append(-999.9)
+        
+    connection.close()
+
+    return completeness

GDE_TOOLS_read_config_file.py

@@ -149,6 +149,10 @@ def check_parameters(config, section_name):
             raise IOError(
                 "ERROR!! version_of_SERA_metadata PARAMETER MISSING FROM CONFIG FILE!!"
             )
+        if not config.has_option("GDE_gather_SERA_and_OBM", "read_completeness"):
+            raise IOError(
+                "ERROR!! read_completeness PARAMETER MISSING FROM CONFIG FILE!!"
+            )
         if not config.has_option("GDE_gather_SERA_and_OBM", "print_screen_during_run"):
             raise IOError(
                 "ERROR!! print_screen_during_run PARAMETER MISSING FROM CONFIG FILE!!"
@@ -296,25 +300,25 @@ def check_parameters(config, section_name):
             "SERA_mapping_admin_units_to_cells_add_GHS_from_HDF5", "ghs_input_filename"
         ):
             raise IOError("ERROR!! ghs_input_filename PARAMETER MISSING FROM CONFIG FILE!!")
-    elif section_name == "SERA_mapping_admin_units_to_cells_add_GHS_from_OBM_tiles":
+    elif section_name == "OBM tiles":
         if not config.has_option(
-            "SERA_mapping_admin_units_to_cells_add_GHS_from_OBM_tiles", "obm_tiles_db_hostname"
+            "OBM tiles", "obm_tiles_db_hostname"
         ):
             raise IOError("ERROR!! obm_tiles_db_hostname PARAMETER MISSING FROM CONFIG FILE!!")
         if not config.has_option(
-            "SERA_mapping_admin_units_to_cells_add_GHS_from_OBM_tiles", "obm_tiles_db_name"
+            "OBM tiles", "obm_tiles_db_name"
         ):
             raise IOError("ERROR!! obm_tiles_db_name PARAMETER MISSING FROM CONFIG FILE!!")
         if not config.has_option(
-            "SERA_mapping_admin_units_to_cells_add_GHS_from_OBM_tiles", "obm_tiles_db_username"
+            "OBM tiles", "obm_tiles_db_username"
         ):
             raise IOError("ERROR!! obm_tiles_db_username PARAMETER MISSING FROM CONFIG FILE!!")
         if not config.has_option(
-            "SERA_mapping_admin_units_to_cells_add_GHS_from_OBM_tiles", "obm_tiles_db_table"
+            "OBM tiles", "obm_tiles_db_table"
         ):
             raise IOError("ERROR!! obm_tiles_db_table PARAMETER MISSING FROM CONFIG FILE!!")
         if not config.has_option(
-            "SERA_mapping_admin_units_to_cells_add_GHS_from_OBM_tiles", "obm_tiles_source_id"
+            "OBM tiles", "obm_tiles_source_built_up_id"
         ):
             raise IOError("ERROR!! obm_tiles_source_id PARAMETER MISSING FROM CONFIG FILE!!")
     elif section_name == "SERA_mapping_admin_units_to_cells_add_Sat":

GDE_config_file_TEMPLATE.ini

@@ -63,6 +63,15 @@ db_admin_username = DB_USERNAME
 db_admin_schema_name = DB_SCHEMA_NAME
 db_admin_table_name = DB_TABLE_NAME
 
+[OBM tiles]
+# See https://git.gfz-potsdam.de/dynamicexposure/server-components/deployments/databases#obm-tiles
+obm_tiles_db_hostname = SERVER_NAME
+obm_tiles_db_name = DB_NAME
+obm_tiles_db_username = DB_USERNAME
+obm_tiles_db_table = DB_TABLE_NAME
+# Source ID for built-up area to retrieve:
+obm_tiles_source_built_up_id = 1
+
 [Cells to Process]
 # PARAMETER TO DECIDE HOW THE LIST OF CELLS IS DEFINED
 # Options are:
@@ -102,6 +111,8 @@ version_of_SERA_metadata = YYYY_MM_DD
 # SELECT CASE OF SERA DISAGGREGATION TO CONSIDER:
 # Options are: area, gpw_2015_pop, ghs, sat_27f and sat_27f_model
 sera_disaggregation_to_consider = sat_27f
+# Read completeness from CSV or OBM tiles (options: "csv", "obm_tiles"):
+read_completeness = obm_tiles
 # Print messages while running the code (for debugging purposes):
 print_screen_during_run = False
 # Occupancy cases in which the OBM buildings will be classified ('Oth' is implicit in the method):
@@ -202,13 +213,6 @@ occupancy_cases = Res, Com, Ind
 # Countries to process (if many, separate with comma and space):
 countries = Greece
 
-[SERA_mapping_admin_units_to_cells_add_GHS_from_OBM_tiles]
-obm_tiles_db_hostname = nozomi.gfz-potsdam.de
-obm_tiles_db_name = obm_tiles
-obm_tiles_db_username = consumer
-obm_tiles_db_table = obm_built_area_assessments
-obm_tiles_source_id = 1
-
 [SERA_mapping_admin_units_to_cells_add_GHS_from_HDF5]
 ghs_input_filename = FILENAME.hdf5
 

GDE_gather_SERA_and_OBM.py

@@ -65,6 +65,11 @@ def run_this_file(config_dict):
     DB_name_grid= config_dict['Tiles Database']['db_tiles_name']
     DB_schema_name_grid= config_dict['Tiles Database']['db_tiles_schema_name']
     DB_table_name_grid= config_dict['Tiles Database']['db_tiles_table_name'] 
+    # Database from which the completeness values are read:
+    obm_tiles_db_hostname = config_dict['OBM tiles']['obm_tiles_db_hostname']
+    obm_tiles_db_name = config_dict['OBM tiles']['obm_tiles_db_name']
+    obm_tiles_db_username = config_dict['OBM tiles']['obm_tiles_db_username']
+    obm_tiles_db_table = config_dict['OBM tiles']['obm_tiles_db_table']
     # Case of SERA disaggregation to consider (['area', 'gpw_2015_pop', 'ghs', 'sat_27f', 'sat_27f_model']):
     sera_disaggregation_to_consider= config_dict['GDE_gather_SERA_and_OBM']['sera_disaggregation_to_consider']
     # Select whether to print to screen intermediate values or not (useful for debugging):
@@ -88,6 +93,8 @@ def run_this_file(config_dict):
         min_grid_cell_id= np.min(grid_cells_list) - 1
     # Version of SERA metadata to use:
     version_of_SERA_metadata= config_dict['GDE_gather_SERA_and_OBM']['version_of_SERA_metadata']
+    # Read completeness from CSV or OBM tiles (options: "csv", "obm_tiles"):
+    read_completeness= config_dict['GDE_gather_SERA_and_OBM']['read_completeness']
     ####################################################
     # START
     ####################################################
@@ -107,7 +114,12 @@ def run_this_file(config_dict):
     else:
         print('ERROR!!! METHOD FOR DEFINING LIST OF CELL IDs UNKNOWN. ERROR!!!')
     print('Retrieving completeness information for grid cells...')
-    compl_cells= gdet_gral.retrieve_completeness_for_cells_from_CSV(grid_cells_list, completeness_filepath) # array with completeness values (integers)
+    if read_completeness == "csv":
+        compl_cells= gdet_gral.retrieve_completeness_for_cells_from_CSV(grid_cells_list, completeness_filepath) # array with completeness values (integers)
+    elif read_completeness == "obm_tiles":
+        compl_cells= gdet_gral.get_completeness_from_obm_tiles_for_several(grid_cells_list, obm_tiles_db_hostname, obm_tiles_db_name, obm_tiles_db_username, obm_tiles_db_table)
+    else:
+        print("ERROR: 'read_completeness' VALUE IN CONFIGURATION FILE COULD NOT BE RECOGNISED!!!")
     # To keep track of rows for writing the OpenQuake input files:
     next_row_id= [0 for case in occupancy_cases]
     # Initialise admin_summary_dict:
@@ -174,15 +186,16 @@ def run_this_file(config_dict):
                 cell_summary_dict= gdet_gral.fill_in_cell_summary_dictionary_with_SERA_LeftOver_or_Total_bdgs_dwells_ppl(cell_summary_dict, case, 'SERA', sera_num_bdgs, sera_num_dwells, sera_num_ppl, sera_cost)
                 admin_summary_dict= gdet_gral.fill_in_admin_summary_dictionary_with_SERA_LeftOver_or_Total_bdgs_dwells_ppl(admin_summary_dict, case, 'SERA', sera_country_adminIDs, sera_country_adminIDs_levels, sera_num_bdgs_per_class_and_adm_unit, sera_num_dwells_per_class_and_adm_unit, sera_num_ppl_per_class_and_adm_unit, sera_cost_per_class_and_adm_unit, sera_shp_path, os.path.join(out_path, 'Europe_SERA_metadata_v_'+version_of_SERA_metadata+'.hdf5'), input_kind='per_class_and_adm_unit')
                 obm_with_classes_num_bdgs_per_class_and_adm_unit, obm_with_classes_num_dwells_per_class_and_adm_unit, obm_with_classes_num_ppl_per_class_and_adm_unit, obm_with_classes_cost_per_class_and_adm_unit, obm_list_bdg_classes_of_arrays, obm_list_country_adminIDs_of_arrays= gdet_gral.disaggregate_OBM_by_class_and_adm_unit_main(obm_dict, sera_country_adminIDs, sera_classes)       
-                if compl_cells[i]==5 and len(obm_dict.keys())>0: # 5:'Irrelevant (Water)'
-                    error_message= '  ERROR: completeness of cell ID '+str(grid_cell_id)+' is 5 but OBM buildings have been found --> TREATING AS INCOMPLETE CELL NOW.'
-                    print('\n')
-                    print('      '+ error_message)
-                    log.append(error_message)
-                    print('\n')
-                    compl_cells[i]==3
-                    cell_summary_dict['Completeness']= 999
-                if compl_cells[i]==1: # cell is complete, no additional OQ input files need to be written
+                for compl_value in [5, 6]:  # 5: 'water', 6: 'empty'
+                    if compl_cells[i] == compl_value and len(obm_dict.keys()) > 0: # 5:'Irrelevant (Water)'
+                        error_message= '  ERROR: completeness of cell ID %s is %s but OBM buildings have been found --> TREATING AS INCOMPLETE CELL NOW.' % (str(grid_cell_id), str(compl_value))
+                        print('\n')
+                        print('      '+ error_message)
+                        log.append(error_message)
+                        print('\n')
+                        compl_cells[i]==3
+                        cell_summary_dict['Completeness']= 999
+                if compl_cells[i] == 1 or compl_cells[i] == 5 or compl_cells[i] == 6:  # cell is complete (1) or water (5) or empty (6), no additional OQ input files need to be written
                     leftover_num_bdgs= 0.0
                     leftover_num_dwells= 0.0
                     leftover_num_ppl= 0.0

SERA_mapping_admin_units_to_cells_add_GHS_from_OBM_tiles.py

@@ -50,12 +50,11 @@ def run_this_file(config_dict):
     # Path for GHS HDF5 file:
     ghs_path = config_dict['File Paths']['ghs_path'] 
     # Database from which the GHS built-up values are read:
-    self_dict = config_dict['SERA_mapping_admin_units_to_cells_add_GHS_from_OBM_tiles']
-    obm_tiles_db_hostname = self_dict['obm_tiles_db_hostname']
-    obm_tiles_db_name = self_dict['obm_tiles_db_name']
-    obm_tiles_db_username = self_dict['obm_tiles_db_username']
-    obm_built_area_assessments = self_dict['obm_tiles_db_table']
-    obm_tiles_source_id = int(self_dict['obm_tiles_source_id'])
+    obm_tiles_db_hostname = config_dict['OBM tiles']['obm_tiles_db_hostname']
+    obm_tiles_db_name = config_dict['OBM tiles']['obm_tiles_db_name']
+    obm_tiles_db_username = config_dict['OBM tiles']['obm_tiles_db_username']
+    obm_tiles_db_table = config_dict['OBM tiles']['obm_tiles_db_table']
+    obm_tiles_source_id = int(config_dict['OBM tiles']['obm_tiles_source_built_up_id'])
     # Tiles database:
     DB_username_grid = config_dict['Tiles Database']['db_tiles_username']
     DB_name_grid = config_dict['Tiles Database']['db_tiles_name']
@@ -118,7 +117,7 @@ def run_this_file(config_dict):
             ghs_built_up_i = gdet_psql.get_ghs_built_up_area_from_obm_tiles_opened_connection(
                 str(grid_cell_id),
                 cursor,
-                obm_built_area_assessments,
+                obm_tiles_db_table,
                 obm_tiles_source_id)
             print(str(grid_cell_id)+": "+str(ghs_built_up_i))
             # Write GHS built up area to database (distributed proportionally to the area of
@@ -160,7 +159,7 @@ if __name__=='__main__':
     section_names_to_validate= ['File Paths',
                                 'Tiles Database',
                                 'Cells to Process',
-                                os.path.basename(__file__).split('.')[0]]
+                                'OBM tiles']
     config_dict= gdet_conf.read_config_parameters(os.path.join(os.getcwd(),
                                                                config_filename),
                                                   section_names_to_validate)

docs/guide_to_use_of_config_file.csv

-FILE,Self,File Paths,OBM Database,Tiles Database,Admin Units Database,Ocuppancy String Groups,Available Results,Cells to Process
-GDE_check_consistency,Y,Y,Y,N,N,Y,Y,N
-GDE_check_OQ_input_files,N,Y,N,N,N,N,Y,N
-GDE_gather_SERA_and_OBM,Y,Y,N,Y,N,N,N,Y
-GDE_plot_maps,Y,Y,N,N,N,N,Y,N
-OBM_assign_cell_ids_and_adm_ids_to_footprints,Y,N,Y,N,Y,N,N,N
-OBM_buildings_per_cell,Y,Y,Y,Y,N,Y,N,Y
-SERA_create_HDF5_metadata,Y,Y,N,N,N,N,N,N
-SERA_create_OQ_input_files,Y,Y,N,Y,N,N,N,N
-SERA_create_outputs_QGIS_for_checking,Y,Y,N,Y,N,N,N,N
-SERA_create_visual_output_of_grid_model_full_files,Y,Y,N,Y,N,N,N,Y
-SERA_distributing_exposure_to_cells,Y,Y,N,Y,N,N,N,N
-SERA_exploration_check_Ids_unequivocally_defined,N,Y,N,N,N,N,N,N
-SERA_exploration_check_one_point_per_ID,N,Y,N,N,N,N,N,N
-SERA_exploration_countries_read,N,Y,N,N,N,N,N,N
-SERA_exploration_investigate_full_CSV_files,Y,Y,N,N,N,N,N,N
-SERA_exploration_investigate_macro_taxonomies,Y,Y,N,N,N,N,N,N
-SERA_exploration_longest_strings,N,Y,N,N,N,N,N,N
-SERA_exploration_summarise_columns_in_full_CSV_files,Y,Y,N,N,N,N,N,N
-SERA_mapping_admin_units_to_cells,Y,Y,N,Y,N,N,N,N
-SERA_mapping_admin_units_to_cells_add_GHS_from_HDF5,Y,Y,N,Y,N,N,N,Y
-SERA_mapping_admin_units_to_cells_add_GHS_from_OBM_tiles,Y,Y,N,Y,N,N,N,Y
-SERA_mapping_admin_units_to_cells_add_GPW,N,Y,N,Y,N,N,N,Y
-SERA_mapping_admin_units_to_cells_add_Sat,Y,Y,N,Y,N,N,N,Y
-SERA_testing_compare_visual_output_vs_OQ_input_files,Y,Y,N,N,N,N,N,N
-SERA_testing_mapping_admin_units_to_cells_qualitycontrol,Y,Y,N,Y,N,N,N,N
-SERA_testing_rebuilding_exposure_from_cells_alternative_01,Y,Y,N,Y,N,N,N,N
-SERA_testing_rebuilding_exposure_from_cells_alternative_02,Y,Y,N,N,N,N,N,N
-SERA_testing_rebuilding_exposure_from_cells_alternative_03,Y,Y,N,N,N,N,N,N
-GDE_check_tiles_vs_visual_CSVs,Y,Y,N,N,N,N,N,N
+FILE,Self,File Paths,OBM Database,Tiles Database,Admin Units Database,OBM tiles,Ocuppancy String Groups,Available Results,Cells to Process
+GDE_check_consistency,Y,Y,Y,N,N,N,Y,Y,N
+GDE_check_OQ_input_files,N,Y,N,N,N,N,N,Y,N
+GDE_gather_SERA_and_OBM,Y,Y,N,Y,N,Y,N,N,Y
+GDE_plot_maps,Y,Y,N,N,N,N,N,Y,N
+OBM_assign_cell_ids_and_adm_ids_to_footprints,Y,N,Y,N,Y,N,N,N,N
+OBM_buildings_per_cell,Y,Y,Y,Y,N,N,Y,N,Y
+SERA_create_HDF5_metadata,Y,Y,N,N,N,N,N,N,N
+SERA_create_OQ_input_files,Y,Y,N,Y,N,N,N,N,N
+SERA_create_outputs_QGIS_for_checking,Y,Y,N,Y,N,N,N,N,N
+SERA_create_visual_output_of_grid_model_full_files,Y,Y,N,Y,N,N,N,N,Y
+SERA_distributing_exposure_to_cells,Y,Y,N,Y,N,N,N,N,N
+SERA_exploration_check_Ids_unequivocally_defined,N,Y,N,N,N,N,N,N,N
+SERA_exploration_check_one_point_per_ID,N,Y,N,N,N,N,N,N,N
+SERA_exploration_countries_read,N,Y,N,N,N,N,N,N,N
+SERA_exploration_investigate_full_CSV_files,Y,Y,N,N,N,N,N,N,N
+SERA_exploration_investigate_macro_taxonomies,Y,Y,N,N,N,N,N,N,N
+SERA_exploration_longest_strings,N,Y,N,N,N,N,N,N,N
+SERA_exploration_summarise_columns_in_full_CSV_files,Y,Y,N,N,N,N,N,N,N
+SERA_mapping_admin_units_to_cells,Y,Y,N,Y,N,N,N,N,N
+SERA_mapping_admin_units_to_cells_add_GHS_from_HDF5,Y,Y,N,Y,N,N,N,N,Y
+SERA_mapping_admin_units_to_cells_add_GHS_from_OBM_tiles,N,Y,N,Y,N,Y,N,N,Y
+SERA_mapping_admin_units_to_cells_add_GPW,N,Y,N,Y,N,N,N,N,Y
+SERA_mapping_admin_units_to_cells_add_Sat,Y,Y,N,Y,N,N,N,N,Y
+SERA_testing_compare_visual_output_vs_OQ_input_files,Y,Y,N,N,N,N,N,N,N
+SERA_testing_mapping_admin_units_to_cells_qualitycontrol,Y,Y,N,Y,N,N,N,N,N
+SERA_testing_rebuilding_exposure_from_cells_alternative_01,Y,Y,N,Y,N,N,N,N,N
+SERA_testing_rebuilding_exposure_from_cells_alternative_02,Y,Y,N,N,N,N,N,N,N
+SERA_testing_rebuilding_exposure_from_cells_alternative_03,Y,Y,N,N,N,N,N,N,N
+GDE_check_tiles_vs_visual_CSVs,Y,Y,N,N,N,N,N,N,N