Commit 9b2bf257 authored by Cecilia Nievas's avatar Cecilia Nievas
Browse files

Merge branch 'feature/quadtiles' into 'master'

Adaptation of the GDE prototype code to work with quadtiles instead of 10-arcsec cells

See merge request !18
parents c8a2a0b8 79f71100
...@@ -914,7 +914,7 @@ def fill_in_admin_summary_dictionary_with_OBM(adm_summ_dict, obm_dict, obm_list_ ...@@ -914,7 +914,7 @@ def fill_in_admin_summary_dictionary_with_OBM(adm_summ_dict, obm_dict, obm_list_
adm_lev= obm_country_adminIDs_levels[j] adm_lev= obm_country_adminIDs_levels[j]
if adm_lev>-1: if adm_lev>-1:
country_name= gdet_serah.get_country_name_from_iso2_and_metadata_HDF5(adm_id.split('_')[0], sera_meta_path) country_name= gdet_serah.get_country_name_from_iso2_and_metadata_HDF5(adm_id.split('_')[0], sera_meta_path)
adm_summ_dict[adm_id]['geometry'], _= get_admin_unit_geometry_from_ID(country_name, adm_lev, adm_id.split('_')[1], countries_shp_path) adm_summ_dict[adm_id]['geometry'], _= get_admin_unit_geometry_from_ID(country_name, adm_lev, '_'.join(adm_id.split('_')[1:]), countries_shp_path)
else: else:
adm_summ_dict[adm_id]['geometry']= 'NotAvailable' adm_summ_dict[adm_id]['geometry']= 'NotAvailable'
return adm_summ_dict return adm_summ_dict
...@@ -979,7 +979,7 @@ def fill_in_admin_summary_dictionary_with_SERA_LeftOver_or_Total_bdgs_dwells_ppl ...@@ -979,7 +979,7 @@ def fill_in_admin_summary_dictionary_with_SERA_LeftOver_or_Total_bdgs_dwells_ppl
adm_lev= list_country_adminIDs_levels[j] adm_lev= list_country_adminIDs_levels[j]
if adm_lev>-1: if adm_lev>-1:
country_name= gdet_serah.get_country_name_from_iso2_and_metadata_HDF5(country_admin_ID.split('_')[0], sera_meta_path) country_name= gdet_serah.get_country_name_from_iso2_and_metadata_HDF5(country_admin_ID.split('_')[0], sera_meta_path)
adm_summ_dict[country_admin_ID]['geometry'], _= get_admin_unit_geometry_from_ID(country_name, adm_lev, country_admin_ID.split('_')[1], countries_shp_path) adm_summ_dict[country_admin_ID]['geometry'], _= get_admin_unit_geometry_from_ID(country_name, adm_lev, '_'.join(country_admin_ID.split('_')[1:]), countries_shp_path)
else: else:
adm_summ_dict[country_admin_ID]['geometry']= 'NotAvailable' adm_summ_dict[country_admin_ID]['geometry']= 'NotAvailable'
return adm_summ_dict return adm_summ_dict
...@@ -1261,7 +1261,7 @@ def define_grid_cell_list_to_process(configuration_dict, script_name): ...@@ -1261,7 +1261,7 @@ def define_grid_cell_list_to_process(configuration_dict, script_name):
""" """
out_grid_cells_def_method= configuration_dict[script_name]['method'] out_grid_cells_def_method= configuration_dict[script_name]['method']
if out_grid_cells_def_method=='arbitrary': if out_grid_cells_def_method=='arbitrary':
out_grid_cells_list= [int(i) for i in configuration_dict[script_name]['arbitr_list'].split(', ')] out_grid_cells_list= [i for i in configuration_dict[script_name]['arbitr_list'].split(', ')]
elif out_grid_cells_def_method=='country_in_DB': elif out_grid_cells_def_method=='country_in_DB':
out_grid_cells_list= gdet_psql.retrieve_cell_IDs_by_country(configuration_dict['Tiles Database']['db_tiles_name'], out_grid_cells_list= gdet_psql.retrieve_cell_IDs_by_country(configuration_dict['Tiles Database']['db_tiles_name'],
configuration_dict['Tiles Database']['db_tiles_username'], configuration_dict['Tiles Database']['db_tiles_username'],
......
...@@ -27,6 +27,7 @@ Tools used by the GDE code to access/query/write to the OBM extract and cells da ...@@ -27,6 +27,7 @@ Tools used by the GDE code to access/query/write to the OBM extract and cells da
""" """
import psycopg2 # Documentation: http://initd.org/psycopg/docs/ import psycopg2 # Documentation: http://initd.org/psycopg/docs/
import getpass
import numpy as np import numpy as np
import pandas as pd import pandas as pd
from copy import deepcopy from copy import deepcopy
...@@ -74,7 +75,7 @@ def retrieve_parameters_of_buildings_in_cell_raw(db_name, db_username, sch_name, ...@@ -74,7 +75,7 @@ def retrieve_parameters_of_buildings_in_cell_raw(db_name, db_username, sch_name,
conn.set_session(autocommit=True) conn.set_session(autocommit=True)
cur = conn.cursor() # Open a cursor to perform database operations cur = conn.cursor() # Open a cursor to perform database operations
aux_query_str= ','.join(parameter_names) aux_query_str= ','.join(parameter_names)
cur.execute('SELECT %s FROM %s.%s WHERE cell_id=%s;' %(aux_query_str, sch_name, table_name, in_cell_id)) cur.execute("SELECT %s FROM %s.%s WHERE cell_id='%s';" %(aux_query_str, sch_name, table_name, in_cell_id))
exec_result = cur.fetchall() exec_result = cur.fetchall()
conn.close() conn.close()
return exec_result return exec_result
...@@ -617,6 +618,205 @@ def get_obm_buildings_by_broad_occupancy(db_name, db_username, sch_name, table_n ...@@ -617,6 +618,205 @@ def get_obm_buildings_by_broad_occupancy(db_name, db_username, sch_name, table_n
return out_broad_occus, out_num_bdgs, gem_occus, num_entries return out_broad_occus, out_num_bdgs, gem_occus, num_entries
def get_ghs_built_up_area_from_obm_tiles(grid_cell_id, db_hostname, db_name,
db_username, db_table, source_id=1):
"""This function retrieves the GHS built-up area from the OBM tiles database for a specified
zoom level 18 quadkey.
Note that a password is required to access the database. If not provided as input, this
function will prompt the user to indicate it. If the connection to the database and a cursor
are opened before calling this function, consider using the function
get_ghs_built_up_area_from_obm_tiles_opened_connection() instead.
Args:
grid_cell_id (str): Quadkey of the zoom level 18 target tile.
db_hostname (str): Name of the server where the database is located.
db_name (str): Name of the database.
db_username (str): Username for access to the database.
db_table (str): Name of the table where this information is stored.
source_id (int): Integer indicating the source ID of the GHS built-up area. Deafault: 1.
Returns:
ghs_built_up (float): Built-up area (m2) in the target tile according to the Global
Human Settlement GHSL-BUILT multitemporal dataset available in
30-m resolution at https://ghsl.jrc.ec.europa.eu/download.php.
"""
# Connect to the database:
connection = psycopg2.connect('host=%s dbname=%s user=%s password=%s'
% (db_hostname, db_name, db_username, getpass.getpass()))
connection.set_session(autocommit=True)
# Open a cursor to perform database operations
cursor = connection.cursor()
# Query the database:
sql_query = "SELECT built_area_size FROM %s WHERE (quadkey='%s' AND source_id=%s);" \
% (db_table, grid_cell_id, source_id)
cursor.execute(sql_query)
exec_result = cursor.fetchall()
connection.close()
if len(exec_result) == 0: # If the quadkey is not found the built-up area is zero
ghs_built_up = 0.0
elif len(exec_result) == 1:
ghs_built_up = exec_result[0][0]
else: # More than one entries found, this is an error
print("ERROR IN get_ghs_built_up_area_from_obm_tiles: "
"MORE THAN ONE ENTRY FOUND FOR CELL ID %s"
% (grid_cell_id))
ghs_built_up = -999.9
return ghs_built_up
def get_ghs_built_up_area_from_obm_tiles_opened_connection(grid_cell_id, open_cursor,
db_table, source_id=1):
"""This function retrieves the GHS built-up area from the OBM tiles database for a specified
zoom level 18 quadkey.
Note that this function requires that a connection to the database has been established and
a cursor opened. The function get_ghs_built_up_area_from_obm_tiles() can be used if this is
not the case.
Args:
grid_cell_id (str): Quadkey of the zoom level 18 target tile.
open_cursor: Cursor connecting to the OBM tiles database (already open).
db_table (str): Name of the table where this information is stored.
source_id (int): Integer indicating the source ID of the GHS built-up area. Deafault: 1.
Returns:
ghs_built_up (float): Built-up area (m2) in the target tile according to the Global
Human Settlement GHSL-BUILT multitemporal dataset available in
30-m resolution at https://ghsl.jrc.ec.europa.eu/download.php.
"""
# Query the database:
sql_query = "SELECT built_area_size FROM %s WHERE (quadkey='%s' AND source_id=%s);" \
% (db_table, grid_cell_id, source_id)
open_cursor.execute(sql_query)
exec_result = open_cursor.fetchall()
if len(exec_result) == 0: # If the quadkey is not found the built-up area is zero
ghs_built_up = 0.0
elif len(exec_result) == 1:
ghs_built_up = exec_result[0][0]
else: # More than one entries found, this is an error
print("ERROR IN get_ghs_built_up_area_from_obm_tiles: "
"MORE THAN ONE ENTRY FOUND FOR CELL ID %s"
% (grid_cell_id))
ghs_built_up = -999.9
return ghs_built_up
def get_completeness_from_obm_tiles(grid_cell_id, db_hostname, db_name,
db_username, db_table):
"""This function retrieves the completeness value from the OBM tiles database for a
specified zoom level 18 quadkey.
Note that a password is required to access the database. If not provided as input, this
function will prompt the user to indicate it.
Args:
grid_cell_id (str): Quadkey of the zoom level 18 target tile.
db_hostname (str): Name of the server where the database is located.
db_name (str): Name of the database.
db_username (str): Username for access to the database.
db_table (str): Name of the table where this information is stored.
Returns:
completeness (int): Completeness value in the target tile. The possible values are:
0: unknown
1: complete
2: almost complete
3: incomplete
4: undecidable
5: water
6: empty
"""
# Connect to the database:
connection = psycopg2.connect('host=%s dbname=%s user=%s password=%s'
% (db_hostname, db_name, db_username, getpass.getpass()))
connection.set_session(autocommit=True)
# Open a cursor to perform database operations
cursor = connection.cursor()
# Query the database:
sql_query = "SELECT completeness FROM %s WHERE quadkey='%s';" \
% (db_table, grid_cell_id)
cursor.execute(sql_query)
exec_result = cursor.fetchall()
connection.close()
if len(exec_result) == 0: # If the quadkey is not found, assume completeness unknown
completeness = 0
elif len(exec_result) == 1:
completeness = exec_result[0][0]
else: # More than one entries found, this is an error
print("ERROR IN get_completeness_from_obm_tiles: "
"MORE THAN ONE ENTRY FOUND FOR CELL ID %s"
% (grid_cell_id))
completeness = -999.9
return completeness
def get_completeness_from_obm_tiles_for_several(grid_cell_ids, db_hostname, db_name,
db_username, db_table):
"""This function retrieves the completeness value from the OBM tiles database for a
specified zoom level 18 quadkey.
Note that a password is required to access the database. If not provided as input, this
function will prompt the user to indicate it.
Args:
grid_cell_ids (list of str): List of quadkeys of the zoom level 18 target tiles.
db_hostname (str): Name of the server where the database is located.
db_name (str): Name of the database.
db_username (str): Username for access to the database.
db_table (str): Name of the table where this information is stored.
Returns:
completeness (list of int): List of completeness values of the target tiles. The
possible values are:
0: unknown
1: complete
2: almost complete
3: incomplete
4: undecidable
5: water
6: empty
"""
# Connect to the database:
connection = psycopg2.connect('host=%s dbname=%s user=%s password=%s'
% (db_hostname, db_name, db_username, getpass.getpass()))
connection.set_session(autocommit=True)
# Open a cursor to perform database operations
cursor = connection.cursor()
completeness = []
for grid_cell_id in grid_cell_ids:
# Query the database:
sql_query = "SELECT completeness FROM %s WHERE quadkey='%s';" \
% (db_table, grid_cell_id)
cursor.execute(sql_query)
exec_result = cursor.fetchall()
if len(exec_result) == 0: # If the quadkey is not found, assume completeness unknown
completeness.append(0)
elif len(exec_result) == 1:
completeness.append(exec_result[0][0])
else: # More than one entries found, this is an error
print("ERROR IN get_completeness_from_obm_tiles: "
"MORE THAN ONE ENTRY FOUND FOR CELL ID %s"
% (grid_cell_id))
completeness.append(-999.9)
connection.close()
return completeness
...@@ -775,9 +775,21 @@ def transform_dict_into_dataframe(dict_of_subclasses, bdg_classes): ...@@ -775,9 +775,21 @@ def transform_dict_into_dataframe(dict_of_subclasses, bdg_classes):
collect_values[b,col]= dict_of_subclasses[bdgclass]['Proportions'][which[0]] collect_values[b,col]= dict_of_subclasses[bdgclass]['Proportions'][which[0]]
elif len(which)>1: elif len(which)>1:
print('ERROR!! SOMETHING WENT WRONG IN transform_dict_into_dataframe, length='+str(len(which))) print('ERROR!! SOMETHING WENT WRONG IN transform_dict_into_dataframe, length='+str(len(which)))
if np.any(np.logical_or(collect_values.sum(axis=1)>1.0001,collect_values.sum(axis=1)<0.9999)): sum_per_row = collect_values.sum(axis=1)
print('WARNING IN transform_dict_into_dataframe: THE SUM PER ROW IS NOT 1.0 IN AT LEAST ONE ROW!!!') for row in range(0,collect_values.shape[0]):
errors_found= True # The row does not add up to 1 but it also does not add up to zero (measured as < 1E-5):
if np.logical_or(sum_per_row[row] > 1.0001, sum_per_row[row] < 0.9999) and (sum_per_row[row] > 1E-5):
print('WARNING IN transform_dict_into_dataframe: THE SUM PER ROW IS NOT 1.0 IN AT LEAST ONE ROW!!!')
errors_found= True
elif sum_per_row[row] < 1E-5:
# If the row sums up to zero (because there are zero buildings in these admin units), replace the
# zeroes with a uniform distribution:
aux_ids = dict_of_subclasses[bdg_classes[row]]['Country_Admin_IDs'] # admin IDs for which this building class exists
uniform_proportion = 1.0/float(len(aux_ids))
for admin_id in aux_ids:
which_col = np.where(unique_involved_ids == admin_id)[0][0]
collect_values[row, which_col] = uniform_proportion
return bdg_classes, unique_involved_ids, collect_values, errors_found return bdg_classes, unique_involved_ids, collect_values, errors_found
......
...@@ -149,6 +149,10 @@ def check_parameters(config, section_name): ...@@ -149,6 +149,10 @@ def check_parameters(config, section_name):
raise IOError( raise IOError(
"ERROR!! version_of_SERA_metadata PARAMETER MISSING FROM CONFIG FILE!!" "ERROR!! version_of_SERA_metadata PARAMETER MISSING FROM CONFIG FILE!!"
) )
if not config.has_option("GDE_gather_SERA_and_OBM", "read_completeness"):
raise IOError(
"ERROR!! read_completeness PARAMETER MISSING FROM CONFIG FILE!!"
)
if not config.has_option("GDE_gather_SERA_and_OBM", "print_screen_during_run"): if not config.has_option("GDE_gather_SERA_and_OBM", "print_screen_during_run"):
raise IOError( raise IOError(
"ERROR!! print_screen_during_run PARAMETER MISSING FROM CONFIG FILE!!" "ERROR!! print_screen_during_run PARAMETER MISSING FROM CONFIG FILE!!"
...@@ -291,11 +295,36 @@ def check_parameters(config, section_name): ...@@ -291,11 +295,36 @@ def check_parameters(config, section_name):
elif section_name == "SERA_mapping_admin_units_to_cells": elif section_name == "SERA_mapping_admin_units_to_cells":
if not config.has_option("SERA_mapping_admin_units_to_cells", "countries"): if not config.has_option("SERA_mapping_admin_units_to_cells", "countries"):
raise IOError("ERROR!! countries PARAMETER MISSING FROM CONFIG FILE!!") raise IOError("ERROR!! countries PARAMETER MISSING FROM CONFIG FILE!!")
elif section_name == "SERA_mapping_admin_units_to_cells_add_GHS": elif section_name == "SERA_mapping_admin_units_to_cells_add_GHS_from_HDF5":
if not config.has_option( if not config.has_option(
"SERA_mapping_admin_units_to_cells_add_GHS", "ghs_input_filename" "SERA_mapping_admin_units_to_cells_add_GHS_from_HDF5", "ghs_input_filename"
): ):
raise IOError("ERROR!! ghs_input_filename PARAMETER MISSING FROM CONFIG FILE!!") raise IOError("ERROR!! ghs_input_filename PARAMETER MISSING FROM CONFIG FILE!!")
elif section_name == "OBM tiles":
if not config.has_option(
"OBM tiles", "obm_tiles_db_hostname"
):
raise IOError("ERROR!! obm_tiles_db_hostname PARAMETER MISSING FROM CONFIG FILE!!")
if not config.has_option(
"OBM tiles", "obm_tiles_db_name"
):
raise IOError("ERROR!! obm_tiles_db_name PARAMETER MISSING FROM CONFIG FILE!!")
if not config.has_option(
"OBM tiles", "obm_tiles_db_username"
):
raise IOError("ERROR!! obm_tiles_db_username PARAMETER MISSING FROM CONFIG FILE!!")
if not config.has_option(
"OBM tiles", "obm_tiles_db_table_built_up"
):
raise IOError("ERROR!! obm_tiles_db_table_built_up PARAMETER MISSING FROM CONFIG FILE!!")
if not config.has_option(
"OBM tiles", "obm_tiles_db_table_completeness"
):
raise IOError("ERROR!! obm_tiles_db_table_completeness PARAMETER MISSING FROM CONFIG FILE!!")
if not config.has_option(
"OBM tiles", "obm_tiles_source_built_up_id"
):
raise IOError("ERROR!! obm_tiles_source_id PARAMETER MISSING FROM CONFIG FILE!!")
elif section_name == "SERA_mapping_admin_units_to_cells_add_Sat": elif section_name == "SERA_mapping_admin_units_to_cells_add_Sat":
if not config.has_option("SERA_mapping_admin_units_to_cells_add_Sat", "apply_model"): if not config.has_option("SERA_mapping_admin_units_to_cells_add_Sat", "apply_model"):
raise IOError("ERROR!! apply_model PARAMETER MISSING FROM CONFIG FILE!!") raise IOError("ERROR!! apply_model PARAMETER MISSING FROM CONFIG FILE!!")
......
This diff is collapsed.
...@@ -63,6 +63,16 @@ db_admin_username = DB_USERNAME ...@@ -63,6 +63,16 @@ db_admin_username = DB_USERNAME
db_admin_schema_name = DB_SCHEMA_NAME db_admin_schema_name = DB_SCHEMA_NAME
db_admin_table_name = DB_TABLE_NAME db_admin_table_name = DB_TABLE_NAME
[OBM tiles]
# See https://git.gfz-potsdam.de/dynamicexposure/server-components/deployments/databases#obm-tiles
obm_tiles_db_hostname = SERVER_NAME
obm_tiles_db_name = DB_NAME
obm_tiles_db_username = DB_USERNAME
obm_tiles_db_table_built_up = DB_TABLE_NAME_BUILT_UP
obm_tiles_db_table_completeness = DB_TABLE_NAME_COMPLETENESS
# Source ID for built-up area to retrieve:
obm_tiles_source_built_up_id = 1
[Cells to Process] [Cells to Process]
# PARAMETER TO DECIDE HOW THE LIST OF CELLS IS DEFINED # PARAMETER TO DECIDE HOW THE LIST OF CELLS IS DEFINED
# Options are: # Options are:
...@@ -102,6 +112,8 @@ version_of_SERA_metadata = YYYY_MM_DD ...@@ -102,6 +112,8 @@ version_of_SERA_metadata = YYYY_MM_DD
# SELECT CASE OF SERA DISAGGREGATION TO CONSIDER: # SELECT CASE OF SERA DISAGGREGATION TO CONSIDER:
# Options are: area, gpw_2015_pop, ghs, sat_27f and sat_27f_model # Options are: area, gpw_2015_pop, ghs, sat_27f and sat_27f_model
sera_disaggregation_to_consider = sat_27f sera_disaggregation_to_consider = sat_27f
# Read completeness from CSV or OBM tiles (options: "csv", "obm_tiles"):
read_completeness = obm_tiles
# Print messages while running the code (for debugging purposes): # Print messages while running the code (for debugging purposes):
print_screen_during_run = False print_screen_during_run = False
# Occupancy cases in which the OBM buildings will be classified ('Oth' is implicit in the method): # Occupancy cases in which the OBM buildings will be classified ('Oth' is implicit in the method):
...@@ -202,7 +214,7 @@ occupancy_cases = Res, Com, Ind ...@@ -202,7 +214,7 @@ occupancy_cases = Res, Com, Ind
# Countries to process (if many, separate with comma and space): # Countries to process (if many, separate with comma and space):
countries = Greece countries = Greece
[SERA_mapping_admin_units_to_cells_add_GHS] [SERA_mapping_admin_units_to_cells_add_GHS_from_HDF5]
ghs_input_filename = FILENAME.hdf5 ghs_input_filename = FILENAME.hdf5
[SERA_mapping_admin_units_to_cells_add_Sat] [SERA_mapping_admin_units_to_cells_add_Sat]
......
...@@ -65,6 +65,11 @@ def run_this_file(config_dict): ...@@ -65,6 +65,11 @@ def run_this_file(config_dict):
DB_name_grid= config_dict['Tiles Database']['db_tiles_name'] DB_name_grid= config_dict['Tiles Database']['db_tiles_name']
DB_schema_name_grid= config_dict['Tiles Database']['db_tiles_schema_name'] DB_schema_name_grid= config_dict['Tiles Database']['db_tiles_schema_name']
DB_table_name_grid= config_dict['Tiles Database']['db_tiles_table_name'] DB_table_name_grid= config_dict['Tiles Database']['db_tiles_table_name']
# Database from which the completeness values are read:
obm_tiles_db_hostname = config_dict['OBM tiles']['obm_tiles_db_hostname']
obm_tiles_db_name = config_dict['OBM tiles']['obm_tiles_db_name']
obm_tiles_db_username = config_dict['OBM tiles']['obm_tiles_db_username']
obm_tiles_db_table = config_dict['OBM tiles']['obm_tiles_db_table_completeness']
# Case of SERA disaggregation to consider (['area', 'gpw_2015_pop', 'ghs', 'sat_27f', 'sat_27f_model']): # Case of SERA disaggregation to consider (['area', 'gpw_2015_pop', 'ghs', 'sat_27f', 'sat_27f_model']):
sera_disaggregation_to_consider= config_dict['GDE_gather_SERA_and_OBM']['sera_disaggregation_to_consider'] sera_disaggregation_to_consider= config_dict['GDE_gather_SERA_and_OBM']['sera_disaggregation_to_consider']
# Select whether to print to screen intermediate values or not (useful for debugging): # Select whether to print to screen intermediate values or not (useful for debugging):
...@@ -88,6 +93,8 @@ def run_this_file(config_dict): ...@@ -88,6 +93,8 @@ def run_this_file(config_dict):
min_grid_cell_id= np.min(grid_cells_list) - 1 min_grid_cell_id= np.min(grid_cells_list) - 1
# Version of SERA metadata to use: # Version of SERA metadata to use:
version_of_SERA_metadata= config_dict['GDE_gather_SERA_and_OBM']['version_of_SERA_metadata'] version_of_SERA_metadata= config_dict['GDE_gather_SERA_and_OBM']['version_of_SERA_metadata']
# Read completeness from CSV or OBM tiles (options: "csv", "obm_tiles"):
read_completeness= config_dict['GDE_gather_SERA_and_OBM']['read_completeness']
#################################################### ####################################################
# START # START
#################################################### ####################################################
...@@ -107,13 +114,18 @@ def run_this_file(config_dict): ...@@ -107,13 +114,18 @@ def run_this_file(config_dict):
else: else:
print('ERROR!!! METHOD FOR DEFINING LIST OF CELL IDs UNKNOWN. ERROR!!!') print('ERROR!!! METHOD FOR DEFINING LIST OF CELL IDs UNKNOWN. ERROR!!!')
print('Retrieving completeness information for grid cells...') print('Retrieving completeness information for grid cells...')
compl_cells= gdet_gral.retrieve_completeness_for_cells_from_CSV(grid_cells_list, completeness_filepath) # array with completeness values (integers) if read_completeness == "csv":
compl_cells= gdet_gral.retrieve_completeness_for_cells_from_CSV(grid_cells_list, completeness_filepath) # array with completeness values (integers)
elif read_completeness == "obm_tiles":
compl_cells= gdet_psql.get_completeness_from_obm_tiles_for_several(grid_cells_list, obm_tiles_db_hostname, obm_tiles_db_name, obm_tiles_db_username, obm_tiles_db_table)
else:
print("ERROR: 'read_completeness' VALUE IN CONFIGURATION FILE COULD NOT BE RECOGNISED!!!")
# To keep track of rows for writing the OpenQuake input files: # To keep track of rows for writing the OpenQuake input files:
next_row_id= [0 for case in occupancy_cases] next_row_id= [0 for case in occupancy_cases]
# Initialise admin_summary_dict: # Initialise admin_summary_dict:
admin_summary_dict= {} admin_summary_dict= {}
for i, grid_cell_id in enumerate(grid_cells_list): for i, grid_cell_id in enumerate(grid_cells_list):
if grid_cell_id>min_grid_cell_id: if int(grid_cell_id) > min_grid_cell_id:
print('\r Working on cell ID '+str(grid_cell_id)+'. Cell '+str(i+1)+' of '+str(len(grid_cells_list))+'.', end='') print('\r Working on cell ID '+str(grid_cell_id)+'. Cell '+str(i+1)+' of '+str(len(grid_cells_list))+'.', end='')
cell_summary_dict= gdet_gral.initialise_cell_summary_dictionary() cell_summary_dict= gdet_gral.initialise_cell_summary_dictionary()
cell_summary_dict['Completeness']= compl_cells[i] cell_summary_dict['Completeness']= compl_cells[i]
...@@ -174,15 +186,16 @@ def run_this_file(config_dict): ...@@ -174,15 +186,16 @@ def run_this_file(config_dict):
cell_summary_dict= gdet_gral.fill_in_cell_summary_dictionary_with_SERA_LeftOver_or_Total_bdgs_dwells_ppl(cell_summary_dict, case, 'SERA', sera_num_bdgs, sera_num_dwells, sera_num_ppl, sera_cost) cell_summary_dict= gdet_gral.fill_in_cell_summary_dictionary_with_SERA_LeftOver_or_Total_bdgs_dwells_ppl(cell_summary_dict, case, 'SERA', sera_num_bdgs, sera_num_dwells, sera_num_ppl, sera_cost)
admin_summary_dict= gdet_gral.fill_in_admin_summary_dictionary_with_SERA_LeftOver_or_Total_bdgs_dwells_ppl(admin_summary_dict, case, 'SERA', sera_country_adminIDs, sera_country_adminIDs_levels, sera_num_bdgs_per_class_and_adm_unit, sera_num_dwells_per_class_and_adm_unit, sera_num_ppl_per_class_and_adm_unit, sera_cost_per_class_and_adm_unit, sera_shp_path, os.path.join(out_path, 'Europe_SERA_metadata_v_'+version_of_SERA_metadata+'.hdf5'), input_kind='per_class_and_adm_unit') admin_summary_dict= gdet_gral.fill_in_admin_summary_dictionary_with_SERA_LeftOver_or_Total_bdgs_dwells_ppl(admin_summary_dict, case, 'SERA', sera_country_adminIDs, sera_country_adminIDs_levels, sera_num_bdgs_per_class_and_adm_unit, sera_num_dwells_per_class_and_adm_unit, sera_num_ppl_per_class_and_adm_unit, sera_cost_per_class_and_adm_unit, sera_shp_path, os.path.join(out_path, 'Europe_SERA_metadata_v_'+version_of_SERA_metadata+'.hdf5'), input_kind='per_class_and_adm_unit')
obm_with_classes_num_bdgs_per_class_and_adm_unit, obm_with_classes_num_dwells_per_class_and_adm_unit, obm_with_classes_num_ppl_per_class_and_adm_unit, obm_with_classes_cost_per_class_and_adm_unit, obm_list_bdg_classes_of_arrays, obm_list_country_adminIDs_of_arrays= gdet_gral.disaggregate_OBM_by_class_and_adm_unit_main(obm_dict, sera_country_adminIDs, sera_classes) obm_with_classes_num_bdgs_per_class_and_adm_unit, obm_with_classes_num_dwells_per_class_and_adm_unit, obm_with_classes_num_ppl_per_class_and_adm_unit, obm_with_classes_cost_per_class_and_adm_unit, obm_list_bdg_classes_of_arrays, obm_list_country_adminIDs_of_arrays= gdet_gral.disaggregate_OBM_by_class_and_adm_unit_main(obm_dict, sera_country_adminIDs, sera_classes)
if compl_cells[i]==5 and len(obm_dict.keys())>0: # 5:'Irrelevant (Water)' for compl_value in [5, 6]: # 5: 'water', 6: 'empty'
error_message= ' ERROR: completeness of cell ID '+str(grid_cell_id)+' is 5 but OBM buildings have been found --> TREATING AS INCOMPLETE CELL NOW.' if compl_cells[i] == compl_value and len(obm_dict.keys()) > 0: # 5:'Irrelevant (Water)'
print('\n') error_message= ' ERROR: completeness of cell ID %s is %s but OBM buildings have been found --> TREATING AS INCOMPLETE CELL NOW.' % (str(grid_cell_id), str(compl_value))
print(' '+ error_message) print('\n')
log.append(error_message) print(' '+ error_message)
print('\n') log.append(error_message)
compl_cells[i]==3 print('\n')
cell_summary_dict['Completeness']= 999 compl_cells[i]= 3
if compl_cells[i]==1: # cell is complete, no additional OQ input files need to be written cell_summary_dict['Completeness']= 999
if compl_cells[i] == 1 or compl_cells[i] == 5 or compl_cells[i] == 6: # cell is complete (1) or water (5) or empty (6), no additional OQ input files need to be written
leftover_num_bdgs= 0.0 leftover_num_bdgs= 0.0
leftover_num_dwells= 0.0 leftover_num_dwells= 0.0
leftover_num_ppl= 0.0 leftover_num_ppl= 0.0
...@@ -276,7 +289,7 @@ def run_this_file(config_dict): ...@@ -276,7 +289,7 @@ def run_this_file(config_dict):
if __name__=='__main__': if __name__=='__main__':
# This code needs to be run from the command line as python3 namefile.py configfile.ini # This code needs to be run from the command line as python3 namefile.py configfile.ini
config_filename= sys.argv[1] # sys.argv retrieves all the commands entered in the command line; position [0] is this code, position [1] will be the config file name config_filename= sys.argv[1] # sys.argv retrieves all the commands entered in the command line; position [0] is this code, position [1] will be the config file name
section_names_to_validate= ['File Paths', 'Tiles Database', 'Cells to Process', os.path.basename(__file__).split('.')[0]] section_names_to_validate= ['File Paths', 'Tiles Database', 'Cells to Process', 'OBM tiles', os.path.basename(__file__).split('.')[0]]
config_dict= gdet_conf.read_config_parameters(os.path.join(os.getcwd(), config_filename), section_names_to_validate) config_dict= gdet_conf.read_config_parameters(os.path.join(os.getcwd(), config_filename), section_names_to_validate)
run_this_file(config_dict) run_this_file(config_dict)
......
...@@ -85,7 +85,7 @@ def run_this_file(config_dict): ...@@ -85,7 +85,7 @@ def run_this_file(config_dict):
#################################################### ####################################################
log= [] # list that I fill in for logging purposes and then dump to a txt file log= [] # list that I fill in for logging purposes and then dump to a txt file
for i, grid_cell_id in enumerate(grid_cells_list): for i, grid_cell_id in enumerate(grid_cells_list):
if grid_cell_id>min_grid_cell_id: if int(grid_cell_id) > min_grid_cell_id:
print('\r Working on cell ID '+str(grid_cell_id)+'. Cell '+str(i+1)+' of '+str(len(grid_cells_list))+'.', end='') print('\r Working on cell ID '+str(grid_cell_id)+'. Cell '+str(i+1)+' of '+str(len(grid_cells_list))+'.', end='')
bdgs_df_i= gdet_psql.retrieve_parameters_of_buildings_in_cell_as_pandas(DB_name_bdgs, DB_username_bdgs, DB_schema_name_bdgs, DB_table_name_bdgs, grid_cell_id, parameter_names=['osm_id', 'gem_occupancy', 'building_levels', 'way_area', 'country_iso2', 'res_adm_id', 'res_adm_level', 'com_adm_id', 'com_adm_level', 'ind_adm_id', 'ind_adm_level'], get_centroid_coords=True) bdgs_df_i= gdet_psql.retrieve_parameters_of_buildings_in_cell_as_pandas(DB_name_bdgs, DB_username_bdgs, DB_schema_name_bdgs, DB_table_name_bdgs, grid_cell_id, parameter_names=['osm_id', 'gem_occupancy', 'building_levels', 'way_area', 'country_iso2', 'res_adm_id', 'res_adm_level', 'com_adm_id', 'com_adm_level', 'ind_adm_id', 'ind_adm_level'], get_centroid_coords=True)
bdgs_df_i= bdgs_df_i.rename(columns={'way_area': 'area', 'ST_X(ST_Transform(ST_Centroid(way),4326))': 'lon', 'ST_Y(ST_Transform(ST_Centroid(way),4326))': 'lat'}) bdgs_df_i= bdgs_df_i.rename(columns={'way_area': 'area', 'ST_X(ST_Transform(ST_Centroid(way),4326))': 'lon', 'ST_Y(ST_Transform(ST_Centroid(way),4326))': 'lat'})
......
...@@ -20,6 +20,7 @@ Dependencies (not all are needed for all individual scripts): ...@@ -20,6 +20,7 @@ Dependencies (not all are needed for all individual scripts):
- json (2.0.9 or later) - json (2.0.9 or later)
- iso3166 (1.0.1 or later) - iso3166 (1.0.1 or later)
- pyproj (1.9.5.1 or later) - pyproj (1.9.5.1 or later)
- mercantile (1.2.1 or later)
# Copyright and Copyleft # Copyright and Copyleft
......
"""
Copyright (C) 2021
Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
Global Dynamic Exposure Model
Helmholtz Centre Potsdam
GFZ German Research Centre for Geosciences
Section 2.6: Seismic Hazard and Risk Dynamics
SERA_mapping_admin_units_to_cells_add_GHS_from_OBM_tiles
========================================================
This code adds the surface of built area per tile according to the Global Human Settlement (GHS)
layer. It retrieves the corresponding value from the OBM Tiles database]. The values are stored
in the PSQL database (field `ghs_km2`), and are thus allocated to intersections of cells and
administrative units proportionally to the areas of the intersections with respect to that of
the complete cell.
"""
import sys
import os
import shapely
import datetime
import h5py
import getpass
import psycopg2
import numpy as np
import GDE_TOOLS_psql as gdet_psql
import GDE_TOOLS_world_grid as gdet_grid
import GDE_TOOLS_general as gdet_gral
import GDE_TOOLS_read_config_file as gdet_conf