Commit c7e627a1 authored by Cecilia Nievas's avatar Cecilia Nievas
Browse files

Added feature to store factors for distribution of people in time

parent d98da817
Pipeline #35878 passed with stage
in 2 minutes and 31 seconds
...@@ -81,7 +81,7 @@ class ExposureEntity: ...@@ -81,7 +81,7 @@ class ExposureEntity:
| | of the dictionary are the IDs of the corresponding data units. | | of the dictionary are the IDs of the corresponding data units.
| |_ population_time_distribution (dict): | |_ population_time_distribution (dict):
| | Dictionary containing factors by which the census population per | | Dictionary containing factors by which the census population per
| | building can be multiplied to obtain an estimate of the | | building can be multiplied to obtain an estimate of the people
| | in the buildings at a certain time of the day. It contains the | | in the buildings at a certain time of the day. It contains the
| | following keys: | | following keys:
| | Day (float): | | Day (float):
...@@ -316,8 +316,7 @@ class ExposureEntity: ...@@ -316,8 +316,7 @@ class ExposureEntity:
aggregated_source_id (int): aggregated_source_id (int):
ID of the source of the aggregated exposure model. ID of the source of the aggregated exposure model.
occupancy_case (str): occupancy_case (str):
Name of the occupancy case (e.g. "residential", "commercial", "industrial") Name of the occupancy case (e.g. "residential", "commercial", "industrial").
associated with this data_unit.
Returns: Returns:
This function writes to the table with name db_table in the database whose This function writes to the table with name db_table in the database whose
credentials are indicated in db_data_units_config. credentials are indicated in db_data_units_config.
...@@ -396,6 +395,138 @@ class ExposureEntity: ...@@ -396,6 +395,138 @@ class ExposureEntity:
return return
def write_people_distribution_in_time_to_database(
self,
db_data_units_config,
db_table,
aggregated_source_id,
occupancy_case,
):
"""This function writes the factors by which the census population per building can be
building can be multiplied to obtain an estimate of the people in the buildings at a
certain time of the day (day, night, and during transit hours) to the table with name
db_table in the database whose credentials are indicated in db_data_units_config. If an
entry already exists in the database for this ExposureEntity, occupancy_case and
aggregated_source_id, the attributes are updated. If an entry does not exist beforehand,
it is created.
Args:
db_data_units_config (dict):
Dictionary containing the credentials needed to connect to the SQL database in
which information on the cost assumptions is stored. The keys of the dictionary
need to be:
host (str):
SQL database host address.
dbname (str):
Name of the SQL database.
port (int):
Port where the SQL database can be found.
username (str):
User name to connect to the SQL database.
password (str):
Password associated with self.username.
db_table (str):
Name of the table of the SQL database where the factors to distribute people at
different times of the day will be stored. It is assumed that this table
contains, at least, the following fields:
exposure_entity (str):
3-character code of the exposure entity.
occupancy_case (enum):
SQL enumerated type describing the building occupancy cases.
aggregated_source_id (int):
ID of the source of the aggregated exposure model.
day (float):
Factor to obtain the number of people expected to be inside the
buildings during the day (approx. 10 am to 6 pm).
night (float):
Factor to obtain the number of people expected to be inside the
buildings during the night (approx. 10 pm to 6 am).
transit (float):
Factor to obtain the number of people expected to be inside the
buildings during transit times (approx. 6 am to 10 am and 6 pm to 10
pm).
aggregated_source_id (int):
ID of the source of the aggregated exposure model.
occupancy_case (str):
Name of the occupancy case (e.g. "residential", "commercial", "industrial").
Returns:
This function writes to the table with name db_table in the database whose
credentials are indicated in db_data_units_config.
"""
sql_commands = {}
sql_commands["query"] = "SELECT COUNT(*) FROM %s"
sql_commands["query"] += " WHERE (exposure_entity='%s' AND occupancy_case='%s'"
sql_commands["query"] += " AND aggregated_source_id='%s');"
sql_commands["update"] = "UPDATE %s"
sql_commands["update"] += " SET (day, night, transit) = ('%s','%s','%s')"
sql_commands["update"] += " WHERE (exposure_entity='%s' AND occupancy_case='%s'"
sql_commands["update"] += " AND aggregated_source_id='%s');"
sql_commands["insert"] = "INSERT INTO"
sql_commands["insert"] += " %s(exposure_entity, occupancy_case, aggregated_source_id,"
sql_commands["insert"] += " day, night, transit)"
sql_commands["insert"] += " VALUES('%s','%s','%s','%s','%s','%s');"
population_time_distribution = self.occupancy_cases[occupancy_case][
"population_time_distribution"
]
db_gde_tiles = Database(**db_data_units_config)
db_gde_tiles.create_connection_and_cursor()
# Check if an entry already exists for this exposure entity, occupancy case and
# aggregated source ID
db_gde_tiles.cursor.execute(
sql_commands["query"]
% (
db_table,
self.code,
occupancy_case,
str(aggregated_source_id),
)
)
exec_result = db_gde_tiles.cursor.fetchall()
if exec_result[0][0] > 0: # Entry exists --> update
db_gde_tiles.cursor.execute(
sql_commands["update"]
% (
db_table,
population_time_distribution["Day"],
population_time_distribution["Night"],
population_time_distribution["Transit"],
self.code,
occupancy_case,
str(aggregated_source_id),
)
)
elif exec_result[0][0] == 0: # No entry for this combination exists --> append
db_gde_tiles.cursor.execute(
sql_commands["insert"]
% (
db_table,
self.code,
occupancy_case,
str(aggregated_source_id),
population_time_distribution["Day"],
population_time_distribution["Night"],
population_time_distribution["Transit"],
)
)
else: # More than one entries found, this is an error
logger.error(
"ERROR IN write_people_distribution_in_time_to_database: MORE THAN ONE ENTRY"
" FOUND FOR exposure_entity='%s' AND occupancy_case='%s'"
" AND aggregated_source_id='%s'"
% (self.code, occupancy_case, aggregated_source_id)
)
db_gde_tiles.close_connection()
return
def _interpret_exposure_entities_code(self, config_code): def _interpret_exposure_entities_code(self, config_code):
"""This function interprets the value of exposure_entities_code given as configuration. """This function interprets the value of exposure_entities_code given as configuration.
......
...@@ -78,6 +78,14 @@ def main(): ...@@ -78,6 +78,14 @@ def main():
aem_source_id, aem_source_id,
occupancy_case, occupancy_case,
) )
aem.exposure_entities[
exposure_entity_name
].write_people_distribution_in_time_to_database(
config.database_gde_tiles,
"exposure_entities_population_time_distribution",
aem_source_id,
occupancy_case,
)
aem.exposure_entities[exposure_entity_name].create_data_unit_tiles( aem.exposure_entities[exposure_entity_name].create_data_unit_tiles(
occupancy_case, occupancy_case,
config.number_cores, config.number_cores,
......
...@@ -33,6 +33,7 @@ def test_db(): ...@@ -33,6 +33,7 @@ def test_db():
- data_units (of the GDE Tiles database) - data_units (of the GDE Tiles database)
- data_unit_tiles (of the GDE Tiles database) - data_unit_tiles (of the GDE Tiles database)
- exposure_entities_costs_assumptions (of the GDE Tiles database) - exposure_entities_costs_assumptions (of the GDE Tiles database)
- exposure_entities_population_time_distribution (of the GDE Tiles database)
""" """
init_test_db() init_test_db()
...@@ -41,8 +42,8 @@ def test_db(): ...@@ -41,8 +42,8 @@ def test_db():
def init_test_db(): def init_test_db():
"""Populates the test database that simulates to contain obm_built_area_assessments, """Populates the test database that simulates to contain obm_built_area_assessments,
aggregated_sources, data_units, data_unit_tiles and exposure_entities_costs_assumptions with aggregated_sources, data_units, data_unit_tiles, exposure_entities_costs_assumptions and
a basic schema and data. exposure_entities_population_time_distribution with a basic schema and data.
""" """
if "GDEIMPORTER_DB_HOST" in os.environ: # When running the CI pipeline if "GDEIMPORTER_DB_HOST" in os.environ: # When running the CI pipeline
......
...@@ -3,6 +3,7 @@ DROP TABLE IF EXISTS aggregated_sources; ...@@ -3,6 +3,7 @@ DROP TABLE IF EXISTS aggregated_sources;
DROP TABLE IF EXISTS data_units; DROP TABLE IF EXISTS data_units;
DROP TABLE IF EXISTS data_unit_tiles; DROP TABLE IF EXISTS data_unit_tiles;
DROP TABLE IF EXISTS exposure_entities_costs_assumptions; DROP TABLE IF EXISTS exposure_entities_costs_assumptions;
DROP TABLE IF EXISTS exposure_entities_population_time_distribution;
DROP TYPE IF EXISTS occupancycase; DROP TYPE IF EXISTS occupancycase;
CREATE TYPE occupancycase AS ENUM ('residential', 'commercial', 'industrial'); CREATE TYPE occupancycase AS ENUM ('residential', 'commercial', 'industrial');
...@@ -108,3 +109,22 @@ INSERT INTO exposure_entities_costs_assumptions(exposure_entity, ...@@ -108,3 +109,22 @@ INSERT INTO exposure_entities_costs_assumptions(exposure_entity,
contents, contents,
currency) currency)
VALUES ('GRC','commercial',19,0.0,0.0,0.0,'USD'); VALUES ('GRC','commercial',19,0.0,0.0,0.0,'USD');
CREATE TABLE exposure_entities_population_time_distribution
(
aggregated_source_id SMALLINT,
exposure_entity CHAR(3),
occupancy_case occupancycase,
day FLOAT,
night FLOAT,
transit FLOAT,
PRIMARY KEY (exposure_entity, occupancy_case, aggregated_source_id)
);
INSERT INTO exposure_entities_population_time_distribution(exposure_entity,
occupancy_case,
aggregated_source_id,
day,
night,
transit)
VALUES ('GRC','commercial',19,0.0,0.0,0.0);
...@@ -135,8 +135,9 @@ def query_costs_assumptions(credentials, exposure_entity, occupancy_case, aggreg ...@@ -135,8 +135,9 @@ def query_costs_assumptions(credentials, exposure_entity, occupancy_case, aggreg
Returns: Returns:
result (list of tuples): result (list of tuples):
Each tuple of the list corresponds to one entry in the 'data_units' table of the Each tuple of the list corresponds to one entry in the
test database that matches the search criteria. The elements within the tuple are: 'exposure_entities_costs_assumptions' table of the test database that matches the
search criteria. The elements within the tuple are:
structural (float): structural (float):
Factor to obtain the cost of the structural components. Factor to obtain the cost of the structural components.
non_structural (float): non_structural (float):
...@@ -160,3 +161,128 @@ def query_costs_assumptions(credentials, exposure_entity, occupancy_case, aggreg ...@@ -160,3 +161,128 @@ def query_costs_assumptions(credentials, exposure_entity, occupancy_case, aggreg
result = db_test.cursor.fetchall() result = db_test.cursor.fetchall()
return result return result
def test_ExposureEntity_write_people_distribution_in_time_to_database(test_db):
# Create an exposure entity
returned_exposureentity = ExposureEntity("Greece", "ISO3", "EUR 2020")
returned_exposureentity.occupancy_cases = {
"residential": {
"population_time_distribution": {
"Day": 0.2302575,
"Night": 0.952945,
"Transit": 0.527497,
},
},
"commercial": {
"population_time_distribution": {
"Day": 0.4954065,
"Night": 0.042445,
"Transit": 0.0907965,
},
},
"industrial": {
"population_time_distribution": {
"Day": 0.4954065,
"Night": 0.042445,
"Transit": 0.0907965,
},
},
}
# Load configuration file to retrieve database credentials
config = Configuration(
os.path.join(os.path.dirname(__file__), "data", "config_for_testing_good.yml")
)
for occupancy_case in returned_exposureentity.occupancy_cases:
returned_exposureentity.write_people_distribution_in_time_to_database(
config.database_gde_tiles,
"exposure_entities_population_time_distribution",
19,
occupancy_case,
)
# Test that cost assumptions have been stored correctly
for occupancy_case in returned_exposureentity.occupancy_cases:
query_result = query_people_distribution_in_time(
config.database_gde_tiles, returned_exposureentity.code, occupancy_case, 19
)
assert len(query_result) == 1 # one entry found in the database
assert round(query_result[0][0], 5) == round(
returned_exposureentity.occupancy_cases[occupancy_case][
"population_time_distribution"
]["Day"],
5,
)
assert round(query_result[0][1], 5) == round(
returned_exposureentity.occupancy_cases[occupancy_case][
"population_time_distribution"
]["Night"],
5,
)
assert round(query_result[0][2], 5) == round(
returned_exposureentity.occupancy_cases[occupancy_case][
"population_time_distribution"
]["Transit"],
5,
)
def query_people_distribution_in_time(
credentials, exposure_entity, occupancy_case, aggregated_source_id
):
"""This auxiliary function queries the 'exposure_entities_population_time_distribution'
table of the test database to find all entries corresponding to 'exposure_entity',
'occupancy_case' and 'aggregated_source_id'.
Args:
credentials (dict):
Dictionary containing the credentials needed to connect to the test SQL database.
The keys of the dictionary need to be:
host (str):
SQL database host address.
dbname (str):
Name of the SQL database.
port (int):
Port where the SQL database can be found.
username (str):
User name to connect to the SQL database.
password (str):
Password associated with self.username.
exposure_entity (str):
3-character string identifying an exposure entity.
occupancy_case (str):
Occupancy case.
aggregated_source_id (int):
ID of the aggregated exposure model source associated with exposure_entity.
Returns:
result (list of tuples):
Each tuple of the list corresponds to one entry in the
'exposure_entities_population_time_distribution' table of the test database that
matches the search criteria. The elements within the tuple are:
day (float):
Factor to obtain the number of people expected to be inside the buildings
during the day (approx. 10 am to 6 pm).
night (float):
Factor to obtain the number of people expected to be inside the buildings
during the night (approx. 10 pm to 6 am).
transit (float):
Factor to obtain the number of people expected to be inside the buildings
during transit times (approx. 6 am to 10 am and 6 pm to 10 pm).
"""
sql_command = (
"SELECT day, night, transit"
" FROM exposure_entities_population_time_distribution"
" WHERE (exposure_entity='%s' AND occupancy_case='%s' AND aggregated_source_id='%s');"
% (exposure_entity, occupancy_case, aggregated_source_id)
)
db_test = Database(**credentials)
db_test.create_connection_and_cursor()
db_test.cursor.execute(sql_command)
result = db_test.cursor.fetchall()
return result
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment