Skip to content
Snippets Groups Projects
Commit 00f3e618 authored by Cecilia Nievas's avatar Cecilia Nievas
Browse files

Added feature to store number of OBM and remainder buildings

parent c861cd3d
No related branches found
No related tags found
1 merge request!16Added feature to store number of OBM and remainder buildings
Pipeline #41471 passed
......@@ -87,6 +87,10 @@ database where information on the GDE tiles is stored.
- `database_obm_buildings`: Credentials for the
[OBM Buildings](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/database-obmbuildings)
database where information on the OBM buildings is stored.
- `database_completeness`: Credentials for the
[OBM Tiles](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/database-obmtiles)
database where information on the OSM-completeness of tiles is stored.
- `number_cores`: Number of cores used for parallelisation.
## Running gde-core
......
#!/usr/bin/env python3
# Copyright (C) 2022:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see http://www.gnu.org/licenses/.
import logging
from gdeimporter.tools.database import Database
logger = logging.getLogger()
class DatabaseStorage:
"""This class contains methods used to store values to the Global Dynamic Exposure (GDE)
database.
"""
@staticmethod
def store_number_OBM_and_remainder_buildings(
data_unit_id,
occupancy_case,
aggregated_source_id,
data_unit_tiles,
db_gde_tiles_config,
db_table,
):
"""This function writes to the table with name 'db_table' in the database whose
credentials are indicated in 'db_gde_tiles_config' the number of OBM and remainder
buildings in each data-unit tile contained in 'data_unit_tiles', all of which are
associated with 'data_unit_id', 'occupancy_case' and 'aggregated_source_id'.
Args:
data_unit_id (str):
ID of the data unit for which the data-unit tiles will be retrieved.
occupancy_case (str):
Name of the occupancy case (e.g. "residential", "commercial", "industrial")
for which the data-unit tiles will be retrieved.
aggregated_source_id (int):
ID of the source of the aggregated exposure model for which the data-unit tiles
will be retrieved.
data_unit_tiles:
Pandas DataFrame with data-unit tiles. It contains the following columns:
quadkey (str):
String indicating the quadkey of a tile.
aggregated_buildings (float):
Number of buildings in the data-unit tile as per the aggregated exposure
model with ID 'aggregated_source_id'.
obm_buildings (int):
Number of OBM buildings in the data-unit tile.
remainder_buildings (float):
Number of remainder buildings in the data-unit tile.
complete (bool):
True if the tile is OSM-complete, False if it is OSM-incomplete.
db_gde_tiles_config (dict):
Dictionary containing the credentials needed to connect to the SQL database in
which information on the data-unit tiles is stored. The keys of the dictionary
need to be:
host (str):
SQL database host address.
dbname (str):
Name of the SQL database.
port (int):
Port where the SQL database can be found.
username (str):
User name to connect to the SQL database.
password (str):
Password associated with self.username.
db_table (str):
Name of the table of the SQL database where the data-unit tiles are stored. It
is assumed that this table contains, at least, the following fields:
quadkey (str):
String indicating the quadkey of a tile.
aggregated_source_id (int):
ID of the source of the aggregated exposure model.
occupancy_case (enum):
SQL enumerated type describing the building occupancy cases.
data_unit_id (str):
ID of the data unit.
aggregated_buildings (float):
Number of buildings in the data-unit tile as per the aggregated exposure
model with ID 'aggregated_source_id'.
obm_buildings (int):
Number of OBM buildings in the data-unit tile.
remainder_buildings (float):
Number of remainder buildings in the data-unit tile as per the
aggregated exposure model with ID 'aggregated_source_id'.
"""
sql_commands = {}
sql_commands["query"] = "SELECT COUNT(*) FROM %s"
sql_commands["query"] += " WHERE (quadkey='%s' AND data_unit_id='%s' "
sql_commands["query"] += " AND occupancy_case='%s' AND aggregated_source_id='%s');"
sql_commands["update"] = "UPDATE %s SET (obm_buildings, remainder_buildings)"
sql_commands["update"] += " = (%s,%s)"
sql_commands["update"] += " WHERE (quadkey='%s' AND data_unit_id='%s' "
sql_commands["update"] += " AND occupancy_case='%s' AND aggregated_source_id='%s');"
db_gde_tiles = Database(**db_gde_tiles_config)
db_gde_tiles.create_connection_and_cursor()
for i, quadkey in enumerate(data_unit_tiles["quadkey"].to_numpy()):
db_gde_tiles.cursor.execute(
sql_commands["query"]
% (db_table, quadkey, data_unit_id, occupancy_case, aggregated_source_id)
)
exec_result = db_gde_tiles.cursor.fetchall()
if exec_result[0][0] == 1: # One entry exists (expected)
db_gde_tiles.cursor.execute(
sql_commands["update"]
% (
db_table,
data_unit_tiles["obm_buildings"].to_numpy()[i],
data_unit_tiles["remainder_buildings"].to_numpy()[i],
quadkey,
data_unit_id,
occupancy_case,
aggregated_source_id,
)
)
else:
logger.error(
"DatabaseStorage.store_number_OBM_and_remainder_buildings() has found "
"either more than one entry or no entry for quadkey='%s' AND "
"data_unit_id='%s' AND occupancy_case='%s' AND aggregated_source_id='%s'. "
"Numbers of OBM and remainder buildings were not stored "
"for this data-unit tile."
)
db_gde_tiles.close_connection()
return
......@@ -21,6 +21,7 @@ import sys
from copy import deepcopy
from gdecore.configuration import Configuration
from gdecore.database_queries import DatabaseQueries
from gdecore.database_storage import DatabaseStorage
from gdecore.processor import GDEProcessor
# Add a logger printing error, warning, info and debug messages to the screen
......@@ -189,6 +190,16 @@ def main():
config.number_cores,
)
# Store number of OBM and remainder buildings of the data-unit tiles
DatabaseStorage.store_number_OBM_and_remainder_buildings(
data_unit_id,
occupancy_case,
aggregated_source_id,
data_unit_tiles,
config.database_gde_tiles,
"data_unit_tiles",
)
# Leave the program
logger.info("gde-core has finished")
sys.exit()
......
......@@ -771,6 +771,8 @@ class GDEProcessor:
aggregated_buildings (float):
Number of buildings in the data-unit tile as per the aggregated exposure
model with ID 'aggregated_source_id'.
obm_buildings (int):
Number of OBM buildings in the data-unit tile.
remainder_buildings (float):
Number of remainder buildings in the data-unit tile.
complete (bool):
......
......@@ -179,6 +179,8 @@ CREATE TABLE data_unit_tiles
fraction_data_unit_area FLOAT,
fraction_data_unit_built_up_area FLOAT,
aggregated_buildings FLOAT,
obm_buildings SMALLINT,
remainder_buildings FLOAT,
PRIMARY KEY (quadkey, aggregated_source_id, occupancy_case, data_unit_id)
);
......@@ -197,7 +199,11 @@ VALUES ('122010321033023130', 2, 'residential', 'ABC', 'ABC_10269', 0.0, 0.0, 0.
('122010321033023120', 2, 'residential', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 39.1),
('122010321033023120', 2, 'commercial', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 17.6),
('122010321033023132', 2, 'residential', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 34.4),
('122010321033023132', 2, 'commercial', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 11.5);
('122010321033023132', 2, 'commercial', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 11.5),
('122010321033023121', 2, 'residential', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 26.2),
('122010321033023121', 2, 'commercial', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 0.0),
('122010321033023123', 2, 'residential', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 16.5),
('122010321033023123', 2, 'commercial', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 0.0);
CREATE TABLE obm_built_area_assessments
(
......
......@@ -348,11 +348,17 @@ def test_get_data_unit_tiles_of_data_unit_as_DataFrame(test_db):
os.path.join(os.path.dirname(__file__), "data", "config_for_testing_good.yml")
)
expected_quadkeys = ["122010321033023130", "122010321033023120", "122010321033023132"]
expected_quadkeys = [
"122010321033023130",
"122010321033023120",
"122010321033023132",
"122010321033023121",
"122010321033023123",
]
expected_aggregated_buildings = {}
expected_aggregated_buildings["residential"] = [15.7, 39.1, 34.4]
expected_aggregated_buildings["commercial"] = [23.4, 17.6, 11.5]
expected_aggregated_buildings["residential"] = [15.7, 39.1, 34.4, 26.2, 16.5]
expected_aggregated_buildings["commercial"] = [23.4, 17.6, 11.5, 0.0, 0.0]
for occupancy in expected_aggregated_buildings:
returned_data_unit_tiles = (
......
#!/usr/bin/env python3
# Copyright (C) 2022:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see http://www.gnu.org/licenses/.
import os
import pandas
from gdeimporter.tools.database import Database
from gdecore.configuration import Configuration
from gdecore.database_storage import DatabaseStorage
def test_store_number_OBM_and_remainder_buildings(test_db):
# Database connection (the Configuration class will define the credentials based on whether
# the code is running in the CI or locally)
config = Configuration(
os.path.join(os.path.dirname(__file__), "data", "config_for_testing_good.yml")
)
data_unit_tiles = pandas.DataFrame(
{
"quadkey": [
"122010321033023120",
"122010321033023132",
"122010321033023130",
"122010321033023121",
"122010321033023123",
],
"aggregated_buildings": [39.1, 34.4, 15.7, 26.2, 16.5],
"obm_buildings": [41, 12, 3, 0, 0],
"remainder_buildings": [0.0, 0.0, 12.7, 26.2, 0.0],
}
)
DatabaseStorage.store_number_OBM_and_remainder_buildings(
"ABC_10269",
"residential",
2,
data_unit_tiles,
config.database_gde_tiles,
"data_unit_tiles",
)
for i, quadkey in enumerate(data_unit_tiles["quadkey"].to_numpy()):
returned_obm_buildings, returned_remainder_buildings = query_obm_and_remainder(
config.database_gde_tiles, quadkey, "ABC_10269", "residential", 2
)
assert returned_obm_buildings == data_unit_tiles["obm_buildings"].to_numpy()[i]
assert round(returned_remainder_buildings, 2) == round(
data_unit_tiles["remainder_buildings"].to_numpy()[i], 2
)
def query_obm_and_remainder(
credentials, quadkey, data_unit_id_full, occupancy_case, aggregated_source_id
):
"""This auxiliary function queries the 'data_unit_tiles' table of the test database to
retrieve the number of OBM and remainder buildings of the entry corresponding to 'quadkey',
'data_unit_id_full', 'occupancy_case' and 'aggregated_source_id'.
Args:
credentials (dict):
Dictionary containing the credentials needed to connect to the test SQL database.
The keys of the dictionary need to be:
host (str):
SQL database host address.
dbname (str):
Name of the SQL database.
port (int):
Port where the SQL database can be found.
username (str):
User name to connect to the SQL database.
password (str):
Password associated with self.username.
quadkey (str):
Zoom level 18 tile identifier of the data-unit tile.
data_unit_id_full (str):
ID of the Data Unit of the data-unit tile, including the 3-character code of its
corresponding exposure entity.
occupancy_case (str):
Occupancy case of the data-unit tile.
aggregated_source_id (int):
ID of the aggregated exposure model source associated with the data-unit tile.
Returns:
obm_buildings (int):
Number of OBM buildings in the data-unit tile.
remainder_buildings (float):
Number of remainder buildings in the data-unit tile.
"""
sql_command = (
"SELECT obm_buildings, remainder_buildings FROM data_unit_tiles "
"WHERE (quadkey='%s' AND data_unit_id='%s' AND occupancy_case='%s' "
"AND aggregated_source_id='%s');"
% (quadkey, data_unit_id_full, occupancy_case, aggregated_source_id)
)
db_test = Database(**credentials)
db_test.create_connection_and_cursor()
db_test.cursor.execute(sql_command)
result = db_test.cursor.fetchall()
obm_buildings = result[0][0]
remainder_buildings = result[0][1]
db_test.close_connection()
return obm_buildings, remainder_buildings
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment