Commit 00f3e618 authored by Cecilia Nievas's avatar Cecilia Nievas
Browse files

Added feature to store number of OBM and remainder buildings

parent c861cd3d
Pipeline #41471 passed with stage
in 2 minutes and 28 seconds
......@@ -87,6 +87,10 @@ database where information on the GDE tiles is stored.
- `database_obm_buildings`: Credentials for the
[OBM Buildings](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/database-obmbuildings)
database where information on the OBM buildings is stored.
- `database_completeness`: Credentials for the
[OBM Tiles](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/database-obmtiles)
database where information on the OSM-completeness of tiles is stored.
- `number_cores`: Number of cores used for parallelisation.
## Running gde-core
......
#!/usr/bin/env python3
# Copyright (C) 2022:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see http://www.gnu.org/licenses/.
import logging
from gdeimporter.tools.database import Database
logger = logging.getLogger()
class DatabaseStorage:
"""This class contains methods used to store values to the Global Dynamic Exposure (GDE)
database.
"""
@staticmethod
def store_number_OBM_and_remainder_buildings(
data_unit_id,
occupancy_case,
aggregated_source_id,
data_unit_tiles,
db_gde_tiles_config,
db_table,
):
"""This function writes to the table with name 'db_table' in the database whose
credentials are indicated in 'db_gde_tiles_config' the number of OBM and remainder
buildings in each data-unit tile contained in 'data_unit_tiles', all of which are
associated with 'data_unit_id', 'occupancy_case' and 'aggregated_source_id'.
Args:
data_unit_id (str):
ID of the data unit for which the data-unit tiles will be retrieved.
occupancy_case (str):
Name of the occupancy case (e.g. "residential", "commercial", "industrial")
for which the data-unit tiles will be retrieved.
aggregated_source_id (int):
ID of the source of the aggregated exposure model for which the data-unit tiles
will be retrieved.
data_unit_tiles:
Pandas DataFrame with data-unit tiles. It contains the following columns:
quadkey (str):
String indicating the quadkey of a tile.
aggregated_buildings (float):
Number of buildings in the data-unit tile as per the aggregated exposure
model with ID 'aggregated_source_id'.
obm_buildings (int):
Number of OBM buildings in the data-unit tile.
remainder_buildings (float):
Number of remainder buildings in the data-unit tile.
complete (bool):
True if the tile is OSM-complete, False if it is OSM-incomplete.
db_gde_tiles_config (dict):
Dictionary containing the credentials needed to connect to the SQL database in
which information on the data-unit tiles is stored. The keys of the dictionary
need to be:
host (str):
SQL database host address.
dbname (str):
Name of the SQL database.
port (int):
Port where the SQL database can be found.
username (str):
User name to connect to the SQL database.
password (str):
Password associated with self.username.
db_table (str):
Name of the table of the SQL database where the data-unit tiles are stored. It
is assumed that this table contains, at least, the following fields:
quadkey (str):
String indicating the quadkey of a tile.
aggregated_source_id (int):
ID of the source of the aggregated exposure model.
occupancy_case (enum):
SQL enumerated type describing the building occupancy cases.
data_unit_id (str):
ID of the data unit.
aggregated_buildings (float):
Number of buildings in the data-unit tile as per the aggregated exposure
model with ID 'aggregated_source_id'.
obm_buildings (int):
Number of OBM buildings in the data-unit tile.
remainder_buildings (float):
Number of remainder buildings in the data-unit tile as per the
aggregated exposure model with ID 'aggregated_source_id'.
"""
sql_commands = {}
sql_commands["query"] = "SELECT COUNT(*) FROM %s"
sql_commands["query"] += " WHERE (quadkey='%s' AND data_unit_id='%s' "
sql_commands["query"] += " AND occupancy_case='%s' AND aggregated_source_id='%s');"
sql_commands["update"] = "UPDATE %s SET (obm_buildings, remainder_buildings)"
sql_commands["update"] += " = (%s,%s)"
sql_commands["update"] += " WHERE (quadkey='%s' AND data_unit_id='%s' "
sql_commands["update"] += " AND occupancy_case='%s' AND aggregated_source_id='%s');"
db_gde_tiles = Database(**db_gde_tiles_config)
db_gde_tiles.create_connection_and_cursor()
for i, quadkey in enumerate(data_unit_tiles["quadkey"].to_numpy()):
db_gde_tiles.cursor.execute(
sql_commands["query"]
% (db_table, quadkey, data_unit_id, occupancy_case, aggregated_source_id)
)
exec_result = db_gde_tiles.cursor.fetchall()
if exec_result[0][0] == 1: # One entry exists (expected)
db_gde_tiles.cursor.execute(
sql_commands["update"]
% (
db_table,
data_unit_tiles["obm_buildings"].to_numpy()[i],
data_unit_tiles["remainder_buildings"].to_numpy()[i],
quadkey,
data_unit_id,
occupancy_case,
aggregated_source_id,
)
)
else:
logger.error(
"DatabaseStorage.store_number_OBM_and_remainder_buildings() has found "
"either more than one entry or no entry for quadkey='%s' AND "
"data_unit_id='%s' AND occupancy_case='%s' AND aggregated_source_id='%s'. "
"Numbers of OBM and remainder buildings were not stored "
"for this data-unit tile."
)
db_gde_tiles.close_connection()
return
......@@ -21,6 +21,7 @@ import sys
from copy import deepcopy
from gdecore.configuration import Configuration
from gdecore.database_queries import DatabaseQueries
from gdecore.database_storage import DatabaseStorage
from gdecore.processor import GDEProcessor
# Add a logger printing error, warning, info and debug messages to the screen
......@@ -189,6 +190,16 @@ def main():
config.number_cores,
)
# Store number of OBM and remainder buildings of the data-unit tiles
DatabaseStorage.store_number_OBM_and_remainder_buildings(
data_unit_id,
occupancy_case,
aggregated_source_id,
data_unit_tiles,
config.database_gde_tiles,
"data_unit_tiles",
)
# Leave the program
logger.info("gde-core has finished")
sys.exit()
......
......@@ -771,6 +771,8 @@ class GDEProcessor:
aggregated_buildings (float):
Number of buildings in the data-unit tile as per the aggregated exposure
model with ID 'aggregated_source_id'.
obm_buildings (int):
Number of OBM buildings in the data-unit tile.
remainder_buildings (float):
Number of remainder buildings in the data-unit tile.
complete (bool):
......
......@@ -179,6 +179,8 @@ CREATE TABLE data_unit_tiles
fraction_data_unit_area FLOAT,
fraction_data_unit_built_up_area FLOAT,
aggregated_buildings FLOAT,
obm_buildings SMALLINT,
remainder_buildings FLOAT,
PRIMARY KEY (quadkey, aggregated_source_id, occupancy_case, data_unit_id)
);
......@@ -197,7 +199,11 @@ VALUES ('122010321033023130', 2, 'residential', 'ABC', 'ABC_10269', 0.0, 0.0, 0.
('122010321033023120', 2, 'residential', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 39.1),
('122010321033023120', 2, 'commercial', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 17.6),
('122010321033023132', 2, 'residential', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 34.4),
('122010321033023132', 2, 'commercial', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 11.5);
('122010321033023132', 2, 'commercial', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 11.5),
('122010321033023121', 2, 'residential', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 26.2),
('122010321033023121', 2, 'commercial', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 0.0),
('122010321033023123', 2, 'residential', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 16.5),
('122010321033023123', 2, 'commercial', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 0.0);
CREATE TABLE obm_built_area_assessments
(
......
......@@ -348,11 +348,17 @@ def test_get_data_unit_tiles_of_data_unit_as_DataFrame(test_db):
os.path.join(os.path.dirname(__file__), "data", "config_for_testing_good.yml")
)
expected_quadkeys = ["122010321033023130", "122010321033023120", "122010321033023132"]
expected_quadkeys = [
"122010321033023130",
"122010321033023120",
"122010321033023132",
"122010321033023121",
"122010321033023123",
]
expected_aggregated_buildings = {}
expected_aggregated_buildings["residential"] = [15.7, 39.1, 34.4]
expected_aggregated_buildings["commercial"] = [23.4, 17.6, 11.5]
expected_aggregated_buildings["residential"] = [15.7, 39.1, 34.4, 26.2, 16.5]
expected_aggregated_buildings["commercial"] = [23.4, 17.6, 11.5, 0.0, 0.0]
for occupancy in expected_aggregated_buildings:
returned_data_unit_tiles = (
......
#!/usr/bin/env python3
# Copyright (C) 2022:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see http://www.gnu.org/licenses/.
import os
import pandas
from gdeimporter.tools.database import Database
from gdecore.configuration import Configuration
from gdecore.database_storage import DatabaseStorage
def test_store_number_OBM_and_remainder_buildings(test_db):
# Database connection (the Configuration class will define the credentials based on whether
# the code is running in the CI or locally)
config = Configuration(
os.path.join(os.path.dirname(__file__), "data", "config_for_testing_good.yml")
)
data_unit_tiles = pandas.DataFrame(
{
"quadkey": [
"122010321033023120",
"122010321033023132",
"122010321033023130",
"122010321033023121",
"122010321033023123",
],
"aggregated_buildings": [39.1, 34.4, 15.7, 26.2, 16.5],
"obm_buildings": [41, 12, 3, 0, 0],
"remainder_buildings": [0.0, 0.0, 12.7, 26.2, 0.0],
}
)
DatabaseStorage.store_number_OBM_and_remainder_buildings(
"ABC_10269",
"residential",
2,
data_unit_tiles,
config.database_gde_tiles,
"data_unit_tiles",
)
for i, quadkey in enumerate(data_unit_tiles["quadkey"].to_numpy()):
returned_obm_buildings, returned_remainder_buildings = query_obm_and_remainder(
config.database_gde_tiles, quadkey, "ABC_10269", "residential", 2
)
assert returned_obm_buildings == data_unit_tiles["obm_buildings"].to_numpy()[i]
assert round(returned_remainder_buildings, 2) == round(
data_unit_tiles["remainder_buildings"].to_numpy()[i], 2
)
def query_obm_and_remainder(
credentials, quadkey, data_unit_id_full, occupancy_case, aggregated_source_id
):
"""This auxiliary function queries the 'data_unit_tiles' table of the test database to
retrieve the number of OBM and remainder buildings of the entry corresponding to 'quadkey',
'data_unit_id_full', 'occupancy_case' and 'aggregated_source_id'.
Args:
credentials (dict):
Dictionary containing the credentials needed to connect to the test SQL database.
The keys of the dictionary need to be:
host (str):
SQL database host address.
dbname (str):
Name of the SQL database.
port (int):
Port where the SQL database can be found.
username (str):
User name to connect to the SQL database.
password (str):
Password associated with self.username.
quadkey (str):
Zoom level 18 tile identifier of the data-unit tile.
data_unit_id_full (str):
ID of the Data Unit of the data-unit tile, including the 3-character code of its
corresponding exposure entity.
occupancy_case (str):
Occupancy case of the data-unit tile.
aggregated_source_id (int):
ID of the aggregated exposure model source associated with the data-unit tile.
Returns:
obm_buildings (int):
Number of OBM buildings in the data-unit tile.
remainder_buildings (float):
Number of remainder buildings in the data-unit tile.
"""
sql_command = (
"SELECT obm_buildings, remainder_buildings FROM data_unit_tiles "
"WHERE (quadkey='%s' AND data_unit_id='%s' AND occupancy_case='%s' "
"AND aggregated_source_id='%s');"
% (quadkey, data_unit_id_full, occupancy_case, aggregated_source_id)
)
db_test = Database(**credentials)
db_test.create_connection_and_cursor()
db_test.cursor.execute(sql_command)
result = db_test.cursor.fetchall()
obm_buildings = result[0][0]
remainder_buildings = result[0][1]
db_test.close_connection()
return obm_buildings, remainder_buildings
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment