Added feature to store number of OBM and remainder buildings

README.md

@@ -87,6 +87,10 @@ database where information on the GDE tiles is stored.
 - `database_obm_buildings`: Credentials for the 
 [OBM Buildings](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/database-obmbuildings) 
 database where information on the OBM buildings is stored.
+- `database_completeness`: Credentials for the 
+[OBM Tiles](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/database-obmtiles)
+database where information on the OSM-completeness of tiles is stored.
+- `number_cores`: Number of cores used for parallelisation.
 
 ## Running gde-core
 

gdecore/database_storage.py

+#!/usr/bin/env python3
+
+# Copyright (C) 2022:
+#   Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see http://www.gnu.org/licenses/.
+
+import logging
+from gdeimporter.tools.database import Database
+
+
+logger = logging.getLogger()
+
+
+class DatabaseStorage:
+    """This class contains methods used to store values to the Global Dynamic Exposure (GDE)
+    database.
+    """
+
+    @staticmethod
+    def store_number_OBM_and_remainder_buildings(
+        data_unit_id,
+        occupancy_case,
+        aggregated_source_id,
+        data_unit_tiles,
+        db_gde_tiles_config,
+        db_table,
+    ):
+        """This function writes to the table with name 'db_table' in the database whose
+        credentials are indicated in 'db_gde_tiles_config' the number of OBM and remainder
+        buildings in each data-unit tile contained in 'data_unit_tiles', all of which are
+        associated with 'data_unit_id', 'occupancy_case' and 'aggregated_source_id'.
+
+        Args:
+            data_unit_id (str):
+                 ID of the data unit for which the data-unit tiles will be retrieved.
+            occupancy_case (str):
+                Name of the occupancy case (e.g. "residential", "commercial", "industrial")
+                for which the data-unit tiles will be retrieved.
+            aggregated_source_id (int):
+                ID of the source of the aggregated exposure model for which the data-unit tiles
+                will be retrieved.
+            data_unit_tiles:
+                Pandas DataFrame with data-unit tiles. It contains the following columns:
+                    quadkey (str):
+                        String indicating the quadkey of a tile.
+                    aggregated_buildings (float):
+                        Number of buildings in the data-unit tile as per the aggregated exposure
+                        model with ID 'aggregated_source_id'.
+                    obm_buildings (int):
+                        Number of OBM buildings in the data-unit tile.
+                    remainder_buildings (float):
+                        Number of remainder buildings in the data-unit tile.
+                    complete (bool):
+                        True if the tile is OSM-complete, False if it is OSM-incomplete.
+            db_gde_tiles_config (dict):
+                Dictionary containing the credentials needed to connect to the SQL database in
+                which information on the data-unit tiles is stored. The keys of the dictionary
+                need to be:
+                    host (str):
+                        SQL database host address.
+                    dbname (str):
+                        Name of the SQL database.
+                    port (int):
+                        Port where the SQL database can be found.
+                    username (str):
+                        User name to connect to the SQL database.
+                    password (str):
+                        Password associated with self.username.
+            db_table (str):
+                Name of the table of the SQL database where the data-unit tiles are stored. It
+                is assumed that this table contains, at least, the following fields:
+                    quadkey (str):
+                        String indicating the quadkey of a tile.
+                    aggregated_source_id (int):
+                        ID of the source of the aggregated exposure model.
+                    occupancy_case (enum):
+                        SQL enumerated type describing the building occupancy cases.
+                    data_unit_id (str):
+                        ID of the data unit.
+                    aggregated_buildings (float):
+                        Number of buildings in the data-unit tile as per the aggregated exposure
+                        model with ID 'aggregated_source_id'.
+                    obm_buildings (int):
+                        Number of OBM buildings in the data-unit tile.
+                    remainder_buildings (float):
+                        Number of remainder buildings in the data-unit tile as per the
+                        aggregated exposure model with ID 'aggregated_source_id'.
+        """
+
+        sql_commands = {}
+
+        sql_commands["query"] = "SELECT COUNT(*) FROM %s"
+        sql_commands["query"] += " WHERE (quadkey='%s' AND data_unit_id='%s' "
+        sql_commands["query"] += " AND occupancy_case='%s' AND aggregated_source_id='%s');"
+
+        sql_commands["update"] = "UPDATE %s SET (obm_buildings, remainder_buildings)"
+        sql_commands["update"] += " = (%s,%s)"
+        sql_commands["update"] += " WHERE (quadkey='%s' AND data_unit_id='%s' "
+        sql_commands["update"] += " AND occupancy_case='%s' AND aggregated_source_id='%s');"
+
+        db_gde_tiles = Database(**db_gde_tiles_config)
+        db_gde_tiles.create_connection_and_cursor()
+
+        for i, quadkey in enumerate(data_unit_tiles["quadkey"].to_numpy()):
+            db_gde_tiles.cursor.execute(
+                sql_commands["query"]
+                % (db_table, quadkey, data_unit_id, occupancy_case, aggregated_source_id)
+            )
+            exec_result = db_gde_tiles.cursor.fetchall()
+
+            if exec_result[0][0] == 1:  # One entry exists (expected)
+                db_gde_tiles.cursor.execute(
+                    sql_commands["update"]
+                    % (
+                        db_table,
+                        data_unit_tiles["obm_buildings"].to_numpy()[i],
+                        data_unit_tiles["remainder_buildings"].to_numpy()[i],
+                        quadkey,
+                        data_unit_id,
+                        occupancy_case,
+                        aggregated_source_id,
+                    )
+                )
+            else:
+                logger.error(
+                    "DatabaseStorage.store_number_OBM_and_remainder_buildings() has found "
+                    "either more than one entry or no entry for quadkey='%s' AND "
+                    "data_unit_id='%s' AND occupancy_case='%s' AND aggregated_source_id='%s'. "
+                    "Numbers of OBM and remainder buildings were not stored "
+                    "for this data-unit tile."
+                )
+
+        db_gde_tiles.close_connection()
+
+        return

gdecore/gdecore.py

@@ -21,6 +21,7 @@ import sys
 from copy import deepcopy
 from gdecore.configuration import Configuration
 from gdecore.database_queries import DatabaseQueries
+from gdecore.database_storage import DatabaseStorage
 from gdecore.processor import GDEProcessor
 
 # Add a logger printing error, warning, info and debug messages to the screen
@@ -189,6 +190,16 @@ def main():
                     config.number_cores,
                 )
 
+                # Store number of OBM and remainder buildings of the data-unit tiles
+                DatabaseStorage.store_number_OBM_and_remainder_buildings(
+                    data_unit_id,
+                    occupancy_case,
+                    aggregated_source_id,
+                    data_unit_tiles,
+                    config.database_gde_tiles,
+                    "data_unit_tiles",
+                )
+
     # Leave the program
     logger.info("gde-core has finished")
     sys.exit()

gdecore/processor.py

@@ -771,6 +771,8 @@ class GDEProcessor:
                     aggregated_buildings (float):
                         Number of buildings in the data-unit tile as per the aggregated exposure
                         model with ID 'aggregated_source_id'.
+                    obm_buildings (int):
+                        Number of OBM buildings in the data-unit tile.
                     remainder_buildings (float):
                         Number of remainder buildings in the data-unit tile.
                     complete (bool):

tests/data/test_database_set_up.sql

@@ -179,6 +179,8 @@ CREATE TABLE data_unit_tiles
     fraction_data_unit_area FLOAT,
     fraction_data_unit_built_up_area FLOAT,
     aggregated_buildings FLOAT,
+    obm_buildings SMALLINT,
+    remainder_buildings FLOAT,
 
     PRIMARY KEY (quadkey, aggregated_source_id, occupancy_case, data_unit_id)
 );
@@ -197,7 +199,11 @@ VALUES ('122010321033023130', 2, 'residential', 'ABC', 'ABC_10269', 0.0, 0.0, 0.
 ('122010321033023120', 2, 'residential', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 39.1),
 ('122010321033023120', 2, 'commercial', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 17.6),
 ('122010321033023132', 2, 'residential', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 34.4),
-('122010321033023132', 2, 'commercial', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 11.5);
+('122010321033023132', 2, 'commercial', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 11.5),
+('122010321033023121', 2, 'residential', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 26.2),
+('122010321033023121', 2, 'commercial', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 0.0),
+('122010321033023123', 2, 'residential', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 16.5),
+('122010321033023123', 2, 'commercial', 'ABC', 'ABC_10269', 0.0, 0.0, 0.0, 0.0, 0.0);
 
 CREATE TABLE obm_built_area_assessments
 (

tests/test_database_queries.py

@@ -348,11 +348,17 @@ def test_get_data_unit_tiles_of_data_unit_as_DataFrame(test_db):
         os.path.join(os.path.dirname(__file__), "data", "config_for_testing_good.yml")
     )
 
-    expected_quadkeys = ["122010321033023130", "122010321033023120", "122010321033023132"]
+    expected_quadkeys = [
+        "122010321033023130",
+        "122010321033023120",
+        "122010321033023132",
+        "122010321033023121",
+        "122010321033023123",
+    ]
 
     expected_aggregated_buildings = {}
-    expected_aggregated_buildings["residential"] = [15.7, 39.1, 34.4]
-    expected_aggregated_buildings["commercial"] = [23.4, 17.6, 11.5]
+    expected_aggregated_buildings["residential"] = [15.7, 39.1, 34.4, 26.2, 16.5]
+    expected_aggregated_buildings["commercial"] = [23.4, 17.6, 11.5, 0.0, 0.0]
 
     for occupancy in expected_aggregated_buildings:
         returned_data_unit_tiles = (

tests/test_database_storage.py

+#!/usr/bin/env python3
+
+# Copyright (C) 2022:
+#   Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see http://www.gnu.org/licenses/.
+
+import os
+import pandas
+from gdeimporter.tools.database import Database
+from gdecore.configuration import Configuration
+from gdecore.database_storage import DatabaseStorage
+
+
+def test_store_number_OBM_and_remainder_buildings(test_db):
+    # Database connection (the Configuration class will define the credentials based on whether
+    # the code is running in the CI or locally)
+    config = Configuration(
+        os.path.join(os.path.dirname(__file__), "data", "config_for_testing_good.yml")
+    )
+
+    data_unit_tiles = pandas.DataFrame(
+        {
+            "quadkey": [
+                "122010321033023120",
+                "122010321033023132",
+                "122010321033023130",
+                "122010321033023121",
+                "122010321033023123",
+            ],
+            "aggregated_buildings": [39.1, 34.4, 15.7, 26.2, 16.5],
+            "obm_buildings": [41, 12, 3, 0, 0],
+            "remainder_buildings": [0.0, 0.0, 12.7, 26.2, 0.0],
+        }
+    )
+
+    DatabaseStorage.store_number_OBM_and_remainder_buildings(
+        "ABC_10269",
+        "residential",
+        2,
+        data_unit_tiles,
+        config.database_gde_tiles,
+        "data_unit_tiles",
+    )
+
+    for i, quadkey in enumerate(data_unit_tiles["quadkey"].to_numpy()):
+        returned_obm_buildings, returned_remainder_buildings = query_obm_and_remainder(
+            config.database_gde_tiles, quadkey, "ABC_10269", "residential", 2
+        )
+
+        assert returned_obm_buildings == data_unit_tiles["obm_buildings"].to_numpy()[i]
+        assert round(returned_remainder_buildings, 2) == round(
+            data_unit_tiles["remainder_buildings"].to_numpy()[i], 2
+        )
+
+
+def query_obm_and_remainder(
+    credentials, quadkey, data_unit_id_full, occupancy_case, aggregated_source_id
+):
+    """This auxiliary function queries the 'data_unit_tiles' table of the test database to
+    retrieve the number of OBM and remainder buildings of the entry corresponding to 'quadkey',
+    'data_unit_id_full', 'occupancy_case' and 'aggregated_source_id'.
+
+    Args:
+        credentials (dict):
+            Dictionary containing the credentials needed to connect to the test SQL database.
+            The keys of the dictionary need to be:
+                host (str):
+                    SQL database host address.
+                dbname (str):
+                    Name of the SQL database.
+                port (int):
+                    Port where the SQL database can be found.
+                username (str):
+                    User name to connect to the SQL database.
+                password (str):
+                    Password associated with self.username.
+        quadkey (str):
+            Zoom level 18 tile identifier of the data-unit tile.
+        data_unit_id_full (str):
+            ID of the Data Unit of the data-unit tile, including the 3-character code of its
+            corresponding exposure entity.
+        occupancy_case (str):
+            Occupancy case of the data-unit tile.
+        aggregated_source_id (int):
+            ID of the aggregated exposure model source associated with the data-unit tile.
+
+    Returns:
+        obm_buildings (int):
+            Number of OBM buildings in the data-unit tile.
+        remainder_buildings (float):
+            Number of remainder buildings in the data-unit tile.
+    """
+
+    sql_command = (
+        "SELECT obm_buildings, remainder_buildings FROM data_unit_tiles "
+        "WHERE (quadkey='%s' AND data_unit_id='%s' AND occupancy_case='%s' "
+        "AND aggregated_source_id='%s');"
+        % (quadkey, data_unit_id_full, occupancy_case, aggregated_source_id)
+    )
+
+    db_test = Database(**credentials)
+    db_test.create_connection_and_cursor()
+    db_test.cursor.execute(sql_command)
+    result = db_test.cursor.fetchall()
+
+    obm_buildings = result[0][0]
+    remainder_buildings = result[0][1]
+
+    db_test.close_connection()
+
+    return obm_buildings, remainder_buildings