Convert stories to the GEM Taxonomy height attribute

.gitlab-ci.yml

@@ -11,11 +11,11 @@ cache:
     - venv/
 
 stages:
-  - linters
+  - tests
   - release
 
 linters:
-  stage: linters
+  stage: tests
   before_script:
     - python3 -V
     - pip3 install virtualenv
@@ -26,6 +26,20 @@ linters:
     - flake8 --max-line-length=$LINE_LENGTH $SOURCES
     - black --check --line-length $LINE_LENGTH $SOURCES
 
+tests:
+  stage: tests
+  before_script:
+    - python3 -V
+    - pip3 install virtualenv
+    - virtualenv venv
+    - source venv/bin/activate
+    - pip3 install pytest
+    - pip3 install https://git.gfz-potsdam.de/globaldynamicexposure/libraries/rule-lib/-/archive/main/rule-lib-main.zip
+  script:
+    - echo "The project runs in '$CI_PROJECT_DIR'"
+    - echo "The project path is '$CI_PROJECT_PATH'"
+    - pytest tests
+
 release_job:
   stage: release
   image: registry.gitlab.com/gitlab-org/release-cli:latest

building/01_select/ghsl_characteristics_information/ghsl_charateristics_information.py

+#!/usr/bin/env python3
+
+# Copyright (C) 2023:
+#   Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see http://www.gnu.org/licenses/.
+
+
+class GHSLCharacteristicsRule:
+    def __call__(self, database, geometry, *args, **kwargs):
+        """
+        Intersect the building geometry with the GHSL characteristics layer and return the type
+        with the most overlap.
+
+        Args:
+            database (PostGISDatabase):
+                OSM Replication database.
+            geometry (string):
+                The WKT representation of the geometry of the building.
+
+        Returns:
+            A dictionary with the most overlapping GHSL characteristics type.
+        """
+
+        # Get the total area of the intersection between a GHSL type and the building.
+        get_area_function = f"""
+            SUM(
+                ST_Area(
+                    ST_Intersection(geom, ST_Buffer(ST_GeomFromText('{geometry}', 4326),0)
+                ), True)
+            ) AS area
+            """
+
+        # Select the type that most overlaps the building.
+        sql_statement = f"""
+            SELECT type
+            FROM (
+                SELECT type, {get_area_function}
+                FROM ghsl_characteristics
+                WHERE geom && ST_GeomFromText('{geometry}', 4326)
+                GROUP BY type
+            ) AS ghsl_area
+            ORDER BY area DESC
+            LIMIT 1
+            """
+
+        database.cursor.execute(sql_statement)
+        result = database.cursor.fetchone()
+
+        if result:
+            return {"ghsl_characteristics_type": result[0]}
+        else:
+            return {"ghsl_characteristics_type": None}

building/01_select/ghsl_characteristics_information/rule.xml

+<?xml version="1.0" encoding="UTF-8" ?>
+<rule name="GHSLCharacteristicsRule" category="building">
+    <input>
+        <param type="PostGISDatabase">database</param>
+        <param type="string">geometry</param>
+    </input>
+    <function filepath="ghsl_characteristics_information.py"/>
+    <dependencies>
+        <dependency name="ObmBuildingsInformationRule"/>
+    </dependencies>
+
+    <output>
+        <param type="dict">ghsl_characteristics_information</param>
+    </output>
+</rule>

building/01_select/obm_buildings_information/obm_buildings_information.py

@@ -17,7 +17,7 @@
 # along with this program. If not, see http://www.gnu.org/licenses/.
 
 
-class ObmBuildingsInformation:
+class ObmBuildingsInformationRule:
     def __call__(self, database, key, *args, **kwargs):
         """
         Get the information of one building, including the attributes of the building, the
@@ -43,7 +43,7 @@ class ObmBuildingsInformation:
                 hstore_to_matrix(obp.tags) AS tags,
                 ST_X(ST_Centroid(geometry)) AS longitude,
                 ST_Y(ST_Centroid(geometry)) AS latitude,
-                ST_AsText(geometry) AS geometry,
+                ST_AsText(geometry) AS geometry_wkt,
                 ST_Area(geometry, True) AS area
             FROM osm_building_polygons AS obp
             WHERE obp.osm_id = {key}

building/01_select/obm_buildings_information/rule.xml

 <?xml version="1.0" encoding="UTF-8" ?>
-<rule name="ObmBuildingsInformation" category="building">
+<rule name="ObmBuildingsInformationRule" category="building">
     <input>
         <param type="PostGISDatabase">database</param>
         <param type="int">key</param>

building/02_process/geometry/geometry.py

 class GeometryRule:
-    def __call__(self, geometry, *args, **kwargs):
+    def __call__(self, geometry_wkt, *args, **kwargs):
         """
         Wrap the WKT formatted geometry of a building in the `ST_GeomFromText()` function
 
@@ -12,4 +12,4 @@ class GeometryRule:
             `ST_GeomFromText()` function.
         """
 
-        return {"geometry": f"ST_GeomFromText('{geometry}', 4326)"}
+        return {"geometry": f"ST_GeomFromText('{geometry_wkt}', 4326)"}

building/02_process/geometry/rule.xml

 <?xml version="1.0" encoding="UTF-8" ?>
 <rule name="GeometryRule">
     <input>
-        <param type="str">geometry</param>
+        <param type="str">geometry_wkt</param>
     </input>
     <function filepath="geometry.py"/>
     <output>
-        <param type="str">geometry</param>
+        <param type="str">geometry_wkt</param>
     </output>
 </rule>

building/02_process/height_and_floorspace/height_and_floorspace.py

+#!/usr/bin/env python3
+
+# Copyright (c) 2023-2024:
+#   Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see http://www.gnu.org/licenses/.
+
+
+class HeightAndFloorspaceRule:
+    def __call__(self, tags, area, ghsl_characteristics_type, *args, **kwargs):
+        """
+        Find the `building:levels` tag in the attributes of the building or one of the building
+        relations and save this as the number of stories. Calculate the floorspace of the
+        building, based on the footprint size of the building and the number of stories.
+
+        Args:
+            tags (dict):
+                Building tags, such as the building levels or building type.
+            area (float):
+                Footprint size of the building.
+            ghsl_characteristics_type (int):
+                The GHSL characteristics type most overlapping the building.
+
+        Returns:
+            A dictionary with the number of stories and the floorspace of the building.
+        """
+
+        gem_taxonomy_height = []
+
+        # The number of stories ideally comes from the tags from OpenStreetMap. If we are able
+        # to use that, we can also calculate the floorspace. The height tags in
+        # OSM are also parsed, but no floorspace is estimated.
+        try:
+            stories, floorspace = self.get_stories_and_floorspace_from_osm(tags, area)
+        # OSM can have unexpected data types in their tagging scheme, therefore we do not raise
+        # an exception in case of a ValueError, but instead ignore the values.
+        except ValueError:
+            stories, floorspace = None, None
+        if stories:
+            gem_taxonomy_height.append(stories)
+
+        try:
+            height = self.get_height_from_osm(tags)
+        # OSM can have unexpected data types in their tagging scheme, therefore we do not raise
+        # an exception in case of a ValueError, but instead ignore the values.
+        except ValueError:
+            height = None
+        if height:
+            gem_taxonomy_height.append(height)
+
+        # If any of the number of stories and height are found in OSM, these are returned.
+        if len(gem_taxonomy_height) > 0:
+            return {
+                "height": "+".join(gem_taxonomy_height),
+                "floorspace": floorspace,
+            }
+
+        # If no information can be retrieved from OSM, we use the height estimation from GHSL
+        # and do not estimate any floorspace.
+        ghsl_height = self.get_height_from_ghsl(ghsl_characteristics_type)
+        if ghsl_height:
+            return {"height": ghsl_height, "floorspace": None}
+
+        return {"height": None, "floorspace": None}
+
+    def get_stories_and_floorspace_from_osm(self, tags: dict, area: float):
+        """
+        Get the number of stories and the floorspace, based on the tags `building:levels`,
+        `roof:levels`, `building:levels:underground` and `building:min_level` and the footprint
+        area of the building.
+
+        Args:
+            tags (dict):
+                Building tags, such as the building levels or building type.
+            area (float):
+                Footprint size of the building.
+
+        Returns:
+            A string formatted according to the `height` tag in the GEM Taxonomy.
+        """
+
+        from math import ceil
+
+        # Parse the story tags from OpenStreetMap.
+        main_stories = self.tag_to_float(tags, "building:levels")
+        roof_stories = self.tag_to_float(tags, "roof:levels")
+        underground_stories = self.tag_to_float(tags, "building:levels:underground")
+        min_stories = self.tag_to_float(tags, "building:min_level")
+
+        # Parse the main stories and roof stories.
+        stories = None
+        floorspace = None
+        if main_stories or roof_stories:
+            stories = 0
+            floorspace = 0
+
+            if main_stories:
+                stories += main_stories
+                floorspace += main_stories * area
+            if roof_stories:
+                stories += roof_stories
+                # Take only 50% of the floorspace for roof stories
+                floorspace += roof_stories * area * 0.5
+            if min_stories:
+                stories -= min_stories
+                floorspace -= min_stories * area
+
+            # Ceil the number of stories
+            stories = ceil(stories)
+
+            # Take a factor of 90% for the floorspace
+            floorspace *= 0.9
+
+        # Parse the underground story tag.
+        if underground_stories:
+            underground_stories = ceil(underground_stories)
+
+        # Check if anything is wrong with the tags.
+        if stories < 0 or underground_stories < 0:
+            raise ValueError("Number of stories cannot be below 0.")
+        elif stories == 0 and not (underground_stories > 0):
+            raise ValueError(
+                "Number of stories cannot be 0, if there are no underground stories."
+            )
+        elif underground_stories == 0 and not (stories > 0):
+            raise ValueError(
+                "Number of underground stories cannot be 0, if there are no stories."
+            )
+        elif stories > 175:
+            raise ValueError("Number of stories cannot be above 175.")
+
+        # Create the story tag according to the GEM taxonomy.
+        story_tags = []
+        if stories:
+            story_tags.append(f"H:{stories}")
+        if underground_stories:
+            story_tags.append(f"HBEX:{underground_stories}")
+
+        return "+".join(story_tags), floorspace
+
+    def get_height_from_osm(self, tags: dict):
+        """
+        Get the heightb of a building, based on the `height` and `min_height` tags.
+
+        Args:
+            tags (dict):
+                Building tags, such as the building levels or building type.
+
+        Returns:
+            A string formatted according to the `height` tag in the GEM Taxonomy.
+        """
+
+        # Parse the height tag from OpenStreetMap.
+        height = self.tag_to_float(tags, "height")
+
+        # Check if height exists
+        if not height:
+            return None
+
+        # Check if there is a `min_height` tag
+        min_height = self.tag_to_float(tags, "min_height")
+        if min_height:
+            height -= min_height
+
+        if height < 0:
+            raise ValueError("Height cannot be below 0.")
+
+        return f"HHT:{height:.2f}"
+
+    def get_height_from_ghsl(self, ghsl_characteristics_type: int):
+        """
+        Get the GEM Taxonomy height tag, based on the type in the GHSL
+        characteristics layer that has the most overlap with the building.
+
+        Args:
+            ghsl_characteristics_type (int):
+                The GHSL characteristics type most overlapping the building.
+
+        Returns:
+            A string formatted according to the `height` tag in the GEM Taxonomy.
+        """
+
+        ghsl_type_map = {
+            11: "HBET:1-2",  # res_3
+            12: "HBET:1-2",  # res_3_6
+            13: "HBET:1-5",  # res_6_15
+            14: None,  # res_15_30
+            15: None,  # res_30
+            21: "HBET:1-2",  # nonres_3
+            22: "HBET:1-2",  # nonres_3_6
+            23: "HBET:1-5",  # nonres_6_15
+            24: None,  # nonres_15_30
+            25: None,  # nonres_30
+        }
+        return ghsl_type_map.get(ghsl_characteristics_type, None)
+
+    def tag_to_float(self, tags: dict, tag_key: str):
+        """
+        Try to parse a tag as a floating point value. If the value cannot be parsed, return
+        None.
+
+        Args:
+            tags (dict):
+                Building tags, such as the building levels or building type.
+            tag_key (str):
+                Name of the tag that should be converted.
+
+        Returns:
+            Tag value converted to a float.
+        """
+
+        # Try to get the tag value.
+        tag_value = tags.get(tag_key, None)
+
+        # If the tag does not exist, return None.
+        if tag_value is None or tag_value == "":
+            return None
+
+        # If the tag does exist, try to convert it to float. If that resolves in a value error,
+        # return None.
+        try:
+            return float(tag_value)
+        except ValueError:
+            return None

building/02_process/stories_and_floorspace/rule.xml → building/02_process/height_and_floorspace/rule.xml

 <?xml version="1.0" encoding="UTF-8" ?>
-<rule name="StoriesAndFloorspaceRule" category="building">
+<rule name="HeightAndFloorspaceRule" category="building">
     <input>
-        <param type="int">tags</param>
-        <param type="float">relations</param>
+        <param type="dict">tags</param>
         <param type="float">area</param>
+        <param type="int">ghsl_characteristics_type</param>
     </input>
-    <function filepath="stories_and_floorspace.py"/>
+    <function filepath="height_and_floorspace.py"/>
     <output>
-        <param type="float">storeys</param>
+        <param type="str">height</param>
         <param type="float">floorspace</param>
     </output>
 </rule>

building/02_process/stories_and_floorspace/stories_and_floorspace.py

-#!/usr/bin/env python3
-
-# Copyright (c) 2023-2024:
-#   Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
-#
-# This program is free software: you can redistribute it and/or modify it
-# under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or (at
-# your option) any later version.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
-# General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see http://www.gnu.org/licenses/.
-
-
-class StoriesAndFloorspaceRule:
-    def __call__(self, tags, relations, area, *args, **kwargs):
-        """
-        Find the `building:levels` tag in the attributes of the building or one of the building
-        relations and save this as the number of stories. Calculate the floorspace of the
-        building, based on the footprint size of the building and the number of stories.
-
-        Args:
-            tags (dict):
-                Building tags, such as the building levels or building type.
-            relations (list):
-                List of the attributes of all relations that the building is member of.
-            area (float):
-                Footprint size of the building.
-
-        Returns:
-            A dictionary with the number of stories and the floorspace of the building.
-        """
-
-        from math import ceil
-
-        all_building_tags = [tags] + [building["tags"] for building in relations]
-
-        for tags in all_building_tags:
-            story_string = self.get_story_tag(tags)
-            if story_string is None or story_string == "":
-                continue
-            try:
-                stories = ceil(float(story_string))
-                if stories < 1:
-                    raise ValueError("Number of stories cannot be below 1")
-                elif stories > 175:
-                    raise ValueError("Number of stories cannot be above 175")
-
-                floorspace = stories * area
-                return {"stories": stories, "floorspace": floorspace}
-            except ValueError:
-                continue
-        return {"stories": None, "floorspace": None}
-
-    @staticmethod
-    def get_story_tag(tags):
-        """
-        Get the number of stories, if the attribute `building:levels` exist
-
-        Args:
-            tags:
-                Building tags, such as the building levels or building type.
-
-        Returns:
-            Number of stories, if the attribute `building:levels` exist, otherwise `None`.
-        """
-
-        return tags.get("building:levels", None)

tests/__init__.py

+#!/usr/bin/env python3
+
+# Copyright (C) 2023:
+#   Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see http://www.gnu.org/licenses/.

tests/conftest.py

+#!/usr/bin/env python3
+
+# Copyright (C) 2023:
+#   Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see http://www.gnu.org/licenses/.
+
+import pytest
+import shutil
+import tempfile
+import os
+
+from rulelib import Rule
+
+
+@pytest.fixture
+def height_and_floorspace_rule():
+    # Get the filepath of the rule
+    project_dir = os.getenv("CI_PROJECT_DIR", "")
+    rule_dir = os.path.join(project_dir, "building/02_process/height_and_floorspace")
+
+    # Create a temporary ZIP file from the `height_and_floorspace` rule.
+    tmp_dir = tempfile.mkdtemp()
+    file_path = os.path.join(tmp_dir + "height_and_floorspace")
+    shutil.make_archive(file_path, "zip", rule_dir)
+
+    # Yield rule.
+    yield Rule.load_rule_from_zip(open(file_path + ".zip", "rb"))
+
+    # Remove temporary ZIP file.
+    shutil.rmtree(tmp_dir, ignore_errors=True)

tests/test_height_and_floorspace_rule.py

+import logging
+import pytest
+
+logger = logging.getLogger()
+
+
+def test_height_and_floorspace_rule(height_and_floorspace_rule):
+    building_information = {
+        "tags": {
+            "building:levels": 5,
+            "roof:levels": 2,
+            "building:min_level": 1,
+            "building:levels:underground": 0,
+        },
+        "area": 300.0,
+        "ghsl_characteristics_type": 11,
+    }
+    result = height_and_floorspace_rule(**building_information)
+    assert result["height"] == "H:6"
+    assert result["floorspace"] == pytest.approx(1350.0)