Convert stories to the GEM Taxonomy height attribute

.gitlab-ci.yml

@@ -11,11 +11,11 @@ cache:
     - venv/
 
 stages:
-  - linters
+  - tests
   - release
 
 linters:
-  stage: linters
+  stage: tests
   before_script:
     - python3 -V
     - pip3 install virtualenv
@@ -26,6 +26,18 @@ linters:
     - flake8 --max-line-length=$LINE_LENGTH $SOURCES
     - black --check --line-length $LINE_LENGTH $SOURCES
 
+tests:
+  stage: tests
+  before_script:
+    - python3 -V
+    - pip3 install virtualenv
+    - virtualenv venv
+    - source venv/bin/activate
+    - pip3 install pytest
+    - pip3 install https://git.gfz-potsdam.de/globaldynamicexposure/libraries/rule-lib/-/archive/main/rule-lib-main.zip
+  script:
+    - pytest tests
+
 release_job:
   stage: release
   image: registry.gitlab.com/gitlab-org/release-cli:latest

building/02_process/height_and_floorspace/height_and_floorspace.py

+#!/usr/bin/env python3
+
+# Copyright (c) 2023-2024:
+#   Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see http://www.gnu.org/licenses/.
+
+
+class HeightAndFloorspaceRule:
+    def __call__(self, tags, area, *args, **kwargs):
+        """
+        Find the `building:levels` tag in the attributes of the building or one of the building
+        relations and save this as the number of stories. Calculate the floorspace of the
+        building, based on the footprint size of the building and the number of stories.
+
+        Args:
+            tags (dict):
+                Building tags, such as the building levels or building type.
+            area (float):
+                Footprint size of the building.
+
+        Returns:
+            A dictionary with the number of stories and the floorspace of the building.
+        """
+
+        gem_taxonomy_height = []
+
+        # The number of stories ideally comes from the `building:levels` tags in OpenStreetMap.
+        # If they are present, the floorspace of a building can be calculated. OpenStreetMap
+        # buildings can also contain a `height` tag but this tag is only parsed without the
+        # floorspace being estimated.
+        try:
+            stories, floorspace = self.get_stories_and_floorspace_from_osm(tags, area)
+        # OSM can have unexpected data types and unexpected content in their tagging scheme,
+        # therefore we do not raise an exception in case of a ValueError, but instead ignore
+        # the values.
+        except ValueError:
+            stories, floorspace = None, None
+        if stories:
+            gem_taxonomy_height.append(stories)
+
+        # If any of the number-of-stories and height tags are found in OpenStreetMap, they are
+        # returned.
+        if len(gem_taxonomy_height) > 0:
+            return {
+                "height": "+".join(gem_taxonomy_height),
+                "floorspace": floorspace,
+            }
+
+        return {"height": None, "floorspace": None}
+
+    def get_stories_and_floorspace_from_osm(self, tags: dict, area: float):
+        """
+        Get the number of stories and the floorspace, based on the tags `building:levels`,
+        `roof:levels`, `building:levels:underground` and `building:min_level` and the footprint
+        area of the building. The information about parsing the tags comes from
+        https://wiki.openstreetmap.org/wiki/Key:building:levels.
+
+        Args:
+            tags (dict):
+                Building tags, such as the building levels or building type.
+            area (float):
+                Footprint size of the building.
+
+        Returns:
+            A string formatted according to the `height` tag in the GEM Taxonomy.
+        """
+
+        from math import ceil
+
+        # Parse the story tags from OpenStreetMap.
+        main_stories = self.tag_to_float(tags, "building:levels")
+        roof_stories = self.tag_to_float(tags, "roof:levels")
+        underground_stories = self.tag_to_float(tags, "building:levels:underground")
+        min_stories = self.tag_to_float(tags, "building:min_level")
+
+        # Parse the number of main stories and roof stories.
+        stories = 0
+        floorspace = 0
+        if main_stories or roof_stories:
+            if main_stories:
+                stories += main_stories
+                floorspace += main_stories * area
+            if roof_stories:
+                stories += roof_stories
+                # Take only 50% of the floorspace for roof stories.
+                floorspace += roof_stories * area * 0.5
+            if min_stories:
+                stories -= min_stories
+                floorspace -= min_stories * area
+
+            # Ceil the number of stories.
+            stories = ceil(stories)
+
+        # Parse the underground story tag.
+        if underground_stories:
+            underground_stories = ceil(underground_stories)
+            floorspace += underground_stories * area
+        else:
+            underground_stories = 0
+
+        # Take a factor of 90% for the floorspace.
+        floorspace *= 0.9
+
+        # Check if anything is wrong with the tags.
+        if stories < 0 or underground_stories < 0:
+            return None, None
+        elif stories == 0 and underground_stories == 0:
+            return None, None
+        elif stories + underground_stories > 175:
+            return None, None
+
+        # Create the number-of-stories tag of the height attribute.
+        story_tags = []
+        if stories != 0:
+            story_tags.append(f"H:{stories}")
+        if underground_stories != 0:
+            story_tags.append(f"HBEX:{underground_stories}")
+
+        return "+".join(story_tags), floorspace
+
+    def tag_to_float(self, tags: dict, tag_key: str):
+        """
+        Try to parse a tag as a floating point value. If the value cannot be parsed, return
+        None.
+
+        Args:
+            tags (dict):
+                Building tags, such as the building levels or building type.
+            tag_key (str):
+                Name of the tag that should be converted.
+
+        Returns:
+            Tag value converted to a float.
+        """
+
+        # Try to get the tag value.
+        tag_value = tags.get(tag_key, None)
+
+        # If the tag does not exist, return None.
+        if tag_value is None or tag_value == "":
+            return None
+
+        # If the tag does exist, try to convert it to float. If that raises a `ValueError`,
+        # return None.
+        try:
+            return float(tag_value)
+        except ValueError:
+            return None

building/02_process/stories_and_floorspace/rule.xml → building/02_process/height_and_floorspace/rule.xml

 <?xml version="1.0" encoding="UTF-8" ?>
-<rule name="StoriesAndFloorspaceRule" category="building">
+<rule name="HeightAndFloorspaceRule" category="building">
     <input>
-        <param type="int">tags</param>
-        <param type="float">relations</param>
+        <param type="dict">tags</param>
         <param type="float">area</param>
+        <param type="int">ghsl_characteristics_type</param>
     </input>
-    <function filepath="stories_and_floorspace.py"/>
+    <function filepath="height_and_floorspace.py"/>
     <output>
-        <param type="float">storeys</param>
+        <param type="str">height</param>
         <param type="float">floorspace</param>
     </output>
 </rule>

building/02_process/stories_and_floorspace/stories_and_floorspace.py

-#!/usr/bin/env python3
-
-# Copyright (c) 2023-2024:
-#   Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
-#
-# This program is free software: you can redistribute it and/or modify it
-# under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or (at
-# your option) any later version.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
-# General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see http://www.gnu.org/licenses/.
-
-
-class StoriesAndFloorspaceRule:
-    def __call__(self, tags, relations, area, *args, **kwargs):
-        """
-        Find the `building:levels` tag in the attributes of the building or one of the building
-        relations and save this as the number of stories. Calculate the floorspace of the
-        building, based on the footprint size of the building and the number of stories.
-
-        Args:
-            tags (dict):
-                Building tags, such as the building levels or building type.
-            relations (list):
-                List of the attributes of all relations that the building is member of.
-            area (float):
-                Footprint size of the building.
-
-        Returns:
-            A dictionary with the number of stories and the floorspace of the building.
-        """
-
-        from math import ceil
-
-        all_building_tags = [tags] + [building["tags"] for building in relations]
-
-        for tags in all_building_tags:
-            story_string = self.get_story_tag(tags)
-            if story_string is None or story_string == "":
-                continue
-            try:
-                stories = ceil(float(story_string))
-                if stories < 1:
-                    raise ValueError("Number of stories cannot be below 1")
-                elif stories > 175:
-                    raise ValueError("Number of stories cannot be above 175")
-
-                floorspace = stories * area
-                return {"stories": stories, "floorspace": floorspace}
-            except ValueError:
-                continue
-        return {"stories": None, "floorspace": None}
-
-    @staticmethod
-    def get_story_tag(tags):
-        """
-        Get the number of stories, if the attribute `building:levels` exist
-
-        Args:
-            tags:
-                Building tags, such as the building levels or building type.
-
-        Returns:
-            Number of stories, if the attribute `building:levels` exist, otherwise `None`.
-        """
-
-        return tags.get("building:levels", None)

tests/__init__.py

+#!/usr/bin/env python3
+
+# Copyright (C) 2024:
+#   Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see http://www.gnu.org/licenses/.

tests/conftest.py

+#!/usr/bin/env python3
+
+# Copyright (C) 2024:
+#   Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see http://www.gnu.org/licenses/.
+
+import pytest
+import shutil
+import tempfile
+import os
+
+from rulelib import Rule
+
+
+@pytest.fixture
+def height_and_floorspace_rule():
+    """
+    Creates a temporary directory, where the `HeightsAndFloorspaceRule` rule is converted to a
+    ZIP file. This ZIP file is parsed using `rule-lib` and the parsed rule is provided as a
+    fixture.
+    """
+
+    # Get the filepath of the rule
+    project_dir = os.getenv("CI_PROJECT_DIR", "")
+    rule_dir = os.path.join(project_dir, "building/02_process/height_and_floorspace")
+
+    # Create a temporary ZIP file from the `height_and_floorspace` rule.
+    tmp_dir = tempfile.mkdtemp()
+    file_path = os.path.join(tmp_dir + "height_and_floorspace")
+    shutil.make_archive(file_path, "zip", rule_dir)
+
+    # Yield rule.
+    yield Rule.load_rule_from_zip(open(file_path + ".zip", "rb"))
+
+    # Remove temporary ZIP file.
+    shutil.rmtree(tmp_dir, ignore_errors=True)

tests/test_height_and_floorspace_rule.py

+#!/usr/bin/env python3
+
+# Copyright (c) 2024:
+#   Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see http://www.gnu.org/licenses/.
+
+
+import logging
+import pytest
+
+logger = logging.getLogger()
+
+
+def test_height_and_floorspace_rule(height_and_floorspace_rule):
+    """
+    Test the retrieval of number of stories, height and the floorspace, based on the tags from
+    OSM and GHSL.
+    """
+
+    building_information = {
+        "tags": {
+            "building:levels": 5,
+            "roof:levels": 2,
+            "building:min_level": 1,
+            "building:levels:underground": 0,
+        },
+        "area": 300.0,
+    }
+    result = height_and_floorspace_rule(**building_information)
+    message = (
+        f"The `height` attribute is not correct. "
+        f"Expected `H:6` but estimated {result['height']}"
+    )
+    assert result["height"] == "H:6", message
+    message = (
+        f"The `floorspace` attribute is not correct. "
+        f"Expected `1350` but estimated {result['floorspace']}"
+    )
+    assert result["floorspace"] == pytest.approx(1350.0), message
+
+
+def test_get_stories_and_floorspace_from_osm(height_and_floorspace_rule):
+    """
+    Test the retrieval of number of stories and the floorspace, based on the tags from OSM.
+    """
+
+    rule_function = height_and_floorspace_rule.function.get_stories_and_floorspace_from_osm
+    test_list = [
+        # Check with only building levels.
+        [{"tags": {"building:levels": 5}, "area": 100.0}, "H:5", 450.0],
+        # Check with only roof levels.
+        [{"tags": {"roof:levels": 5}, "area": 100.0}, "H:5", 225.0],
+        # Check with only levels underground.
+        [{"tags": {"building:levels:underground": 5}, "area": 100.0}, "HBEX:5", 450.0],
+        # Check with a wrong type as input.
+        [{"tags": {"building:levels": "no"}, "area": 100.0}, None, None],
+        # Check with a negative value.
+        [{"tags": {"building:levels:underground": -1}, "area": 100.0}, None, None],
+        # Check with a `building:min_level` value higher than the `building_levels` value.
+        [{"tags": {"building:levels": 5, "building:min_level": 6}, "area": 100.0}, None, None],
+        # Check without any tags.
+        [{"tags": {}, "area": 100.0}, None, None],
+        # Check with all tags.
+        [
+            {
+                "tags": {
+                    "building:levels": 4,
+                    "roof:levels": 2,
+                    "building:min_level": 1,
+                    "building:levels:underground": 1,
+                },
+                "area": 100.0,
+            },
+            "H:5+HBEX:1",
+            450.0,
+        ],
+    ]
+
+    for input_values, correct_height, correct_floorspace in test_list:
+        height, floorspace = rule_function(**input_values)
+
+        # Check the height attribute
+        message = (
+            f"The `height` attribute is not correct. Expected `{correct_height}` but estimated "
+            f"`{height}` using {input_values} as input."
+        )
+        assert height == correct_height, message
+
+        # Check the floorspace attribute
+        message = (
+            f"The `floorspace` attribute is not correct. Expected `{correct_floorspace}` but "
+            f"estimated `{floorspace}` using {input_values} as input."
+        )
+        assert floorspace == correct_floorspace, message
+
+
+def test_tag_to_float(height_and_floorspace_rule):
+    """
+    Test the function `tag_to_float` of the `HeightAndFloorspaceRule.
+    """
+
+    rule_function = height_and_floorspace_rule.function.tag_to_float
+
+    tags = {
+        "correct_01": "1",
+        "correct_02": "1.5",
+        "correct_03": "-1",
+        "incorrect_01": "ground_floor",
+        "incorrect_02": "1,5",
+    }
+
+    assert rule_function(tags, "correct_01") == pytest.approx(1.0)
+    assert rule_function(tags, "correct_02") == pytest.approx(1.5)
+    assert rule_function(tags, "correct_03") == pytest.approx(-1.0)
+    assert rule_function(tags, "incorrect_01") is None
+    assert rule_function(tags, "incorrect_02") is None
+    assert rule_function(tags, "incorrect_03") is None  # Value is not there.