From c3875d3c8623ddcce1da7d009b09bc3c9014fc07 Mon Sep 17 00:00:00 2001
From: Laurens <laurens@gfz-potsdam.de>
Date: Wed, 10 Jul 2024 14:56:50 +0200
Subject: [PATCH] Convert stories to the GEM Taxonomy height attribute
---
.gitlab-ci.yml | 16 +-
.../height_and_floorspace.py | 160 ++++++++++++++++++
.../02_process/height_and_floorspace/rule.xml | 13 ++
.../stories_and_floorspace/rule.xml | 13 --
.../stories_and_floorspace.py | 73 --------
tests/__init__.py | 17 ++
tests/conftest.py | 48 ++++++
tests/test_height_and_floorspace_rule.py | 129 ++++++++++++++
8 files changed, 381 insertions(+), 88 deletions(-)
create mode 100644 building/02_process/height_and_floorspace/height_and_floorspace.py
create mode 100644 building/02_process/height_and_floorspace/rule.xml
delete mode 100644 building/02_process/stories_and_floorspace/rule.xml
delete mode 100644 building/02_process/stories_and_floorspace/stories_and_floorspace.py
create mode 100644 tests/__init__.py
create mode 100644 tests/conftest.py
create mode 100644 tests/test_height_and_floorspace_rule.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 51cc9e0..cfb2be1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -11,11 +11,11 @@ cache:
- venv/
stages:
- - linters
+ - tests
- release
linters:
- stage: linters
+ stage: tests
before_script:
- python3 -V
- pip3 install virtualenv
@@ -26,6 +26,18 @@ linters:
- flake8 --max-line-length=$LINE_LENGTH $SOURCES
- black --check --line-length $LINE_LENGTH $SOURCES
+tests:
+ stage: tests
+ before_script:
+ - python3 -V
+ - pip3 install virtualenv
+ - virtualenv venv
+ - source venv/bin/activate
+ - pip3 install pytest
+ - pip3 install https://git.gfz-potsdam.de/globaldynamicexposure/libraries/rule-lib/-/archive/main/rule-lib-main.zip
+ script:
+ - pytest tests
+
release_job:
stage: release
image: registry.gitlab.com/gitlab-org/release-cli:latest
diff --git a/building/02_process/height_and_floorspace/height_and_floorspace.py b/building/02_process/height_and_floorspace/height_and_floorspace.py
new file mode 100644
index 0000000..d1f24bb
--- /dev/null
+++ b/building/02_process/height_and_floorspace/height_and_floorspace.py
@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2023-2024:
+# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see http://www.gnu.org/licenses/.
+
+
+class HeightAndFloorspaceRule:
+ def __call__(self, tags, area, *args, **kwargs):
+ """
+ Find the `building:levels` tag in the attributes of the building or one of the building
+ relations and save this as the number of stories. Calculate the floorspace of the
+ building, based on the footprint size of the building and the number of stories.
+
+ Args:
+ tags (dict):
+ Building tags, such as the building levels or building type.
+ area (float):
+ Footprint size of the building.
+
+ Returns:
+ A dictionary with the number of stories and the floorspace of the building.
+ """
+
+ gem_taxonomy_height = []
+
+ # The number of stories ideally comes from the `building:levels` tags in OpenStreetMap.
+ # If they are present, the floorspace of a building can be calculated. OpenStreetMap
+ # buildings can also contain a `height` tag but this tag is only parsed without the
+ # floorspace being estimated.
+ try:
+ stories, floorspace = self.get_stories_and_floorspace_from_osm(tags, area)
+ # OSM can have unexpected data types and unexpected content in their tagging scheme,
+ # therefore we do not raise an exception in case of a ValueError, but instead ignore
+ # the values.
+ except ValueError:
+ stories, floorspace = None, None
+ if stories:
+ gem_taxonomy_height.append(stories)
+
+ # If any of the number-of-stories and height tags are found in OpenStreetMap, they are
+ # returned.
+ if len(gem_taxonomy_height) > 0:
+ return {
+ "height": "+".join(gem_taxonomy_height),
+ "floorspace": floorspace,
+ }
+
+ return {"height": None, "floorspace": None}
+
+ def get_stories_and_floorspace_from_osm(self, tags: dict, area: float):
+ """
+ Get the number of stories and the floorspace, based on the tags `building:levels`,
+ `roof:levels`, `building:levels:underground` and `building:min_level` and the footprint
+ area of the building. The information about parsing the tags comes from
+ https://wiki.openstreetmap.org/wiki/Key:building:levels.
+
+ Args:
+ tags (dict):
+ Building tags, such as the building levels or building type.
+ area (float):
+ Footprint size of the building.
+
+ Returns:
+ A string formatted according to the `height` tag in the GEM Taxonomy.
+ """
+
+ from math import ceil
+
+ # Parse the story tags from OpenStreetMap.
+ main_stories = self.tag_to_float(tags, "building:levels")
+ roof_stories = self.tag_to_float(tags, "roof:levels")
+ underground_stories = self.tag_to_float(tags, "building:levels:underground")
+ min_stories = self.tag_to_float(tags, "building:min_level")
+
+ # Parse the number of main stories and roof stories.
+ stories = 0
+ floorspace = 0
+ if main_stories or roof_stories:
+ if main_stories:
+ stories += main_stories
+ floorspace += main_stories * area
+ if roof_stories:
+ stories += roof_stories
+ # Take only 50% of the floorspace for roof stories.
+ floorspace += roof_stories * area * 0.5
+ if min_stories:
+ stories -= min_stories
+ floorspace -= min_stories * area
+
+ # Ceil the number of stories.
+ stories = ceil(stories)
+
+ # Parse the underground story tag.
+ if underground_stories:
+ underground_stories = ceil(underground_stories)
+ floorspace += underground_stories * area
+ else:
+ underground_stories = 0
+
+ # Take a factor of 90% for the floorspace.
+ floorspace *= 0.9
+
+ # Check if anything is wrong with the tags.
+ if stories < 0 or underground_stories < 0:
+ return None, None
+ elif stories == 0 and underground_stories == 0:
+ return None, None
+ elif stories + underground_stories > 175:
+ return None, None
+
+ # Create the number-of-stories tag of the height attribute.
+ story_tags = []
+ if stories != 0:
+ story_tags.append(f"H:{stories}")
+ if underground_stories != 0:
+ story_tags.append(f"HBEX:{underground_stories}")
+
+ return "+".join(story_tags), floorspace
+
+ def tag_to_float(self, tags: dict, tag_key: str):
+ """
+ Try to parse a tag as a floating point value. If the value cannot be parsed, return
+ None.
+
+ Args:
+ tags (dict):
+ Building tags, such as the building levels or building type.
+ tag_key (str):
+ Name of the tag that should be converted.
+
+ Returns:
+ Tag value converted to a float.
+ """
+
+ # Try to get the tag value.
+ tag_value = tags.get(tag_key, None)
+
+ # If the tag does not exist, return None.
+ if tag_value is None or tag_value == "":
+ return None
+
+ # If the tag does exist, try to convert it to float. If that raises a `ValueError`,
+ # return None.
+ try:
+ return float(tag_value)
+ except ValueError:
+ return None
diff --git a/building/02_process/height_and_floorspace/rule.xml b/building/02_process/height_and_floorspace/rule.xml
new file mode 100644
index 0000000..a3e34be
--- /dev/null
+++ b/building/02_process/height_and_floorspace/rule.xml
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<rule name="HeightAndFloorspaceRule" category="building">
+ <input>
+ <param type="dict">tags</param>
+ <param type="float">area</param>
+ <param type="int">ghsl_characteristics_type</param>
+ </input>
+ <function filepath="height_and_floorspace.py"/>
+ <output>
+ <param type="str">height</param>
+ <param type="float">floorspace</param>
+ </output>
+</rule>
diff --git a/building/02_process/stories_and_floorspace/rule.xml b/building/02_process/stories_and_floorspace/rule.xml
deleted file mode 100644
index 24ce38c..0000000
--- a/building/02_process/stories_and_floorspace/rule.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<rule name="StoriesAndFloorspaceRule" category="building">
- <input>
- <param type="int">tags</param>
- <param type="float">relations</param>
- <param type="float">area</param>
- </input>
- <function filepath="stories_and_floorspace.py"/>
- <output>
- <param type="float">storeys</param>
- <param type="float">floorspace</param>
- </output>
-</rule>
diff --git a/building/02_process/stories_and_floorspace/stories_and_floorspace.py b/building/02_process/stories_and_floorspace/stories_and_floorspace.py
deleted file mode 100644
index 847ed97..0000000
--- a/building/02_process/stories_and_floorspace/stories_and_floorspace.py
+++ /dev/null
@@ -1,73 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright (c) 2023-2024:
-# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
-#
-# This program is free software: you can redistribute it and/or modify it
-# under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or (at
-# your option) any later version.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
-# General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see http://www.gnu.org/licenses/.
-
-
-class StoriesAndFloorspaceRule:
- def __call__(self, tags, relations, area, *args, **kwargs):
- """
- Find the `building:levels` tag in the attributes of the building or one of the building
- relations and save this as the number of stories. Calculate the floorspace of the
- building, based on the footprint size of the building and the number of stories.
-
- Args:
- tags (dict):
- Building tags, such as the building levels or building type.
- relations (list):
- List of the attributes of all relations that the building is member of.
- area (float):
- Footprint size of the building.
-
- Returns:
- A dictionary with the number of stories and the floorspace of the building.
- """
-
- from math import ceil
-
- all_building_tags = [tags] + [building["tags"] for building in relations]
-
- for tags in all_building_tags:
- story_string = self.get_story_tag(tags)
- if story_string is None or story_string == "":
- continue
- try:
- stories = ceil(float(story_string))
- if stories < 1:
- raise ValueError("Number of stories cannot be below 1")
- elif stories > 175:
- raise ValueError("Number of stories cannot be above 175")
-
- floorspace = stories * area
- return {"stories": stories, "floorspace": floorspace}
- except ValueError:
- continue
- return {"stories": None, "floorspace": None}
-
- @staticmethod
- def get_story_tag(tags):
- """
- Get the number of stories, if the attribute `building:levels` exist
-
- Args:
- tags:
- Building tags, such as the building levels or building type.
-
- Returns:
- Number of stories, if the attribute `building:levels` exist, otherwise `None`.
- """
-
- return tags.get("building:levels", None)
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..561623c
--- /dev/null
+++ b/tests/__init__.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2024:
+# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see http://www.gnu.org/licenses/.
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 0000000..f88e873
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2024:
+# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see http://www.gnu.org/licenses/.
+
+import pytest
+import shutil
+import tempfile
+import os
+
+from rulelib import Rule
+
+
+@pytest.fixture
+def height_and_floorspace_rule():
+ """
+ Creates a temporary directory, where the `HeightsAndFloorspaceRule` rule is converted to a
+ ZIP file. This ZIP file is parsed using `rule-lib` and the parsed rule is provided as a
+ fixture.
+ """
+
+ # Get the filepath of the rule
+ project_dir = os.getenv("CI_PROJECT_DIR", "")
+ rule_dir = os.path.join(project_dir, "building/02_process/height_and_floorspace")
+
+ # Create a temporary ZIP file from the `height_and_floorspace` rule.
+ tmp_dir = tempfile.mkdtemp()
+ file_path = os.path.join(tmp_dir + "height_and_floorspace")
+ shutil.make_archive(file_path, "zip", rule_dir)
+
+ # Yield rule.
+ yield Rule.load_rule_from_zip(open(file_path + ".zip", "rb"))
+
+ # Remove temporary ZIP file.
+ shutil.rmtree(tmp_dir, ignore_errors=True)
diff --git a/tests/test_height_and_floorspace_rule.py b/tests/test_height_and_floorspace_rule.py
new file mode 100644
index 0000000..9aa34c7
--- /dev/null
+++ b/tests/test_height_and_floorspace_rule.py
@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2024:
+# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see http://www.gnu.org/licenses/.
+
+
+import logging
+import pytest
+
+logger = logging.getLogger()
+
+
+def test_height_and_floorspace_rule(height_and_floorspace_rule):
+ """
+ Test the retrieval of number of stories, height and the floorspace, based on the tags from
+ OSM and GHSL.
+ """
+
+ building_information = {
+ "tags": {
+ "building:levels": 5,
+ "roof:levels": 2,
+ "building:min_level": 1,
+ "building:levels:underground": 0,
+ },
+ "area": 300.0,
+ }
+ result = height_and_floorspace_rule(**building_information)
+ message = (
+ f"The `height` attribute is not correct. "
+ f"Expected `H:6` but estimated {result['height']}"
+ )
+ assert result["height"] == "H:6", message
+ message = (
+ f"The `floorspace` attribute is not correct. "
+ f"Expected `1350` but estimated {result['floorspace']}"
+ )
+ assert result["floorspace"] == pytest.approx(1350.0), message
+
+
+def test_get_stories_and_floorspace_from_osm(height_and_floorspace_rule):
+ """
+ Test the retrieval of number of stories and the floorspace, based on the tags from OSM.
+ """
+
+ rule_function = height_and_floorspace_rule.function.get_stories_and_floorspace_from_osm
+ test_list = [
+ # Check with only building levels.
+ [{"tags": {"building:levels": 5}, "area": 100.0}, "H:5", 450.0],
+ # Check with only roof levels.
+ [{"tags": {"roof:levels": 5}, "area": 100.0}, "H:5", 225.0],
+ # Check with only levels underground.
+ [{"tags": {"building:levels:underground": 5}, "area": 100.0}, "HBEX:5", 450.0],
+ # Check with a wrong type as input.
+ [{"tags": {"building:levels": "no"}, "area": 100.0}, None, None],
+ # Check with a negative value.
+ [{"tags": {"building:levels:underground": -1}, "area": 100.0}, None, None],
+ # Check with a `building:min_level` value higher than the `building_levels` value.
+ [{"tags": {"building:levels": 5, "building:min_level": 6}, "area": 100.0}, None, None],
+ # Check without any tags.
+ [{"tags": {}, "area": 100.0}, None, None],
+ # Check with all tags.
+ [
+ {
+ "tags": {
+ "building:levels": 4,
+ "roof:levels": 2,
+ "building:min_level": 1,
+ "building:levels:underground": 1,
+ },
+ "area": 100.0,
+ },
+ "H:5+HBEX:1",
+ 450.0,
+ ],
+ ]
+
+ for input_values, correct_height, correct_floorspace in test_list:
+ height, floorspace = rule_function(**input_values)
+
+ # Check the height attribute
+ message = (
+ f"The `height` attribute is not correct. Expected `{correct_height}` but estimated "
+ f"`{height}` using {input_values} as input."
+ )
+ assert height == correct_height, message
+
+ # Check the floorspace attribute
+ message = (
+ f"The `floorspace` attribute is not correct. Expected `{correct_floorspace}` but "
+ f"estimated `{floorspace}` using {input_values} as input."
+ )
+ assert floorspace == correct_floorspace, message
+
+
+def test_tag_to_float(height_and_floorspace_rule):
+ """
+ Test the function `tag_to_float` of the `HeightAndFloorspaceRule.
+ """
+
+ rule_function = height_and_floorspace_rule.function.tag_to_float
+
+ tags = {
+ "correct_01": "1",
+ "correct_02": "1.5",
+ "correct_03": "-1",
+ "incorrect_01": "ground_floor",
+ "incorrect_02": "1,5",
+ }
+
+ assert rule_function(tags, "correct_01") == pytest.approx(1.0)
+ assert rule_function(tags, "correct_02") == pytest.approx(1.5)
+ assert rule_function(tags, "correct_03") == pytest.approx(-1.0)
+ assert rule_function(tags, "incorrect_01") is None
+ assert rule_function(tags, "incorrect_02") is None
+ assert rule_function(tags, "incorrect_03") is None # Value is not there.
--
GitLab