From 14fc9b36ed5e6681e2efd6d0ae6fc9fa93ebf4c6 Mon Sep 17 00:00:00 2001
From: Felix Delattre <fd@gfz-potsdam.de>
Date: Thu, 23 Sep 2021 14:43:47 +0000
Subject: [PATCH] Introduced configuration management
---
.gitignore | 4 +-
setup.py | 2 +-
spearhead/__init__.py | 4 +-
spearhead/configure.py | 44 ++++++++++++++++
.../{osm_analyzer.py => diff_analyzer.py} | 34 +++++++------
spearhead/spearhead.py | 50 +++++++++++++------
6 files changed, 106 insertions(+), 32 deletions(-)
create mode 100644 spearhead/configure.py
rename spearhead/{osm_analyzer.py => diff_analyzer.py} (72%)
diff --git a/.gitignore b/.gitignore
index dc767ab..31b5a99 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,7 +4,6 @@
*.egg-info
Pipfile
Pipfile.lock
-.idea
__pycache__
.cache
@@ -13,4 +12,7 @@ build
dist
env
+config.yml
+
+.idea
.vscode
diff --git a/setup.py b/setup.py
index c404e07..5b50343 100644
--- a/setup.py
+++ b/setup.py
@@ -27,7 +27,7 @@ setup(
description="Trigger calculations on the Rabotnik Message Bus based on \
OpenStreetMap's augmented diffs",
license="AGPLv3+",
- install_requires=["osmdiff>=0.1.9"],
+ install_requires=["aiofiles", "osmdiff>=0.1.9"],
extras_require={
"tests": tests_require,
"linters": linters_require,
diff --git a/spearhead/__init__.py b/spearhead/__init__.py
index 8b2b9df..a1a76c7 100644
--- a/spearhead/__init__.py
+++ b/spearhead/__init__.py
@@ -17,7 +17,7 @@
# along with this program. If not, see http://www.gnu.org/licenses/.
-from .osm_analyzer import OSMAnalyzer
+from .diff_analyzer import DiffAnalyzer
-__all__ = ["OSMAnalyzer"]
+__all__ = ["DiffAnalyzer"]
diff --git a/spearhead/configure.py b/spearhead/configure.py
new file mode 100644
index 0000000..96f3a4b
--- /dev/null
+++ b/spearhead/configure.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2020-2021:
+# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see http://www.gnu.org/licenses/.
+
+import logging
+import sys
+import yaml
+
+
+logger = logging.getLogger(__name__)
+logger.addHandler(logging.StreamHandler(sys.stdout))
+
+
+class Configuration:
+ """Configuration.
+
+ Holds configuration data.
+ """
+
+ def __init__(self, config_file: str = "config.yml") -> None:
+
+ if config_file:
+ with open(config_file) as f:
+ config = yaml.load(f.read(), Loader=yaml.FullLoader)
+
+ try:
+ self.overpass_base_url = config["overpass_base_url"]
+ except KeyError as e:
+ logger.info("Error: The variable %s hasn't been set in the config file." % str(e))
+ sys.exit(1)
diff --git a/spearhead/osm_analyzer.py b/spearhead/diff_analyzer.py
similarity index 72%
rename from spearhead/osm_analyzer.py
rename to spearhead/diff_analyzer.py
index 369295a..6cb6608 100644
--- a/spearhead/osm_analyzer.py
+++ b/spearhead/diff_analyzer.py
@@ -17,38 +17,44 @@
# along with this program. If not, see http://www.gnu.org/licenses/.
import logging
-
+import sys
import osmdiff
+from spearhead.configure import Configuration
+
logger = logging.getLogger(__name__)
+logger.addHandler(logging.StreamHandler(sys.stdout))
-class OSMAnalyzer:
- """OSM analyzer.
+class DiffAnalyzer:
+ """DiffAnalyzer.
This class contains the logic for obtaining and interpreting the OpenStreetMap's
"augmented diffs".
"""
- async def get_augmented_diff(self, provider: osmdiff.AugmentedDiff):
+ def __init__(self, diff_provider: osmdiff.AugmentedDiff, config: Configuration):
+ self.diff_provider = diff_provider
+
+ async def get_augmented_diff(self) -> osmdiff.AugmentedDiff:
"""Obtain an augmented diff from Overpass API
Args:
- provider: connection to database
+ diff_provider: connection to database
Returns:
- provider (AugmentedDiff):
+ diff_provider (AugmentedDiff):
An object containing all information of an augmented diff.
"""
+ logger.info("Downloading augmented diff: " + str(self.diff_provider.sequence_number))
+ self.diff_provider.retrieve()
+ return self.diff_provider
- provider.get_state()
- provider.retrieve()
-
- return provider
-
- async def get_buildings(self, augmented_diff):
+ async def get_buildings(self, augmented_diff) -> None:
"""Collect the building IDs of building objects that have changed in a given
augmented diff.
"""
+ logger.info(" " + str(augmented_diff))
+
building_count = 0
# Check on newly created objects
@@ -71,9 +77,9 @@ class OSMAnalyzer:
if self._is_way_or_relation(n):
building_count += 1
- print("building_count: " + str(building_count))
+ logger.info(" building_count: " + str(building_count))
- def _is_way_or_relation(self, element):
+ def _is_way_or_relation(self, element) -> bool:
"""Check if element is of type `osm.Way` or `osm.Relation`"""
return isinstance(element, osmdiff.osm.osm.Way) or isinstance(
element, osmdiff.osm.osm.Relation
diff --git a/spearhead/spearhead.py b/spearhead/spearhead.py
index bdabe7a..29e6e46 100644
--- a/spearhead/spearhead.py
+++ b/spearhead/spearhead.py
@@ -21,10 +21,12 @@ import logging
import sys
import osmdiff
-from spearhead import OSMAnalyzer
+
+from spearhead.configure import Configuration
+from spearhead import DiffAnalyzer
logger = logging.getLogger(__name__)
-logger.setLevel(logging.DEBUG)
+logging.basicConfig(level=logging.DEBUG)
logger.addHandler(logging.StreamHandler(sys.stdout))
@@ -34,22 +36,42 @@ def main():
async def async_main():
- logger.info("spearhead started")
+ logger.info("spearhead started - it can be stopped with CTRL+c")
+
+ config = Configuration("config.yml")
+
+ await Spearhead().get_changed_buildings(config)
+
+
+class Spearhead:
+ """Spearhead.
+
+ This class contains the main entry point for this program.
+ """
+
+ async def get_changed_buildings(self, config: Configuration):
+ """Obtain a list of buiildings that have changed since last check."""
- await get_changed_buildings()
+ diff_provider = osmdiff.AugmentedDiff()
+ diff_provider.base_url = config.overpass_base_url
+ diff_provider.get_state()
+ logger.info("Using Overpass API: " + diff_provider.base_url)
- # Leave the program
- sys.exit()
+ diff_analyzer = DiffAnalyzer(diff_provider, config)
+ while True:
+ try:
-async def get_changed_buildings():
+ # Obtain and read augmented diff from Overpass API
+ diff = await diff_analyzer.get_augmented_diff()
- analyzer = OSMAnalyzer()
+ # Get identifiers of all buildings that have changed
+ await diff_analyzer.get_buildings(diff)
- diff_provider = osmdiff.AugmentedDiff()
- diff_provider.base_url = "https://overpass.openbuildingmap.org/api"
+ # Wait until requesting more data
+ await asyncio.sleep(60)
- while True:
- diff = await analyzer.get_augmented_diff(diff_provider)
- await analyzer.get_buildings(diff)
- await asyncio.sleep(60)
+ except KeyboardInterrupt:
+ # Leave the program
+ logger.info("spearhead stopped")
+ sys.exit()
--
GitLab