Commit 14fc9b36 authored by Felix Delattre's avatar Felix Delattre
Browse files

Introduced configuration management

parent 797d8f47
Pipeline #28190 passed with stage
in 1 minute and 7 seconds
......@@ -4,7 +4,6 @@
*.egg-info
Pipfile
Pipfile.lock
.idea
__pycache__
.cache
......@@ -13,4 +12,7 @@ build
dist
env
config.yml
.idea
.vscode
......@@ -27,7 +27,7 @@ setup(
description="Trigger calculations on the Rabotnik Message Bus based on \
OpenStreetMap's augmented diffs",
license="AGPLv3+",
install_requires=["osmdiff>=0.1.9"],
install_requires=["aiofiles", "osmdiff>=0.1.9"],
extras_require={
"tests": tests_require,
"linters": linters_require,
......
......@@ -17,7 +17,7 @@
# along with this program. If not, see http://www.gnu.org/licenses/.
from .osm_analyzer import OSMAnalyzer
from .diff_analyzer import DiffAnalyzer
__all__ = ["OSMAnalyzer"]
__all__ = ["DiffAnalyzer"]
#!/usr/bin/env python3
# Copyright (c) 2020-2021:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see http://www.gnu.org/licenses/.
import logging
import sys
import yaml
logger = logging.getLogger(__name__)
logger.addHandler(logging.StreamHandler(sys.stdout))
class Configuration:
"""Configuration.
Holds configuration data.
"""
def __init__(self, config_file: str = "config.yml") -> None:
if config_file:
with open(config_file) as f:
config = yaml.load(f.read(), Loader=yaml.FullLoader)
try:
self.overpass_base_url = config["overpass_base_url"]
except KeyError as e:
logger.info("Error: The variable %s hasn't been set in the config file." % str(e))
sys.exit(1)
......@@ -17,38 +17,44 @@
# along with this program. If not, see http://www.gnu.org/licenses/.
import logging
import sys
import osmdiff
from spearhead.configure import Configuration
logger = logging.getLogger(__name__)
logger.addHandler(logging.StreamHandler(sys.stdout))
class OSMAnalyzer:
"""OSM analyzer.
class DiffAnalyzer:
"""DiffAnalyzer.
This class contains the logic for obtaining and interpreting the OpenStreetMap's
"augmented diffs".
"""
async def get_augmented_diff(self, provider: osmdiff.AugmentedDiff):
def __init__(self, diff_provider: osmdiff.AugmentedDiff, config: Configuration):
self.diff_provider = diff_provider
async def get_augmented_diff(self) -> osmdiff.AugmentedDiff:
"""Obtain an augmented diff from Overpass API
Args:
provider: connection to database
diff_provider: connection to database
Returns:
provider (AugmentedDiff):
diff_provider (AugmentedDiff):
An object containing all information of an augmented diff.
"""
logger.info("Downloading augmented diff: " + str(self.diff_provider.sequence_number))
self.diff_provider.retrieve()
return self.diff_provider
provider.get_state()
provider.retrieve()
return provider
async def get_buildings(self, augmented_diff):
async def get_buildings(self, augmented_diff) -> None:
"""Collect the building IDs of building objects that have changed in a given
augmented diff.
"""
logger.info(" " + str(augmented_diff))
building_count = 0
# Check on newly created objects
......@@ -71,9 +77,9 @@ class OSMAnalyzer:
if self._is_way_or_relation(n):
building_count += 1
print("building_count: " + str(building_count))
logger.info(" building_count: " + str(building_count))
def _is_way_or_relation(self, element):
def _is_way_or_relation(self, element) -> bool:
"""Check if element is of type `osm.Way` or `osm.Relation`"""
return isinstance(element, osmdiff.osm.osm.Way) or isinstance(
element, osmdiff.osm.osm.Relation
......
......@@ -21,10 +21,12 @@ import logging
import sys
import osmdiff
from spearhead import OSMAnalyzer
from spearhead.configure import Configuration
from spearhead import DiffAnalyzer
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logging.basicConfig(level=logging.DEBUG)
logger.addHandler(logging.StreamHandler(sys.stdout))
......@@ -34,22 +36,42 @@ def main():
async def async_main():
logger.info("spearhead started")
logger.info("spearhead started - it can be stopped with CTRL+c")
config = Configuration("config.yml")
await Spearhead().get_changed_buildings(config)
class Spearhead:
"""Spearhead.
This class contains the main entry point for this program.
"""
async def get_changed_buildings(self, config: Configuration):
"""Obtain a list of buiildings that have changed since last check."""
await get_changed_buildings()
diff_provider = osmdiff.AugmentedDiff()
diff_provider.base_url = config.overpass_base_url
diff_provider.get_state()
logger.info("Using Overpass API: " + diff_provider.base_url)
# Leave the program
sys.exit()
diff_analyzer = DiffAnalyzer(diff_provider, config)
while True:
try:
async def get_changed_buildings():
# Obtain and read augmented diff from Overpass API
diff = await diff_analyzer.get_augmented_diff()
analyzer = OSMAnalyzer()
# Get identifiers of all buildings that have changed
await diff_analyzer.get_buildings(diff)
diff_provider = osmdiff.AugmentedDiff()
diff_provider.base_url = "https://overpass.openbuildingmap.org/api"
# Wait until requesting more data
await asyncio.sleep(60)
while True:
diff = await analyzer.get_augmented_diff(diff_provider)
await analyzer.get_buildings(diff)
await asyncio.sleep(60)
except KeyboardInterrupt:
# Leave the program
logger.info("spearhead stopped")
sys.exit()
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment