Commit c18bc3eb authored by Felix Delattre's avatar Felix Delattre
Browse files

Extended to handle range of diffs

parent 14fc9b36
Pipeline #28217 passed with stage
in 1 minute and 40 seconds
......@@ -13,6 +13,7 @@ dist
env
config.yml
data/*
.idea
.vscode
......@@ -20,7 +20,6 @@ import logging
import sys
import yaml
logger = logging.getLogger(__name__)
logger.addHandler(logging.StreamHandler(sys.stdout))
......@@ -39,6 +38,8 @@ class Configuration:
try:
self.overpass_base_url = config["overpass_base_url"]
self.filepath = config["filepath"]
self.state_filename = config["state_filename"]
except KeyError as e:
logger.info("Error: The variable %s hasn't been set in the config file." % str(e))
sys.exit(1)
......@@ -18,9 +18,11 @@
import logging
import sys
import osmdiff
from spearhead.configure import Configuration
from spearhead.file_handler import FileHandler
logger = logging.getLogger(__name__)
logger.addHandler(logging.StreamHandler(sys.stdout))
......@@ -35,6 +37,13 @@ class DiffAnalyzer:
def __init__(self, diff_provider: osmdiff.AugmentedDiff, config: Configuration):
self.diff_provider = diff_provider
self.file_handler = FileHandler(config)
async def get_list_of_new_diffs(self):
self.diff_provider.get_state()
new_state = self.diff_provider.sequence_number
old_state = await self.file_handler.read_state_file()
return list(range(old_state, new_state))
async def get_augmented_diff(self) -> osmdiff.AugmentedDiff:
"""Obtain an augmented diff from Overpass API
......@@ -49,6 +58,10 @@ class DiffAnalyzer:
self.diff_provider.retrieve()
return self.diff_provider
async def update_state(self, state: int) -> None:
await self.file_handler.update_state_file(str(state))
async def get_buildings(self, augmented_diff) -> None:
"""Collect the building IDs of building objects that have changed in a given
augmented diff.
......
#!/usr/bin/env python3
# Copyright (C) 2021:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see http://www.gnu.org/licenses/.
import logging
import sys
import aiofiles
logger = logging.getLogger(__name__)
logger.addHandler(logging.StreamHandler(sys.stdout))
class FileHandler:
"""FileHandler."""
def __init__(self, config):
self.config = config
async def read_state_file(self) -> int:
"""Reads state recorded in a text file"""
async with aiofiles.open(
self.config.filepath + self.config.state_filename, mode="r"
) as f:
state = await f.readline()
return int(state)
async def update_state_file(self, state: int) -> None:
"""Replaces state sequence_number in state text file"""
async with aiofiles.open(
self.config.filepath + self.config.state_filename, mode="w"
) as f:
await f.write(state)
async def write_diff_report(self, identifier: int, numbers: list) -> None:
"""Writes a report of analyzer results per augmented diff into a text file"""
async with aiofiles.open(
self.config.filepath + "augmented-diff-" + str(identifier) + ".txt", mode="w"
) as f:
for number in numbers:
await f.write(number)
......@@ -31,47 +31,45 @@ logger.addHandler(logging.StreamHandler(sys.stdout))
def main():
"""Wrapper around the main asynchronous program."""
asyncio.run(async_main())
async def async_main():
"""Main entrypoint of the program."""
logger.info("spearhead started - it can be stopped with CTRL+c")
config = Configuration("config.yml")
await Spearhead().get_changed_buildings(config)
await get_changed_buildings(config)
class Spearhead:
"""Spearhead.
async def get_changed_buildings(config: Configuration):
"""Obtain a list of buiildings that have changed since last check."""
diff_provider = osmdiff.AugmentedDiff()
diff_provider.base_url = config.overpass_base_url
logger.info("Using Overpass API: " + diff_provider.base_url)
This class contains the main entry point for this program.
"""
diff_analyzer = DiffAnalyzer(diff_provider, config)
async def get_changed_buildings(self, config: Configuration):
"""Obtain a list of buiildings that have changed since last check."""
while True:
try:
diff_provider = osmdiff.AugmentedDiff()
diff_provider.base_url = config.overpass_base_url
diff_provider.get_state()
logger.info("Using Overpass API: " + diff_provider.base_url)
# Obtain available list of diffs
diffs = await diff_analyzer.get_list_of_new_diffs()
diff_analyzer = DiffAnalyzer(diff_provider, config)
while True:
try:
for diff_number in diffs:
# Obtain and read augmented diff from Overpass API
diff_provider.sequence_number = diff_number
diff = await diff_analyzer.get_augmented_diff()
# Get identifiers of all buildings that have changed
await diff_analyzer.get_buildings(diff)
# Wait until requesting more data
await asyncio.sleep(60)
await diff_analyzer.update_state(diff_number + 1)
# Wait until requesting more data
await asyncio.sleep(60)
except KeyboardInterrupt:
# Leave the program
logger.info("spearhead stopped")
sys.exit()
except KeyboardInterrupt:
logger.info("spearhead stopped")
sys.exit()
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment