Extended to handle range of diffs

.gitignore

@@ -13,6 +13,7 @@ dist
 env
 
 config.yml
+data/*
 
 .idea
 .vscode

spearhead/configure.py

@@ -20,7 +20,6 @@ import logging
 import sys
 import yaml
 
-
 logger = logging.getLogger(__name__)
 logger.addHandler(logging.StreamHandler(sys.stdout))
 
@@ -39,6 +38,8 @@ class Configuration:
 
         try:
             self.overpass_base_url = config["overpass_base_url"]
+            self.filepath = config["filepath"]
+            self.state_filename = config["state_filename"]
         except KeyError as e:
             logger.info("Error: The variable %s hasn't been set in the config file." % str(e))
             sys.exit(1)

spearhead/diff_analyzer.py

@@ -18,9 +18,11 @@
 
 import logging
 import sys
+
 import osmdiff
 
 from spearhead.configure import Configuration
+from spearhead.file_handler import FileHandler
 
 logger = logging.getLogger(__name__)
 logger.addHandler(logging.StreamHandler(sys.stdout))
@@ -35,6 +37,13 @@ class DiffAnalyzer:
 
     def __init__(self, diff_provider: osmdiff.AugmentedDiff, config: Configuration):
         self.diff_provider = diff_provider
+        self.file_handler = FileHandler(config)
+
+    async def get_list_of_new_diffs(self):
+        self.diff_provider.get_state()
+        new_state = self.diff_provider.sequence_number
+        old_state = await self.file_handler.read_state_file()
+        return list(range(old_state, new_state))
 
     async def get_augmented_diff(self) -> osmdiff.AugmentedDiff:
         """Obtain an augmented diff from Overpass API
@@ -49,6 +58,10 @@ class DiffAnalyzer:
         self.diff_provider.retrieve()
         return self.diff_provider
 
+    async def update_state(self, state: int) -> None:
+
+        await self.file_handler.update_state_file(str(state))
+
     async def get_buildings(self, augmented_diff) -> None:
         """Collect the building IDs of building objects that have changed in a given
         augmented diff.

spearhead/file_handler.py

+#!/usr/bin/env python3
+
+# Copyright (C) 2021:
+#   Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or (at
+# your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
+# General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see http://www.gnu.org/licenses/.
+
+import logging
+import sys
+
+import aiofiles
+
+logger = logging.getLogger(__name__)
+logger.addHandler(logging.StreamHandler(sys.stdout))
+
+
+class FileHandler:
+    """FileHandler."""
+
+    def __init__(self, config):
+        self.config = config
+
+    async def read_state_file(self) -> int:
+        """Reads state recorded in a text file"""
+        async with aiofiles.open(
+            self.config.filepath + self.config.state_filename, mode="r"
+        ) as f:
+            state = await f.readline()
+        return int(state)
+
+    async def update_state_file(self, state: int) -> None:
+        """Replaces state sequence_number in state text file"""
+        async with aiofiles.open(
+            self.config.filepath + self.config.state_filename, mode="w"
+        ) as f:
+            await f.write(state)
+
+    async def write_diff_report(self, identifier: int, numbers: list) -> None:
+        """Writes a report of analyzer results per augmented diff into a text file"""
+        async with aiofiles.open(
+            self.config.filepath + "augmented-diff-" + str(identifier) + ".txt", mode="w"
+        ) as f:
+            for number in numbers:
+                await f.write(number)

spearhead/spearhead.py

@@ -31,47 +31,45 @@ logger.addHandler(logging.StreamHandler(sys.stdout))
 
 
 def main():
+    """Wrapper around the main asynchronous program."""
     asyncio.run(async_main())
 
 
 async def async_main():
-
+    """Main entrypoint of the program."""
     logger.info("spearhead started - it can be stopped with CTRL+c")
-
     config = Configuration("config.yml")
-
-    await Spearhead().get_changed_buildings(config)
+    await get_changed_buildings(config)
 
 
-class Spearhead:
-    """Spearhead.
+async def get_changed_buildings(config: Configuration):
+    """Obtain a list of buiildings that have changed since last check."""
+    diff_provider = osmdiff.AugmentedDiff()
+    diff_provider.base_url = config.overpass_base_url
+    logger.info("Using Overpass API: " + diff_provider.base_url)
 
-    This class contains the main entry point for this program.
-    """
+    diff_analyzer = DiffAnalyzer(diff_provider, config)
 
-    async def get_changed_buildings(self, config: Configuration):
-        """Obtain a list of buiildings that have changed since last check."""
+    while True:
+        try:
 
-        diff_provider = osmdiff.AugmentedDiff()
-        diff_provider.base_url = config.overpass_base_url
-        diff_provider.get_state()
-        logger.info("Using Overpass API: " + diff_provider.base_url)
+            # Obtain available list of diffs
+            diffs = await diff_analyzer.get_list_of_new_diffs()
 
-        diff_analyzer = DiffAnalyzer(diff_provider, config)
-
-        while True:
-            try:
+            for diff_number in diffs:
 
                 # Obtain and read augmented diff from Overpass API
+                diff_provider.sequence_number = diff_number
                 diff = await diff_analyzer.get_augmented_diff()
 
                 # Get identifiers of all buildings that have changed
                 await diff_analyzer.get_buildings(diff)
 
-                # Wait until requesting more data
-                await asyncio.sleep(60)
+                await diff_analyzer.update_state(diff_number + 1)
+
+            # Wait until requesting more data
+            await asyncio.sleep(60)
 
-            except KeyboardInterrupt:
-                # Leave the program
-                logger.info("spearhead stopped")
-                sys.exit()
+        except KeyboardInterrupt:
+            logger.info("spearhead stopped")
+            sys.exit()