diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 90f5c6a3d712e3a94569f532c00031ffc1809a18..204b2cc7b372a18dbcd0f936935d04cc42043703 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,13 @@ image: python:3-bullseye # Make pip cache the installed dependencies variables: PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + RABOTNIK_CELERY_BROKER_HOST: "unset" + RABOTNIK_CELERY_BROKER_USER: "unset" + RABOTNIK_CELERY_BROKER_PASSWORD: "unset" + RABOTNIK_MESSAGE_BUS_HOST: "unset" + RABOTNIK_MESSAGE_BUS_USER: "unset" + RABOTNIK_MESSAGE_BUS_PASSWORD: "unset" + RABOTNIK_STORAGES: "unset" cache: paths: - .cache/pip @@ -13,12 +20,12 @@ before_script: - pip3 install virtualenv - virtualenv venv - source venv/bin/activate - - pip3 install . - - pip3 install .[tests] + - pip3 install . --no-cache + - pip3 install .[tests] --no-cache linters: script: - - pip3 install .[linters] --quiet + - pip3 install .[linters] --quiet --no-cache - make check tests: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 475946f84f499adc62efd0789f71210d81bbd6f4..e5b109ae2d8a24b6b4e5090c260b7cf0e7db8776 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: - id: trailing-whitespace - repo: https://github.com/psf/black - rev: 20.8b1 + rev: 22.3.0 hooks: - id: black args: [ --line-length=96 ] diff --git a/setup.py b/setup.py index e4eecb9b8257dcf1dd4cde8b224c194e83ca5986..db9c20ac95b14af10fa4e515a9c61af43d16badb 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,11 @@ setup( description="Trigger calculations on the Rabotnik Message Bus based on \ OpenStreetMap's augmented diffs", license="AGPLv3+", - install_requires=["aiofiles", "osmdiff>=0.1.9", "rabotnik>=0.0.3"], + install_requires=[ + "aiofiles", + "osmdiff>=0.1.9", + "rabotnik@git+https://git.gfz-potsdam.de/dynamicexposure/rabotnik/rabotnik.git", + ], extras_require={ "tests": tests_require, "linters": linters_require, diff --git a/spearhead/diff_analyzer.py b/spearhead/diff_analyzer.py index 3cc3c9d6045bba495e947bd53f86c01a9cf95bad..b4cd7a9c35e47c79cc28501719a79d034a03f97e 100644 --- a/spearhead/diff_analyzer.py +++ b/spearhead/diff_analyzer.py @@ -53,16 +53,16 @@ class DiffAnalyzer: return list(range(old_state, new_state)) async def get_augmented_diff(self) -> osmdiff.AugmentedDiff: - """Obtain an augmented diff from Overpass API + """ + Obtain an augmented diff from Overpass API - Args: - diff_provider: connection to database Returns: diff_provider (AugmentedDiff): An object containing all information of an augmented diff. """ + logger.info("Downloading augmented diff: " + str(self.diff_provider.sequence_number)) - self.diff_provider.retrieve() + self.diff_provider.retrieve(clear_cache=True) return self.diff_provider async def update_state(self, state: int) -> None: diff --git a/spearhead/spearhead.py b/spearhead/spearhead.py index 699cb694ef6192c5ac3efb4f1156db93eece581b..6d3ec2e3d2c66daa46068bf49e0367a65eecd83f 100644 --- a/spearhead/spearhead.py +++ b/spearhead/spearhead.py @@ -50,11 +50,7 @@ async def get_changed_buildings(config: Configuration): logger.info("Using Overpass API: " + diff_provider.base_url) message_bus = MessageBus() - await message_bus.connect( - host=config.rabotnik["hostname"], - username=config.rabotnik["username"], - password=config.rabotnik["password"], - ) + await message_bus.connect() diff_analyzer = DiffAnalyzer(diff_provider, config, message_bus) diff --git a/tests/conftest.py b/tests/conftest.py index fb400df03d947ea33a82529ae9bc0379f981c0d2..59fee83df96dc77e45910e68e07484d739dfd504 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,7 +17,6 @@ # along with this program. If not, see http://www.gnu.org/licenses/. import pytest - import osmdiff from rabotnik import MessageBus diff --git a/tests/test_diff_analyzer.py b/tests/test_diff_analyzer.py index dbbc2d89c72a1a06fdbf53cf69370647e047a4dd..88ab590810c77938e11cf7dd4997f170abf298bc 100644 --- a/tests/test_diff_analyzer.py +++ b/tests/test_diff_analyzer.py @@ -21,14 +21,15 @@ import logging import osmdiff import random import pytest +import pytest_asyncio from spearhead.diff_analyzer import DiffAnalyzer logger = logging.getLogger() -@pytest.fixture -async def augmented_diff(diff_analyzer): +@pytest_asyncio.fixture +async def augmented_diff(): augmented_diff = osmdiff.AugmentedDiff(file="./tests/data/augmented-diff-4749548.xml") augmented_diff.sequence_number = 4749548 yield augmented_diff