diff --git a/.gitignore b/.gitignore index 6064fb5eeaacddc8add4d8f39b152c93a08ecc86..bb71f1a6bc0bb55ea7e29c3bb1dd33e3e2cfe4f2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ +.idea __pycache__ results.db diff --git a/LICENSE b/LICENSE index 1e86876b32e4b1f731c906c0af93aa11aa4fa096..af513de6cd5d22f98128e48a8407848cd0e08207 100644 --- a/LICENSE +++ b/LICENSE @@ -616,4 +616,4 @@ an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. - END OF TERMS AND CONDITIONS \ No newline at end of file + END OF TERMS AND CONDITIONS diff --git a/README.rst b/README.rst index 9f90627e25056efbdd08b8ac419d9d22d634af9a..85e035e2e43c48919656df66595cf66aed746edb 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ rabotnik demo ============= -Simplest showcase for +Simplest showcase for `rabotnik <https://git.gfz-potsdam.de/dynamicexposure/rabotnik/rabotnik>`__ spinnig up two of its instances with example computation content and a `main message bus <https://git.gfz-potsdam.de/dynamicexposure/rabotnik/docker-rabotnik-message-bus>`__ diff --git a/data/instance-a/instance.py b/data/instance-a/instance.py index 610409b24e2eb268440bbfb067246ace660f2471..61a6dde67492cfe0c1106a2bbd411baf2ab46e46 100644 --- a/data/instance-a/instance.py +++ b/data/instance-a/instance.py @@ -41,6 +41,9 @@ async def main(): # Initiate the message bus message_bus = MessageBus() + # pylint: disable=no-value-for-parameter + await message_bus.connect(config_file="rabotnik-message-bus.yml") + logger.info("running: instance a") # Trigger a very simple 'building' task diff --git a/data/instance-a/rabotnik-message-bus.yml b/data/instance-a/rabotnik-message-bus.yml new file mode 100644 index 0000000000000000000000000000000000000000..149da0bb2b60474f25d7d9db1e31ccb5d0d1bc5c --- /dev/null +++ b/data/instance-a/rabotnik-message-bus.yml @@ -0,0 +1,4 @@ +--- +username: test +password: test +host: rabotnik-message-bus diff --git a/data/instance-b/instance.py b/data/instance-b/instance.py index 36e36f6069ba9fbd0f1ae41a2b88419d906bf2d7..c72f8238322fdca6e30ddecae53814323f6427ce 100644 --- a/data/instance-b/instance.py +++ b/data/instance-b/instance.py @@ -33,8 +33,7 @@ logger = logging.getLogger() logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler(sys.stdout)) -rabotnik = Rabotnik("instance-b", "mockeddb") -message_bus = MessageBus() +rabotnik = Rabotnik("instance-b") logger.info("running: instance b") @@ -45,11 +44,18 @@ def building(building_id): The payload is the *building_id* """ - rules = [BuildingDemo(rabotnik.storage)] + storage = rabotnik.get_storage(selector="mockeddb") + storage.connect() + + rules = [BuildingDemo(storage)] rules[0].evaluate(building_id) async def main(): + message_bus = MessageBus() + + # pylint: disable=no-value-for-parameter + await message_bus.connect(config_file="rabotnik-message-bus.yml") await message_bus.subscribe("building", building) diff --git a/data/instance-b/rabotnik-message-bus.yml b/data/instance-b/rabotnik-message-bus.yml new file mode 100644 index 0000000000000000000000000000000000000000..149da0bb2b60474f25d7d9db1e31ccb5d0d1bc5c --- /dev/null +++ b/data/instance-b/rabotnik-message-bus.yml @@ -0,0 +1,4 @@ +--- +username: test +password: test +host: rabotnik-message-bus diff --git a/data/instance-c/instance.py b/data/instance-c/instance.py index 894cc7e5c61551b3c8d71f35977f7e8681e60b2f..07579907c6cc7bc88a7d4b3830f06931be69d625 100644 --- a/data/instance-c/instance.py +++ b/data/instance-c/instance.py @@ -33,8 +33,7 @@ logger = logging.getLogger() logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler(sys.stdout)) -rabotnik = Rabotnik("instance-c", "mockeddb") -message_bus = MessageBus() +rabotnik = Rabotnik("instance-c") logger.info("running: instance c") @@ -50,6 +49,10 @@ def building(building_id): async def main(): + message_bus = MessageBus() + + # pylint: disable=no-value-for-parameter + await message_bus.connect(config_file="rabotnik-message-bus.yml") await message_bus.subscribe("building", building) diff --git a/data/instance-c/rabotnik-message-bus.yml b/data/instance-c/rabotnik-message-bus.yml new file mode 100644 index 0000000000000000000000000000000000000000..149da0bb2b60474f25d7d9db1e31ccb5d0d1bc5c --- /dev/null +++ b/data/instance-c/rabotnik-message-bus.yml @@ -0,0 +1,4 @@ +--- +username: test +password: test +host: rabotnik-message-bus diff --git a/data/instance-migrate/__init__.py b/data/instance-migrate/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cfc59abc9861f4c1cce141b4664fef8634741398 --- /dev/null +++ b/data/instance-migrate/__init__.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2020-2021: +# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or (at +# your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. diff --git a/data/instance-migrate/instance.py b/data/instance-migrate/instance.py new file mode 100644 index 0000000000000000000000000000000000000000..8507ff31dc7cac7e61eb2764c61332598773dd6c --- /dev/null +++ b/data/instance-migrate/instance.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2020-2021: +# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or (at +# your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. + +import logging +import asyncio + +# pylint: disable=import-error +from rabotnik import Rabotnik +from rabotnik.bus import MessageBus + +from rules.migration_demo import MigrateBuilding + + +rabotnik_logger = logging.getLogger("rabotnik") +rabotnik_logger.setLevel(logging.DEBUG) + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + +# Initiate the rabotnik +rabotnik = Rabotnik("building-migration") + + +async def main(): + + # Initiate the message bus + message_bus = MessageBus() + + # pylint: disable=no-value-for-parameter + await message_bus.connect(config_file="rabotnik-message-bus.yml") + logger.info("running: receiver") + + # Get a storage connection to `database_1` (source) + storage_a = rabotnik.get_storage("postgresql") + await storage_a.connect(config_file="storage-a.yml") + + # Get a storage connection to `database_2` (receiver) + storage_b = rabotnik.get_storage("postgresql") + await storage_b.connect(config_file="storage-b.yml") + + # await message_bus.subscribe('building', print_payload) + rule = MigrateBuilding(storage_a, storage_b) + await message_bus.subscribe("building", rule.evaluate) + + +loop = asyncio.get_event_loop() +loop.create_task(main()) +loop.run_forever() diff --git a/data/instance-migrate/rabotnik-message-bus.yml b/data/instance-migrate/rabotnik-message-bus.yml new file mode 100644 index 0000000000000000000000000000000000000000..149da0bb2b60474f25d7d9db1e31ccb5d0d1bc5c --- /dev/null +++ b/data/instance-migrate/rabotnik-message-bus.yml @@ -0,0 +1,4 @@ +--- +username: test +password: test +host: rabotnik-message-bus diff --git a/data/instance-migrate/rules/__init__.py b/data/instance-migrate/rules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cfc59abc9861f4c1cce141b4664fef8634741398 --- /dev/null +++ b/data/instance-migrate/rules/__init__.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2020-2021: +# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or (at +# your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. diff --git a/data/instance-migrate/rules/migration_demo.py b/data/instance-migrate/rules/migration_demo.py new file mode 100644 index 0000000000000000000000000000000000000000..4ab9deacb8ac5822305f7177a96e414161de4855 --- /dev/null +++ b/data/instance-migrate/rules/migration_demo.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2020-2021: +# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or (at +# your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. + +import logging + +from rabotnik import Rule # pylint: disable=import-error + +logger = logging.getLogger() + + +class MigrateBuilding(Rule): + def __init__(self, storage_from, storage_to): + self.storage_from = storage_from + self.storage_to = storage_to + + async def evaluate(self, payload: dict): + """A simple demo rule accessing a local database.""" + building_id = payload["building_id"] + + # Pull the building data from storage_from + for row in await self.storage_from.get_results( + f"SELECT * FROM buildings WHERE id={building_id}" + ): + + # Push the building data to storage_to + async with self.storage_to.connection.cursor() as cur: + await cur.execute("INSERT INTO buildings (id, geometry) VALUES (%s,%s)", row) diff --git a/data/instance-migrate/storage-a.yml b/data/instance-migrate/storage-a.yml new file mode 100644 index 0000000000000000000000000000000000000000..4f3b0f4153deef39a956d6ce847ec38d3bc84e5b --- /dev/null +++ b/data/instance-migrate/storage-a.yml @@ -0,0 +1,6 @@ +--- +user: postgres +password: postgres +dbname: buildings +host: database-1 +port: 5432 diff --git a/data/instance-migrate/storage-b.yml b/data/instance-migrate/storage-b.yml new file mode 100644 index 0000000000000000000000000000000000000000..a489c80cb27b3ef76897e819613677e75d6bf697 --- /dev/null +++ b/data/instance-migrate/storage-b.yml @@ -0,0 +1,6 @@ +--- +user: postgres +password: postgres +dbname: buildings +host: database-2 +port: 5432 diff --git a/data/rcom-example/run.sh b/data/rcom-example/run.sh index 18b36ac2dc002d295b2644447faedafa0016d798..0f3ad96a932ecdbcaa71380924890649eeb050f8 100755 --- a/data/rcom-example/run.sh +++ b/data/rcom-example/run.sh @@ -28,4 +28,4 @@ psql -h database-2 -U postgres -f /tmp/database/create_database.sql psql -h database-2 -U postgres -d buildings -f /tmp/database/create_table.sql # Run rcom test -wait-for-it -t 30 $RABOTNIK_MESSAGE_BUS_HOST:$RABOTNIK_MESSAGE_BUS_PORT -- sleep 15; rcom --building-id=42 +wait-for-it -t 30 $RABOTNIK_MESSAGE_BUS_HOST:$RABOTNIK_MESSAGE_BUS_PORT -- sleep 15; rcom --username $RABOTNIK_MESSAGE_BUS_USER --password $RABOTNIK_MESSAGE_BUS_PASSWORD --hostname $RABOTNIK_MESSAGE_BUS_HOST:$RABOTNIK_MESSAGE_BUS_PORT --building-id=42 diff --git a/docker-compose.override.yml b/docker-compose.override.yml index e8352a8ea88b971a4e3b6787a7d756e7016dcab5..72fe3ba0314910f944bb0be5fc64a7617a02fd70 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -24,10 +24,6 @@ services: # rabotnik rcom rabotnik-rcom: build: ../../containers/docker-rcom-instance - volumes: - - ../rabotnik:/srv/rabotnik - - ../rcom:/srv/rcom - - ./data/database:/tmp/database # rabotnik instance a rabotnik-instance-a: @@ -46,3 +42,9 @@ services: build: ../../containers/docker-rabotnik-instance volumes: - ../rabotnik:/srv/rabotnik + + # rabotnik instance migration + rabotnik-instance-migration: + build: ../../containers/docker-rabotnik-instance + volumes: + - ../rabotnik:/srv/rabotnik diff --git a/docker-compose.yml b/docker-compose.yml index 9fae63c5041348bc4ce49b24857ac93a333db5b2..ebd0a8bafdabba32d703b271e66fd50f1d6902ff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,11 +71,25 @@ services: volumes: - ./data/instance-c:/srv/rabotnik-instance + # rabotnik instance migration + rabotnik-instance-migration: + env_file: .env + build: https://git.gfz-potsdam.de/dynamicexposure/rabotnik/docker-rabotnik-instance.git + depends_on: + - rabotnik-message-bus + - database-1 + - database-2 + + volumes: + - ./data/instance-migrate:/srv/rabotnik-instance + # Example database 1 database-1: build: https://git.gfz-potsdam.de/dynamicexposure/server-components/containers/docker-obm-database.git expose: - "5432" + ports: + - "5433:5432" environment: POSTGRES_HOST_AUTH_METHOD: trust @@ -84,5 +98,7 @@ services: build: https://git.gfz-potsdam.de/dynamicexposure/server-components/containers/docker-obm-database.git expose: - "5432" + ports: + - "5434:5432" environment: POSTGRES_HOST_AUTH_METHOD: trust