Introduced OpenStreetMap data updater

env-example

+# SPEARHEAD database connection
+#
+SPEARHEAD_DATABASE_HOST=localhost
+SPEARHEAD_DATABASE_NAME=postgres
+SPEARHEAD_DATABASE_USER=gis
+
+# Location of the import conversion style definitions for osm2pgsql
+#
+SPEARHEAD_IMPORT_STYLE=/usr/local/share/osm2pgsql/example.style
+
+# Location of the OpenStreetMap data file to import
+#
+SPEARHEAD_IMPORT_FILE=/tmp/import/data.osm.pbf
+
+# OpenStreetMap replication information
+#
+SPEARHEAD_REPLICATION_DIRECTORY=/var/lib/spearhead
+SPEARHEAD_SEQUENCE_FILE=sequence.txt
+SPEARHEAD_REPLICATION_SERVER=https://planet.osm.org/replication/minute/

includes/database.sh

@@ -42,7 +42,8 @@ function database::wait_for_connection() {
 #   Query command
 #######################################
 function database::execute_command() {
-  psql --host="${SPEARHEAD_DATABASE_HOST}" --username="${SPEARHEAD_DATABASE_USER}" --command="${1}"
+  psql --host="${SPEARHEAD_DATABASE_HOST}" --username="${SPEARHEAD_DATABASE_USER}" \
+    --command="${1}"
 }
 
 #######################################

includes/import.sh

@@ -21,34 +21,65 @@
 #######################################
 # # Import OpenStreetMap Data file into database with osm2pgsql.
 # Globals:
-#   SPEARHEAD_IMPORT_FILE
-#   SPEARHEAD_IMPORT_STYLE
 #   SPEARHEAD_DATABASE_HOST
 #   SPEARHEAD_DATABASE_USER
 #   SPEARHEAD_DATABASE_NAME
+#   SPEARHEAD_IMPORT_STYLE
 # Arguments:
-#   None
+#   importfile
 #######################################
 function import::openstreetmap_data() {
 
-  if [[ -f "${SPEARHEAD_IMPORT_FILE}" ]]; then
-    printf " \nImporting OpenStreetMap data with "
-    osm2pgsql \
-      --host "${SPEARHEAD_DATABASE_HOST}" --username "${SPEARHEAD_DATABASE_USER}" \
+  if [[ -f "${1}" ]]; then
+    printf "\nImporting OpenStreetMap data with "
+
+    osm2pgsql --create --verbose \
+      --host "${SPEARHEAD_DATABASE_HOST}" \
+      --username "${SPEARHEAD_DATABASE_USER}" \
       --database "${SPEARHEAD_DATABASE_NAME}" \
-      --verbose \
-      --create \
-      --slim \
       --style "${SPEARHEAD_IMPORT_STYLE}" \
+      --slim \
       --multi-geometry \
       --hstore \
       --latlong \
       --extra-attributes \
       --hstore-add-index \
-      --cache-strategy dense \
-      "${SPEARHEAD_IMPORT_FILE}"
+      "${1}"
+  else
+    printf "Error: Import file '%s' not found.\nAborting.\n" "${1}"
+    exit 1
+  fi
+}
+
+#######################################
+# # Import OpenStreetMap changeset file into database with osm2pgsql.
+# Globals:
+#   SPEARHEAD_DATABASE_HOST
+#   SPEARHEAD_DATABASE_USER
+#   SPEARHEAD_DATABASE_NAME
+#   SPEARHEAD_IMPORT_STYLE
+# Arguments:
+#   importfile
+#######################################
+function import::openstreetmap_changeset() {
+
+  if [[ -f "${1}" ]]; then
+    printf "\nUpdating OpenStreetMap data with "
+
+    osm2pgsql --append --verbose \
+      --host "${SPEARHEAD_DATABASE_HOST}" \
+      --username "${SPEARHEAD_DATABASE_USER}" \
+      --database "${SPEARHEAD_DATABASE_NAME}" \
+      --style "${SPEARHEAD_IMPORT_STYLE}" \
+      --slim \
+      --multi-geometry \
+      --hstore \
+      --latlong \
+      --extra-attributes \
+      "${1}"
+
   else
-    printf "Error: Import file '%s' not found.\nAborting.\n" "${SPEARHEAD_IMPORT_FILE}"
+    printf "Error: Changeset file '%s' not found.\nAborting.\n" "${1}"
     exit 1
   fi
 }

spearhead

 #!/bin/bash
 #
-# spearhead: Import OpenStreetMap data into OpenBuildingMap
+# spearhead: Import OpenStreetMap data into OpenBuildingMap and update continously.
 
 # Copyright (C) 2021:
 #   Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
@@ -27,14 +27,14 @@ source /usr/local/lib/spearhead/includes/database.sh --source-only
 source /usr/local/lib/spearhead/includes/import.sh --source-only
 
 #######################################
-# Main function of this script, to be executed for initial import.
+# Initialize and populate the database.
 # Globals:
 #   SPEARHEAD_DATABASE_HOST
 #   SPEARHEAD_DATABASE_NAME
 # Arguments:
 #   None
 #######################################
-function main() {
+function initialize() {
   printf "Connecting to PostgreSQL on '%s':\n" "${SPEARHEAD_DATABASE_HOST}"
   database::wait_for_connection
 
@@ -56,13 +56,113 @@ function main() {
   if ${database_exists}; then
     printf "Assuming already imported database: %s\n" "${SPEARHEAD_DATABASE_NAME}"
   else
-    import::openstreetmap_data
+    import::openstreetmap_data "${SPEARHEAD_IMPORT_FILE}"
+
+    # Write last sequence number from import file to sequence file
+    pyosmium-get-changes -v \
+      --start-osm-data "${SPEARHEAD_IMPORT_FILE}" \
+      --sequence-file "${SPEARHEAD_REPLICATION_DIRECTORY}/${SPEARHEAD_SEQUENCE_FILE}" \
+      --server "${SPEARHEAD_REPLICATION_SERVER}" \
+      --ignore-osmosis-headers
+
     printf "OpenStreetMap data has been imported.\n"
   fi
 }
 
+#######################################
+# Define exit handler
+# Globals:
+#   None
+# Arguments:
+#   None
+#######################################
+function onexit() {
+  [ -f "previous-${SPEARHEAD_SEQUENCE_FILE}" ] &&
+    mv "previous-${SPEARHEAD_SEQUENCE_FILE}" "${SPEARHEAD_SEQUENCE_FILE}"
+}
+
+#######################################
+# Constantly download OpenStreetMap data and update the database.
+# Globals:
+#   SPEARHEAD_DATABASE_HOST
+#   SPEARHEAD_DATABASE_NAME
+# Arguments:
+#   None
+#######################################
+function update() {
+
+  # Change to the spearhead replication data directory
+  mkdir -p "${SPEARHEAD_REPLICATION_DIRECTORY}"
+  cd "${SPEARHEAD_REPLICATION_DIRECTORY}"
+
+  # Disable JIT and parallel workers in PostgreSQL
+  export PGOPTIONS="-c jit=off -c max_parallel_workers_per_gather=0"
+
+  # Include an exit handler
+  trap onexit EXIT
+
+  printf "\nStart data-update daemon.\n\n"
+
+  # This runs as daemon and loops indefinitely
+  while true; do
+
+    update_file="changes-$(cat "${SPEARHEAD_SEQUENCE_FILE}").osc.gz"
+
+    # Save previous sequence file for rollback if needed
+    cp "${SPEARHEAD_SEQUENCE_FILE}" "previous-${SPEARHEAD_SEQUENCE_FILE}"
+
+    # Fetch set of OSM changesets
+    pyosmium-get-changes -v \
+      --server="${SPEARHEAD_REPLICATION_SERVER}" \
+      --sequence-file="${SPEARHEAD_SEQUENCE_FILE}" \
+      --outfile="${update_file}"
+
+    # Check for exit sttaus
+    status=$?
+    if [ $status -eq 0 ]; then
+
+      # Exit immediately on error
+      set -e
+
+      printf "Fetched new data from %s to %s into %s" \
+        "$(cat previous-"${SPEARHEAD_SEQUENCE_FILE}")" \
+        "$(cat "${SPEARHEAD_SEQUENCE_FILE}")" \
+        "${update_file}"
+
+      import::openstreetmap_changeset "${update_file}"
+
+      # After successfull import, the previous sequence file can be removed
+      rm "previous-${SPEARHEAD_SEQUENCE_FILE}"
+
+      # Delete old downloads
+      find . -name 'changes-*.gz' -mmin +300 -exec rm -f {} \;
+
+      # Disable exit immediately on error
+      set +e
+
+    elif [ $status -eq 3 ]; then
+      printf "No new data available.\n"
+
+      # Remove file, it is just emty
+      rm "${update_file}"
+
+      sleep 30
+
+    else
+      printf "Failed to fetch changeset file.\n"
+
+      # Remove file, it is just emty
+      rm -f "${update_file}"
+
+      sleep 300
+    fi
+  done
+
+}
+
 # Start the script
-printf "\nWelcome to the OpenBuildingMap importer\n\n"
+printf "\nWelcome to the OpenBuildingMap importer and updater\n\n"
 
-# Call main function of the script
-main
+# Call main functions of the script
+initialize
+update