Commit 0e5cbb44 authored by Felix Delattre's avatar Felix Delattre
Browse files

Introduced OpenStreetMap data updater

parent 44124e52
Pipeline #27981 passed with stage
in 51 seconds
# SPEARHEAD database connection
#
SPEARHEAD_DATABASE_HOST=localhost
SPEARHEAD_DATABASE_NAME=postgres
SPEARHEAD_DATABASE_USER=gis
# Location of the import conversion style definitions for osm2pgsql
#
SPEARHEAD_IMPORT_STYLE=/usr/local/share/osm2pgsql/example.style
# Location of the OpenStreetMap data file to import
#
SPEARHEAD_IMPORT_FILE=/tmp/import/data.osm.pbf
# OpenStreetMap replication information
#
SPEARHEAD_REPLICATION_DIRECTORY=/var/lib/spearhead
SPEARHEAD_SEQUENCE_FILE=sequence.txt
SPEARHEAD_REPLICATION_SERVER=https://planet.osm.org/replication/minute/
......@@ -42,7 +42,8 @@ function database::wait_for_connection() {
# Query command
#######################################
function database::execute_command() {
psql --host="${SPEARHEAD_DATABASE_HOST}" --username="${SPEARHEAD_DATABASE_USER}" --command="${1}"
psql --host="${SPEARHEAD_DATABASE_HOST}" --username="${SPEARHEAD_DATABASE_USER}" \
--command="${1}"
}
#######################################
......
......@@ -21,34 +21,65 @@
#######################################
# # Import OpenStreetMap Data file into database with osm2pgsql.
# Globals:
# SPEARHEAD_IMPORT_FILE
# SPEARHEAD_IMPORT_STYLE
# SPEARHEAD_DATABASE_HOST
# SPEARHEAD_DATABASE_USER
# SPEARHEAD_DATABASE_NAME
# SPEARHEAD_IMPORT_STYLE
# Arguments:
# None
# importfile
#######################################
function import::openstreetmap_data() {
if [[ -f "${SPEARHEAD_IMPORT_FILE}" ]]; then
printf " \nImporting OpenStreetMap data with "
osm2pgsql \
--host "${SPEARHEAD_DATABASE_HOST}" --username "${SPEARHEAD_DATABASE_USER}" \
if [[ -f "${1}" ]]; then
printf "\nImporting OpenStreetMap data with "
osm2pgsql --create --verbose \
--host "${SPEARHEAD_DATABASE_HOST}" \
--username "${SPEARHEAD_DATABASE_USER}" \
--database "${SPEARHEAD_DATABASE_NAME}" \
--verbose \
--create \
--slim \
--style "${SPEARHEAD_IMPORT_STYLE}" \
--slim \
--multi-geometry \
--hstore \
--latlong \
--extra-attributes \
--hstore-add-index \
--cache-strategy dense \
"${SPEARHEAD_IMPORT_FILE}"
"${1}"
else
printf "Error: Import file '%s' not found.\nAborting.\n" "${1}"
exit 1
fi
}
#######################################
# # Import OpenStreetMap changeset file into database with osm2pgsql.
# Globals:
# SPEARHEAD_DATABASE_HOST
# SPEARHEAD_DATABASE_USER
# SPEARHEAD_DATABASE_NAME
# SPEARHEAD_IMPORT_STYLE
# Arguments:
# importfile
#######################################
function import::openstreetmap_changeset() {
if [[ -f "${1}" ]]; then
printf "\nUpdating OpenStreetMap data with "
osm2pgsql --append --verbose \
--host "${SPEARHEAD_DATABASE_HOST}" \
--username "${SPEARHEAD_DATABASE_USER}" \
--database "${SPEARHEAD_DATABASE_NAME}" \
--style "${SPEARHEAD_IMPORT_STYLE}" \
--slim \
--multi-geometry \
--hstore \
--latlong \
--extra-attributes \
"${1}"
else
printf "Error: Import file '%s' not found.\nAborting.\n" "${SPEARHEAD_IMPORT_FILE}"
printf "Error: Changeset file '%s' not found.\nAborting.\n" "${1}"
exit 1
fi
}
#!/bin/bash
#
# spearhead: Import OpenStreetMap data into OpenBuildingMap
# spearhead: Import OpenStreetMap data into OpenBuildingMap and update continously.
# Copyright (C) 2021:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
......@@ -27,14 +27,14 @@ source /usr/local/lib/spearhead/includes/database.sh --source-only
source /usr/local/lib/spearhead/includes/import.sh --source-only
#######################################
# Main function of this script, to be executed for initial import.
# Initialize and populate the database.
# Globals:
# SPEARHEAD_DATABASE_HOST
# SPEARHEAD_DATABASE_NAME
# Arguments:
# None
#######################################
function main() {
function initialize() {
printf "Connecting to PostgreSQL on '%s':\n" "${SPEARHEAD_DATABASE_HOST}"
database::wait_for_connection
......@@ -56,13 +56,113 @@ function main() {
if ${database_exists}; then
printf "Assuming already imported database: %s\n" "${SPEARHEAD_DATABASE_NAME}"
else
import::openstreetmap_data
import::openstreetmap_data "${SPEARHEAD_IMPORT_FILE}"
# Write last sequence number from import file to sequence file
pyosmium-get-changes -v \
--start-osm-data "${SPEARHEAD_IMPORT_FILE}" \
--sequence-file "${SPEARHEAD_REPLICATION_DIRECTORY}/${SPEARHEAD_SEQUENCE_FILE}" \
--server "${SPEARHEAD_REPLICATION_SERVER}" \
--ignore-osmosis-headers
printf "OpenStreetMap data has been imported.\n"
fi
}
#######################################
# Define exit handler
# Globals:
# None
# Arguments:
# None
#######################################
function onexit() {
[ -f "previous-${SPEARHEAD_SEQUENCE_FILE}" ] &&
mv "previous-${SPEARHEAD_SEQUENCE_FILE}" "${SPEARHEAD_SEQUENCE_FILE}"
}
#######################################
# Constantly download OpenStreetMap data and update the database.
# Globals:
# SPEARHEAD_DATABASE_HOST
# SPEARHEAD_DATABASE_NAME
# Arguments:
# None
#######################################
function update() {
# Change to the spearhead replication data directory
mkdir -p "${SPEARHEAD_REPLICATION_DIRECTORY}"
cd "${SPEARHEAD_REPLICATION_DIRECTORY}"
# Disable JIT and parallel workers in PostgreSQL
export PGOPTIONS="-c jit=off -c max_parallel_workers_per_gather=0"
# Include an exit handler
trap onexit EXIT
printf "\nStart data-update daemon.\n\n"
# This runs as daemon and loops indefinitely
while true; do
update_file="changes-$(cat "${SPEARHEAD_SEQUENCE_FILE}").osc.gz"
# Save previous sequence file for rollback if needed
cp "${SPEARHEAD_SEQUENCE_FILE}" "previous-${SPEARHEAD_SEQUENCE_FILE}"
# Fetch set of OSM changesets
pyosmium-get-changes -v \
--server="${SPEARHEAD_REPLICATION_SERVER}" \
--sequence-file="${SPEARHEAD_SEQUENCE_FILE}" \
--outfile="${update_file}"
# Check for exit sttaus
status=$?
if [ $status -eq 0 ]; then
# Exit immediately on error
set -e
printf "Fetched new data from %s to %s into %s" \
"$(cat previous-"${SPEARHEAD_SEQUENCE_FILE}")" \
"$(cat "${SPEARHEAD_SEQUENCE_FILE}")" \
"${update_file}"
import::openstreetmap_changeset "${update_file}"
# After successfull import, the previous sequence file can be removed
rm "previous-${SPEARHEAD_SEQUENCE_FILE}"
# Delete old downloads
find . -name 'changes-*.gz' -mmin +300 -exec rm -f {} \;
# Disable exit immediately on error
set +e
elif [ $status -eq 3 ]; then
printf "No new data available.\n"
# Remove file, it is just emty
rm "${update_file}"
sleep 30
else
printf "Failed to fetch changeset file.\n"
# Remove file, it is just emty
rm -f "${update_file}"
sleep 300
fi
done
}
# Start the script
printf "\nWelcome to the OpenBuildingMap importer\n\n"
printf "\nWelcome to the OpenBuildingMap importer and updater\n\n"
# Call main function of the script
main
# Call main functions of the script
initialize
update
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment