Commit af196aa2 authored by Felix Delattre's avatar Felix Delattre
Browse files

Integrated spearhead as core for import logic

parent 4521a438
# Copyright (c) 2021:
# Copyright (C) 2021:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
......@@ -19,8 +19,16 @@ FROM debian:bullseye-slim
ENV DEBIAN_FRONTEND noninteractive
# Install software
RUN apt-get update
RUN apt-get install --no-install-recommends -y osm2pgsql postgresql-client
RUN apt update
RUN apt install --no-install-recommends -y \
ca-certificates \
git \
osm2pgsql \
postgresql-client
# Install spearhead
RUN cd /usr/local/lib && git clone https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/spearhead.git
RUN ln -s /usr/local/lib/spearhead/spearhead /usr/local/bin/spearhead
# Cleanup
RUN apt-get autoremove --yes $build_dependencies && \
......@@ -30,6 +38,4 @@ RUN apt-get autoremove --yes $build_dependencies && \
VOLUME ["/tmp/import"]
# Start script
COPY ./files/start.sh /usr/local/bin
CMD /usr/local/bin/start.sh
CMD /usr/local/bin/spearhead
......@@ -3,17 +3,17 @@ Docker OBM import OSM
=====================
Import data from OpenStreetMap (OSM) to the OpenBuildingMap (OBM) Buildings database using
`osm2pgsql <https://github.com/openstreetmap/osm2pgsql>`__ and keep it up-to-data through
pulling regular updates from OSM.
`spearhead <https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/spearhead>`__ and keep
it up-to-data through pulling regular updates from OSM.
Define variables
----------------
* :code:`OBM_DATABASE_HOST` - Host url or IP of the database
* :code:`OBM_DATABASE_NAME` - Name of the database
* :code:`OBM_DATABASE_USER` - User to connect to the database
* :code:`OBM_IMPORT_STYLE` - Absolute path to an available osm2pgsql style file
* :code:`OBM_IMPORT_FILE` - Absolute path to an available OSM pbf-file containing data of an area
* :code:`SPEARHEAD_DATABASE_HOST` - Host url or IP of the database
* :code:`SPEARHEAD_DATABASE_NAME` - Name of the database
* :code:`SPEARHEAD_DATABASE_USER` - User to connect to the database
* :code:`SPEARHEAD_IMPORT_STYLE` - Absolute path to an available osm2pgsql style file
* :code:`SPEARHEAD_IMPORT_FILE` - Absolute path to an available OSM pbf-file containing data of an area
Copyright and copyleft
----------------------
......
#!/bin/bash
#
# Initialize import of OpenStreetMap data into OpenBuildingMap
# Copyright (C) 2021:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see http://www.gnu.org/licenses/.
#######################################
# Main function of this script, to be executed for initial import.
# Globals:
# OBM_DATABASE_HOST
# OBM_DATABASE_NAME
# Arguments:
# None
#######################################
function main() {
printf "Connecting to PostgreSQL on '%s':\n" "${OBM_DATABASE_HOST}"
# Wait until database comes up and is available
until execute_database_command "SELECT datname FROM pg_database;"; do
printf "Database is unavailable - trying again in 10s\n"
sleep 10
done
# Initiate new or reuse existing database
if execute_database_command "\connect ${OBM_DATABASE_NAME}"; then
printf "Reuse existing database: %s\n" "${OBM_DATABASE_NAME}"
else
execute_database_command "CREATE DATABASE ${OBM_DATABASE_NAME};"
add_database_extensions
printf "Initiated new database: %s\n" "${OBM_DATABASE_NAME}"
fi
# Obtain mapping information for database schema
assemble_opentstreetmap_database_style
# Import OpenStreetMap data if database is empty
database_exists=$(psql --host="${OBM_DATABASE_HOST}" \
--username="${OBM_DATABASE_USER}" "${OBM_DATABASE_NAME}" --tuples-only \
--command="SELECT CASE WHEN EXISTS \
(SELECT FROM pg_tables WHERE schemaname = 'public' \
AND tablename = 'planet_osm_line' LIMIT 1) THEN 'true' ELSE 'false' end;")
if ${database_exists}; then
printf "Assuming already imported database: %s\n" "${OBM_DATABASE_NAME}"
else
import_openstreetmap_data
printf "OpenStreetMap data has been imported.\n"
fi
}
#######################################
# Exectute postgresql command outside of a specific database.
# Globals:
# OBM_DATABASE_HOST
# OBM_DATABASE_USER
# Arguments:
# Query command
#######################################
function execute_database_command() {
psql --host="${OBM_DATABASE_HOST}" --username="${OBM_DATABASE_USER}" --command="${1}"
}
#######################################
# Exectute postgresql query on a certain database.
# Globals:
# OBM_DATABASE_HOST
# OBM_DATABASE_NAME
# OBM_DATABASE_USER
# Arguments:
# Query command
#######################################
function execute_database_query() {
psql --host="${OBM_DATABASE_HOST}" --username="${OBM_DATABASE_USER}" "${OBM_DATABASE_NAME}" \
--command="${1}"
}
#######################################
# Add geospatial and advanced index extensions to a postgresql database.
# Globals:
# OBM_DATABASE_HOST
# OBM_DATABASE_NAME
# OBM_DATABASE_USER
# Arguments:
# None
#######################################
function add_database_extensions() {
if psql --host"=${OBM_DATABASE_HOST}" --username="${OBM_DATABASE_USER}" \
--command="\connect ${OBM_DATABASE_NAME}"; then
if execute_database_query "CREATE EXTENSION postgis;"; then
printf " postgis extension added to %s database.\n" "${OBM_DATABASE_NAME}"
else
printf " Could not add postgis extension added to %s database.\nAborting.\n" "${OBM_DATABASE_NAME}"
exit 1
fi
if execute_database_query "CREATE EXTENSION hstore;"; then
printf " hstore extension added to %s database.\n" "${OBM_DATABASE_NAME}"
else
printf " Could not add hstore extension added to %s database.\nAborting.\n" "${OBM_DATABASE_NAME}"
exit 1
fi
if execute_database_query "CREATE EXTENSION btree_gin;"; then
printf " btree_gin extension added to %s database.\n" "${OBM_DATABASE_NAME}"
else
printf " Could not add btree_gin extension added to %s database.\nAborting.\n" "${OBM_DATABASE_NAME}"
exit 1
fi
if execute_database_query "CREATE EXTENSION btree_gist;"; then
printf " btree_gist extension added to %s database.\n" "${OBM_DATABASE_NAME}"
else
printf " Could not add btree_gist extension added to %s database.\nAborting.\n" "${OBM_DATABASE_NAME}"
exit 1
fi
else
printf "Error: Unable to connect to database: %s\n" "${OBM_DATABASE_NAME}"
exit 1
fi
}
#######################################
# # Prepare style file for osm2pgsql, based on standard 'empty.style' and provided style file.
# Globals:
# OBM_IMPORT_STYLE
# Arguments:
# None
# Outputs:
# Writes combined style file to '/usr/share/osm2pgsql/the.style'
#######################################
function assemble_opentstreetmap_database_style() {
base_style="/usr/share/osm2pgsql/empty.style"
if [[ ! -r "${base_style}" ]]; then
printf "Cannot find the 'empty.style' file of osm2pgsql\n"
exit 1
fi
cat /usr/share/osm2pgsql/empty.style >/usr/share/osm2pgsql/the.style
cat "${OBM_IMPORT_STYLE}" >>/usr/share/osm2pgsql/the.style
}
#######################################
# # Import OpenStreetMap Data file into database with osm2pgsql.
# Globals:
# OBM_IMPORT_FILE
# OBM_DATABASE_HOST
# OBM_DATABASE_USER
# OBM_DATABASE_NAME
# Arguments:
# None
#######################################
function import_openstreetmap_data() {
if [[ -f "${OBM_IMPORT_FILE}" ]]; then
printf " \nImporting OpenStreetMap data with "
osm2pgsql \
--host "${OBM_DATABASE_HOST}" --username "${OBM_DATABASE_USER}" \
--database "${OBM_DATABASE_NAME}" \
--verbose \
--create \
--slim \
--style "/usr/share/osm2pgsql/the.style" \
--multi-geometry \
--hstore \
--latlong \
--extra-attributes \
--hstore-add-index \
--cache-strategy dense \
"${OBM_IMPORT_FILE}"
else
printf "Error: Import file '%s' not found.\nAborting.\n" "${OBM_IMPORT_FILE}"
exit 1
fi
}
# Start the script
printf "\nWelcome to the OpenBuildingMap importer\n\n"
# Call main function of the script
main
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment