Commit 4c3e9b30 authored by Felix Delattre's avatar Felix Delattre
Browse files

Added Docker setup for initial database import

parent 8a3c4d00
# Copyright (c) 2021:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see http://www.gnu.org/licenses/.
FROM debian:bullseye-slim
ENV DEBIAN_FRONTEND noninteractive
# Install software
RUN apt-get update
RUN apt-get install --no-install-recommends -y osm2pgsql postgresql-client
# Cleanup
RUN apt-get autoremove --yes $build_dependencies && \
apt-get autoremove --yes && \
apt-get clean --yes && \
rm -rf /var/lib/apt/* /var/cache/apt/* /root/.cache
VOLUME ["/tmp/import"]
# Start script
COPY ./files/start.sh /usr/local/bin
CMD /usr/local/bin/start.sh
......@@ -616,4 +616,4 @@ an absolute waiver of all civil liability in connection with the
Program, unless a warranty or assumption of liability accompanies a
copy of the Program in return for a fee.
END OF TERMS AND CONDITIONS
\ No newline at end of file
END OF TERMS AND CONDITIONS
===================
Docker OBM Importer
===================
=====================
Docker OBM import OSM
=====================
This is a Docker container for importing and updating data for the
OpenBuildingMap database.
Import data from OpenStreetMap (OSM) to the OpenBuildingMap (OBM) Buildings database using
`osm2pgsql <https://github.com/openstreetmap/osm2pgsql>`__ and keep it up-to-data through
pulling regular updates from OSM.
Define variables
----------------
* :code:`OBM_DATABASE_HOST` - Host url or IP of the database
* :code:`OBM_DATABASE_MAME` - Name of the database
* :code:`OBM_DATABASE_USER` - User to connect to the database
* :code:`OBM_IMPORT_STYLE` - Absolute path to an available osm2pgsql style file
* :code:`OBM_IMPORT_FILE` - Absolute path to an available osm pbf file containing data of an area
Copyright and copyleft
----------------------
Copyright (C) 2020
Copyright (c) 2021
* Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
......
#!/bin/bash
#
# Initialize import of OpenStreetMap data into OpenBuildingMap
# Copyright (C) 2021:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see http://www.gnu.org/licenses/.
#######################################
# Main function of this script, to be executed for initial import.
# Globals:
# OBM_DATABASE_HOST
# OBM_DATABASE_NAME
# Arguments:
# None
#######################################
function main() {
printf "Connecting to PostgreSQL on '%s':\n" "${OBM_DATABASE_HOST}"
# Wait until database comes up and is available
until execute_database_command "SELECT datname FROM pg_database;"; do
printf "Database is unavailable - trying again in 10s\n"
sleep 10
done
# Initiate new or reuse existing database
if execute_database_command "\connect ${OBM_DATABASE_NAME}"; then
printf "Reuse existing database: %s\n" "${OBM_DATABASE_NAME}"
else
execute_database_command "CREATE DATABASE ${OBM_DATABASE_NAME};"
add_database_extensions
printf "Initiated new database: %s\n" "${OBM_DATABASE_NAME}"
fi
# Obtain mapping information for database schema
assemble_opentstreetmap_database_style
# Import OpenStreetMap data if database is empty
database_exists=$(psql --host="${OBM_DATABASE_HOST}" \
--username="${OBM_DATABASE_USER}" "${OBM_DATABASE_NAME}" --tuples-only \
--command="SELECT CASE WHEN EXISTS \
(SELECT FROM pg_tables WHERE schemaname = 'public' \
AND tablename = 'planet_osm_line' LIMIT 1) THEN 'true' ELSE 'false' end;")
if ${database_exists}; then
printf "Assuming already imported database: %s\n" "${OBM_DATABASE_NAME}"
else
import_openstreetmap_data
printf "OpenStreetMap data has been imported.\n"
fi
}
#######################################
# Exectute postgresql command outside of a specific database.
# Globals:
# OBM_DATABASE_HOST
# OBM_DATABASE_USER
# Arguments:
# Query command
#######################################
function execute_database_command() {
psql --host="${OBM_DATABASE_HOST}" --username="${OBM_DATABASE_USER}" --command="${1}"
}
#######################################
# Exectute postgresql query on a certain database.
# Globals:
# OBM_DATABASE_HOST
# OBM_DATABASE_NAME
# OBM_DATABASE_USER
# Arguments:
# Query command
#######################################
function execute_database_query() {
psql --host="${OBM_DATABASE_HOST}" --username="${OBM_DATABASE_USER}" "${OBM_DATABASE_NAME}" \
--command="${1}"
}
#######################################
# Add geospatial and advanced index extensions to a postgresql database.
# Globals:
# OBM_DATABASE_HOST
# OBM_DATABASE_NAME
# OBM_DATABASE_USER
# Arguments:
# None
#######################################
function add_database_extensions() {
if psql --host"=${OBM_DATABASE_HOST}" --username="${OBM_DATABASE_USER}" \
--command="\connect ${OBM_DATABASE_NAME}"; then
if execute_database_query "CREATE EXTENSION postgis;"; then
printf " postgis extension added to %s database.\n" "${OBM_DATABASE_NAME}"
else
printf " Could not add postgis extension added to %s database.\nAborting.\n" "${OBM_DATABASE_NAME}"
exit 1
fi
if execute_database_query "CREATE EXTENSION hstore;"; then
printf " hstore extension added to %s database.\n" "${OBM_DATABASE_NAME}"
else
printf " Could not add hstore extension added to %s database.\nAborting.\n" "${OBM_DATABASE_NAME}"
exit 1
fi
if execute_database_query "CREATE EXTENSION btree_gin;"; then
printf " btree_gin extension added to %s database.\n" "${OBM_DATABASE_NAME}"
else
printf " Could not add btree_gin extension added to %s database.\nAborting.\n" "${OBM_DATABASE_NAME}"
exit 1
fi
if execute_database_query "CREATE EXTENSION btree_gist;"; then
printf " btree_gist extension added to %s database.\n" "${OBM_DATABASE_NAME}"
else
printf " Could not add btree_gist extension added to %s database.\nAborting.\n" "${OBM_DATABASE_NAME}"
exit 1
fi
else
printf "Error: Unable to connect to database: %s\n" "${OBM_DATABASE_NAME}"
exit 1
fi
}
#######################################
# # Prepare style file for osm2pgsql, based on standard 'empty.style' and provided style file.
# Globals:
# OBM_IMPORT_STYLE
# Arguments:
# None
# Outputs:
# Writes combined style file to '/usr/share/osm2pgsql/the.style'
#######################################
function assemble_opentstreetmap_database_style() {
base_style="/usr/share/osm2pgsql/empty.style"
if [[ ! -r "${base_style}" ]]; then
printf "Cannot find the 'empty.style' file of osm2pgsql\n"
exit 1
fi
cat /usr/share/osm2pgsql/empty.style >/usr/share/osm2pgsql/the.style
cat "${OBM_IMPORT_STYLE}" >>/usr/share/osm2pgsql/the.style
}
#######################################
# # Import OpenStreetMap Data file into database with osm2pgsql.
# Globals:
# OBM_IMPORT_FILE
# OBM_DATABASE_HOST
# OBM_DATABASE_USER
# OBM_DATABASE_NAME
# Arguments:
# None
#######################################
function import_openstreetmap_data() {
if [[ -f "${OBM_IMPORT_FILE}" ]]; then
printf " \nImporting OpenStreetMap data with "
osm2pgsql \
--host "${OBM_DATABASE_HOST}" --username "${OBM_DATABASE_USER}" \
--database "${OBM_DATABASE_NAME}" \
--verbose \
--create \
--slim \
--style "/usr/share/osm2pgsql/the.style" \
--multi-geometry \
--hstore \
--latlong \
--extra-attributes \
--hstore-add-index \
--cache-strategy dense \
"${OBM_IMPORT_FILE}"
else
printf "Error: Import file '%s' not found.\nAborting.\n" "${OBM_IMPORT_FILE}"
exit 1
fi
}
# Start the script
printf "\nWelcome to the OpenBuildingMap importer\n\n"
# Call main function of the script
main
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment