Commit 44124e52 authored by Felix Delattre's avatar Felix Delattre
Browse files

Added first import logic

parent 98288fe0
Pipeline #19737 passed with stage
in 53 seconds
SOURCES=./spearhead
SOURCES=./spearhead includes/*.sh
LENGTH=96
check: $(SOURCES)
......
#!/bin/bash
#
# Helper functions around OpenStreetMap database import
# Copyright (C) 2021:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see http://www.gnu.org/licenses/.
#######################################
# Wait until database comes up and is available
# Globals:
# SPEARHEAD_DATABASE_HOST
# SPEARHEAD_DATABASE_USER
# Arguments:
# Query command
#######################################
function database::wait_for_connection() {
until database::execute_command "SELECT datname FROM pg_database;"; do
printf "Database is unavailable - trying again in 10s\n"
sleep 10
done
}
#######################################
# Execute a PostgreSQL command not addressing a particular database.
# Globals:
# SPEARHEAD_DATABASE_HOST
# SPEARHEAD_DATABASE_USER
# Arguments:
# Query command
#######################################
function database::execute_command() {
psql --host="${SPEARHEAD_DATABASE_HOST}" --username="${SPEARHEAD_DATABASE_USER}" --command="${1}"
}
#######################################
# Execute PostgreSQL query on a certain database.
# Globals:
# SPEARHEAD_DATABASE_HOST
# SPEARHEAD_DATABASE_NAME
# SPEARHEAD_DATABASE_USER
# Arguments:
# Query command
#######################################
function database::execute_query() {
psql --host="${SPEARHEAD_DATABASE_HOST}" --username="${SPEARHEAD_DATABASE_USER}" \
"${SPEARHEAD_DATABASE_NAME}" --command="${1}"
}
#######################################
# Add geospatial and advanced index extensions to a PostgreSQL database.
# Globals:
# SPEARHEAD_DATABASE_HOST
# SPEARHEAD_DATABASE_NAME
# SPEARHEAD_DATABASE_USER
# Arguments:
# None
#######################################
function database::add_extensions() {
if psql --host"=${SPEARHEAD_DATABASE_HOST}" --username="${SPEARHEAD_DATABASE_USER}" \
--command="\connect ${SPEARHEAD_DATABASE_NAME}"; then
if database::execute_query "CREATE EXTENSION postgis;"; then
printf " PostGIS extension added to %s database.\n" "${SPEARHEAD_DATABASE_NAME}"
else
printf " Could not add PostGIS extension added to %s database.\nAborting.\n" \
"${SPEARHEAD_DATABASE_NAME}"
exit 1
fi
if database::execute_query "CREATE EXTENSION hstore;"; then
printf " hstore extension added to %s database.\n" "${SPEARHEAD_DATABASE_NAME}"
else
printf " Could not add hstore extension added to %s database.\nAborting.\n" \
"${SPEARHEAD_DATABASE_NAME}"
exit 1
fi
if database::execute_query "CREATE EXTENSION btree_gin;"; then
printf " btree_gin extension added to %s database.\n" \
"${SPEARHEAD_DATABASE_NAME}"
else
printf " Could not add btree_gin extension added to %s database.\nAborting.\n" \
"${SPEARHEAD_DATABASE_NAME}"
exit 1
fi
if database::execute_query "CREATE EXTENSION btree_gist;"; then
printf " btree_gist extension added to %s database.\n" "${SPEARHEAD_DATABASE_NAME}"
else
printf " Could not add btree_gist extension added to %s database.\nAborting.\n" \
"${SPEARHEAD_DATABASE_NAME}"
exit 1
fi
else
printf "Error: Unable to connect to database: %s\n" "${SPEARHEAD_DATABASE_NAME}"
exit 1
fi
}
#!/bin/bash
#
# Helper functions around OpenStreetMap database import
# Copyright (C) 2021:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see http://www.gnu.org/licenses/.
#######################################
# # Import OpenStreetMap Data file into database with osm2pgsql.
# Globals:
# SPEARHEAD_IMPORT_FILE
# SPEARHEAD_IMPORT_STYLE
# SPEARHEAD_DATABASE_HOST
# SPEARHEAD_DATABASE_USER
# SPEARHEAD_DATABASE_NAME
# Arguments:
# None
#######################################
function import::openstreetmap_data() {
if [[ -f "${SPEARHEAD_IMPORT_FILE}" ]]; then
printf " \nImporting OpenStreetMap data with "
osm2pgsql \
--host "${SPEARHEAD_DATABASE_HOST}" --username "${SPEARHEAD_DATABASE_USER}" \
--database "${SPEARHEAD_DATABASE_NAME}" \
--verbose \
--create \
--slim \
--style "${SPEARHEAD_IMPORT_STYLE}" \
--multi-geometry \
--hstore \
--latlong \
--extra-attributes \
--hstore-add-index \
--cache-strategy dense \
"${SPEARHEAD_IMPORT_FILE}"
else
printf "Error: Import file '%s' not found.\nAborting.\n" "${SPEARHEAD_IMPORT_FILE}"
exit 1
fi
}
#!/bin/bash
#
# spearhead
# spearhead: Import OpenStreetMap data into OpenBuildingMap
# Copyright (C) 2021:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
......@@ -19,15 +19,50 @@
# along with this program. If not, see http://www.gnu.org/licenses/.
#######################################
# Main function of this script
# Import other shell script(s)
#######################################
# shellcheck source=includes/database.sh
source /usr/local/lib/spearhead/includes/database.sh --source-only
# shellcheck source=includes/import.sh
source /usr/local/lib/spearhead/includes/import.sh --source-only
#######################################
# Main function of this script, to be executed for initial import.
# Globals:
# None
# SPEARHEAD_DATABASE_HOST
# SPEARHEAD_DATABASE_NAME
# Arguments:
# None
#######################################
function main() {
printf "Hello world\n"
printf "Connecting to PostgreSQL on '%s':\n" "${SPEARHEAD_DATABASE_HOST}"
database::wait_for_connection
# Initiate new or reuse existing database
if database::execute_command "\connect ${SPEARHEAD_DATABASE_NAME}"; then
printf "Reuse existing database: %s\n" "${SPEARHEAD_DATABASE_NAME}"
else
database::execute_command "CREATE DATABASE ${SPEARHEAD_DATABASE_NAME};"
database::add_extensions
printf "Initiated new database: %s\n" "${SPEARHEAD_DATABASE_NAME}"
fi
# Import OpenStreetMap data if database is empty
database_exists=$(psql --host="${SPEARHEAD_DATABASE_HOST}" \
--username="${SPEARHEAD_DATABASE_USER}" "${SPEARHEAD_DATABASE_NAME}" --tuples-only \
--command="SELECT CASE WHEN EXISTS \
(SELECT FROM pg_tables WHERE schemaname = 'public' \
AND tablename = 'planet_osm_line' LIMIT 1) THEN 'true' ELSE 'false' end;")
if ${database_exists}; then
printf "Assuming already imported database: %s\n" "${SPEARHEAD_DATABASE_NAME}"
else
import::openstreetmap_data
printf "OpenStreetMap data has been imported.\n"
fi
}
# Start the script
printf "\nWelcome to the OpenBuildingMap importer\n\n"
# Call main function of the script
main
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment