Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Dynamic Exposure
OpenBuildingMap
spearhead
Commits
0e5cbb44
Commit
0e5cbb44
authored
Mar 01, 2021
by
Felix Delattre
Browse files
Introduced OpenStreetMap data updater
parent
44124e52
Pipeline
#27981
passed with stage
in 51 seconds
Changes
4
Pipelines
3
Hide whitespace changes
Inline
Side-by-side
env-example
0 → 100644
View file @
0e5cbb44
# SPEARHEAD database connection
#
SPEARHEAD_DATABASE_HOST=localhost
SPEARHEAD_DATABASE_NAME=postgres
SPEARHEAD_DATABASE_USER=gis
# Location of the import conversion style definitions for osm2pgsql
#
SPEARHEAD_IMPORT_STYLE=/usr/local/share/osm2pgsql/example.style
# Location of the OpenStreetMap data file to import
#
SPEARHEAD_IMPORT_FILE=/tmp/import/data.osm.pbf
# OpenStreetMap replication information
#
SPEARHEAD_REPLICATION_DIRECTORY=/var/lib/spearhead
SPEARHEAD_SEQUENCE_FILE=sequence.txt
SPEARHEAD_REPLICATION_SERVER=https://planet.osm.org/replication/minute/
includes/database.sh
View file @
0e5cbb44
...
...
@@ -42,7 +42,8 @@ function database::wait_for_connection() {
# Query command
#######################################
function
database::execute_command
()
{
psql
--host
=
"
${
SPEARHEAD_DATABASE_HOST
}
"
--username
=
"
${
SPEARHEAD_DATABASE_USER
}
"
--command
=
"
${
1
}
"
psql
--host
=
"
${
SPEARHEAD_DATABASE_HOST
}
"
--username
=
"
${
SPEARHEAD_DATABASE_USER
}
"
\
--command
=
"
${
1
}
"
}
#######################################
...
...
includes/import.sh
View file @
0e5cbb44
...
...
@@ -21,34 +21,65 @@
#######################################
# # Import OpenStreetMap Data file into database with osm2pgsql.
# Globals:
# SPEARHEAD_IMPORT_FILE
# SPEARHEAD_IMPORT_STYLE
# SPEARHEAD_DATABASE_HOST
# SPEARHEAD_DATABASE_USER
# SPEARHEAD_DATABASE_NAME
# SPEARHEAD_IMPORT_STYLE
# Arguments:
#
Non
e
#
importfil
e
#######################################
function
import::openstreetmap_data
()
{
if
[[
-f
"
${
SPEARHEAD_IMPORT_FILE
}
"
]]
;
then
printf
"
\n
Importing OpenStreetMap data with "
osm2pgsql
\
--host
"
${
SPEARHEAD_DATABASE_HOST
}
"
--username
"
${
SPEARHEAD_DATABASE_USER
}
"
\
if
[[
-f
"
${
1
}
"
]]
;
then
printf
"
\n
Importing OpenStreetMap data with "
osm2pgsql
--create
--verbose
\
--host
"
${
SPEARHEAD_DATABASE_HOST
}
"
\
--username
"
${
SPEARHEAD_DATABASE_USER
}
"
\
--database
"
${
SPEARHEAD_DATABASE_NAME
}
"
\
--verbose
\
--create
\
--slim
\
--style
"
${
SPEARHEAD_IMPORT_STYLE
}
"
\
--slim
\
--multi-geometry
\
--hstore
\
--latlong
\
--extra-attributes
\
--hstore-add-index
\
--cache-strategy
dense
\
"
${
SPEARHEAD_IMPORT_FILE
}
"
"
${
1
}
"
else
printf
"Error: Import file '%s' not found.
\n
Aborting.
\n
"
"
${
1
}
"
exit
1
fi
}
#######################################
# # Import OpenStreetMap changeset file into database with osm2pgsql.
# Globals:
# SPEARHEAD_DATABASE_HOST
# SPEARHEAD_DATABASE_USER
# SPEARHEAD_DATABASE_NAME
# SPEARHEAD_IMPORT_STYLE
# Arguments:
# importfile
#######################################
function
import::openstreetmap_changeset
()
{
if
[[
-f
"
${
1
}
"
]]
;
then
printf
"
\n
Updating OpenStreetMap data with "
osm2pgsql
--append
--verbose
\
--host
"
${
SPEARHEAD_DATABASE_HOST
}
"
\
--username
"
${
SPEARHEAD_DATABASE_USER
}
"
\
--database
"
${
SPEARHEAD_DATABASE_NAME
}
"
\
--style
"
${
SPEARHEAD_IMPORT_STYLE
}
"
\
--slim
\
--multi-geometry
\
--hstore
\
--latlong
\
--extra-attributes
\
"
${
1
}
"
else
printf
"Error:
Impor
t file '%s' not found.
\n
Aborting.
\n
"
"
${
SPEARHEAD_IMPORT_FILE
}
"
printf
"Error:
Changese
t file '%s' not found.
\n
Aborting.
\n
"
"
${
1
}
"
exit
1
fi
}
spearhead
View file @
0e5cbb44
#!/bin/bash
#
# spearhead: Import OpenStreetMap data into OpenBuildingMap
# spearhead: Import OpenStreetMap data into OpenBuildingMap
and update continously.
# Copyright (C) 2021:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
...
...
@@ -27,14 +27,14 @@ source /usr/local/lib/spearhead/includes/database.sh --source-only
source
/usr/local/lib/spearhead/includes/import.sh
--source-only
#######################################
#
Main function of this script, to be executed for initial import
.
#
Initialize and populate the database
.
# Globals:
# SPEARHEAD_DATABASE_HOST
# SPEARHEAD_DATABASE_NAME
# Arguments:
# None
#######################################
function
ma
in
()
{
function
in
itialize
()
{
printf
"Connecting to PostgreSQL on '%s':
\n
"
"
${
SPEARHEAD_DATABASE_HOST
}
"
database::wait_for_connection
...
...
@@ -56,13 +56,113 @@ function main() {
if
${
database_exists
}
;
then
printf
"Assuming already imported database: %s
\n
"
"
${
SPEARHEAD_DATABASE_NAME
}
"
else
import::openstreetmap_data
import::openstreetmap_data
"
${
SPEARHEAD_IMPORT_FILE
}
"
# Write last sequence number from import file to sequence file
pyosmium-get-changes
-v
\
--start-osm-data
"
${
SPEARHEAD_IMPORT_FILE
}
"
\
--sequence-file
"
${
SPEARHEAD_REPLICATION_DIRECTORY
}
/
${
SPEARHEAD_SEQUENCE_FILE
}
"
\
--server
"
${
SPEARHEAD_REPLICATION_SERVER
}
"
\
--ignore-osmosis-headers
printf
"OpenStreetMap data has been imported.
\n
"
fi
}
#######################################
# Define exit handler
# Globals:
# None
# Arguments:
# None
#######################################
function
onexit
()
{
[
-f
"previous-
${
SPEARHEAD_SEQUENCE_FILE
}
"
]
&&
mv
"previous-
${
SPEARHEAD_SEQUENCE_FILE
}
"
"
${
SPEARHEAD_SEQUENCE_FILE
}
"
}
#######################################
# Constantly download OpenStreetMap data and update the database.
# Globals:
# SPEARHEAD_DATABASE_HOST
# SPEARHEAD_DATABASE_NAME
# Arguments:
# None
#######################################
function
update
()
{
# Change to the spearhead replication data directory
mkdir
-p
"
${
SPEARHEAD_REPLICATION_DIRECTORY
}
"
cd
"
${
SPEARHEAD_REPLICATION_DIRECTORY
}
"
# Disable JIT and parallel workers in PostgreSQL
export
PGOPTIONS
=
"-c jit=off -c max_parallel_workers_per_gather=0"
# Include an exit handler
trap
onexit EXIT
printf
"
\n
Start data-update daemon.
\n\n
"
# This runs as daemon and loops indefinitely
while
true
;
do
update_file
=
"changes-
$(
cat
"
${
SPEARHEAD_SEQUENCE_FILE
}
"
)
.osc.gz"
# Save previous sequence file for rollback if needed
cp
"
${
SPEARHEAD_SEQUENCE_FILE
}
"
"previous-
${
SPEARHEAD_SEQUENCE_FILE
}
"
# Fetch set of OSM changesets
pyosmium-get-changes
-v
\
--server
=
"
${
SPEARHEAD_REPLICATION_SERVER
}
"
\
--sequence-file
=
"
${
SPEARHEAD_SEQUENCE_FILE
}
"
\
--outfile
=
"
${
update_file
}
"
# Check for exit sttaus
status
=
$?
if
[
$status
-eq
0
]
;
then
# Exit immediately on error
set
-e
printf
"Fetched new data from %s to %s into %s"
\
"
$(
cat
previous-
"
${
SPEARHEAD_SEQUENCE_FILE
}
"
)
"
\
"
$(
cat
"
${
SPEARHEAD_SEQUENCE_FILE
}
"
)
"
\
"
${
update_file
}
"
import::openstreetmap_changeset
"
${
update_file
}
"
# After successfull import, the previous sequence file can be removed
rm
"previous-
${
SPEARHEAD_SEQUENCE_FILE
}
"
# Delete old downloads
find
.
-name
'changes-*.gz'
-mmin
+300
-exec
rm
-f
{}
\;
# Disable exit immediately on error
set
+e
elif
[
$status
-eq
3
]
;
then
printf
"No new data available.
\n
"
# Remove file, it is just emty
rm
"
${
update_file
}
"
sleep
30
else
printf
"Failed to fetch changeset file.
\n
"
# Remove file, it is just emty
rm
-f
"
${
update_file
}
"
sleep
300
fi
done
}
# Start the script
printf
"
\n
Welcome to the OpenBuildingMap importer
\n\n
"
printf
"
\n
Welcome to the OpenBuildingMap importer
and updater
\n\n
"
# Call main function of the script
main
# Call main functions of the script
initialize
update
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment