Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Dynamic Exposure
Global Dynamic Exposure
exposure-coquimbo
Commits
45515949
Commit
45515949
authored
Mar 16, 2021
by
Danijel Schorlemmer
Browse files
Changed functions for inserting datasets to tables and redesigned tables
parent
4ec7c2d5
Pipeline
#20789
passed with stage
in 1 minute and 36 seconds
Changes
1
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
exposurecoquimbo/database.py
View file @
45515949
...
...
@@ -109,27 +109,27 @@ class CoquimboDatabase(Database):
def
create_tables
(
self
):
"""
Creates all necessary tables in the database. These are:
Location
: Stores the locations of the SARA exposure data
Asset
: Stores the assets per location
Voronoi
: Stores the Voronoi cells around each location
Building
s
: Stores the OpenStreetMap buildings
Exposure
: Stores the resulting exposure assets for each building
Location: Stores the locations of the SARA exposure data
Asset : Stores the assets per location
Voronoi : Stores the Voronoi cells around each location
Building: Stores the OpenStreetMap buildings
Exposure: Stores the resulting exposure assets for each building
"""
# Create table Location
s
# Create table Location
sql_statement
=
"CREATE TABLE Location ("
sql_statement
+=
"id
x
INTEGER PRIMARY KEY AUTOINCREMENT, "
sql_statement
+=
"occupancy TEXT, "
sql_statement
+=
"
id
INTEGER, "
sql_statement
+=
"
name
TEXT)"
sql_statement
+=
"id
INTEGER PRIMARY KEY AUTOINCREMENT, "
sql_statement
+=
"occupancy
TEXT, "
sql_statement
+=
"
admin_id
INTEGER, "
sql_statement
+=
"
admin_name
TEXT)"
self
.
connection
.
execute
(
sql_statement
)
sql_statement
=
"SELECT AddGeometryColumn('Location', 'geom', 4326, 'POINT', 'XY')"
self
.
connection
.
execute
(
sql_statement
)
logger
.
debug
(
"Table Location
s
created"
)
logger
.
debug
(
"Table Location created"
)
# Create table Asset
sql_statement
=
"CREATE TABLE Asset ("
sql_statement
+=
"location_id
x
INTEGER, "
sql_statement
+=
"location_id
INTEGER, "
sql_statement
+=
"taxonomy TEXT, "
sql_statement
+=
"number REAL, "
sql_statement
+=
"structural REAL, "
...
...
@@ -137,119 +137,163 @@ class CoquimboDatabase(Database):
sql_statement
+=
"structural_per_building REAL, "
sql_statement
+=
"night_per_building REAL)"
self
.
connection
.
execute
(
sql_statement
)
logger
.
debug
(
"Table Asset
s
created"
)
logger
.
debug
(
"Table Asset created"
)
# Create table Voronoi
sql_statement
=
"CREATE TABLE Voronoi ("
sql_statement
+=
"location_id
x
INTEGER)"
sql_statement
+=
"location_id INTEGER)"
self
.
connection
.
execute
(
sql_statement
)
sql_statement
=
"SELECT AddGeometryColumn('Voronoi', 'geom', 4326, 'POLYGON', 'XY')"
self
.
connection
.
execute
(
sql_statement
)
logger
.
debug
(
"Table Voronoi created"
)
# Create table Building
s
# Create table Building
sql_statement
=
"CREATE TABLE Building ("
sql_statement
+=
"id
x
INTEGER PRIMARY KEY AUTOINCREMENT, "
sql_statement
+=
"location_id
x
INTEGER, "
sql_statement
+=
"quadkey
TEXT)"
sql_statement
+=
"id INTEGER PRIMARY KEY AUTOINCREMENT, "
sql_statement
+=
"location_id INTEGER, "
sql_statement
+=
"quadkey TEXT)"
self
.
connection
.
execute
(
sql_statement
)
sql_statement
=
"SELECT AddGeometryColumn('Building', "
sql_statement
+=
"'geom', 4326, 'MULTIPOLYGON', 'XY')"
self
.
connection
.
execute
(
sql_statement
)
sql_statement
=
"SELECT AddGeometryColumn('Building', 'centroid', 4326, 'POINT', 'XY')"
self
.
connection
.
execute
(
sql_statement
)
logger
.
debug
(
"Table Building
s
created"
)
logger
.
debug
(
"Table Building created"
)
# Create table Exposure
sql_statement
=
"CREATE TABLE Exposure ("
sql_statement
+=
"idx INTEGER PRIMARY KEY AUTOINCREMENT, "
sql_statement
+=
"building_idx INTEGER, "
sql_statement
+=
"taxonomy TEXT, "
sql_statement
+=
"number REAL, "
sql_statement
+=
"structural REAL, "
sql_statement
+=
"night REAL)"
sql_statement
=
"CREATE TABLE BuildingAsset ("
sql_statement
+=
"building_id INTEGER, "
sql_statement
+=
"taxonomy TEXT, "
sql_statement
+=
"number REAL, "
sql_statement
+=
"structural REAL, "
sql_statement
+=
"night REAL)"
self
.
connection
.
execute
(
sql_statement
)
logger
.
debug
(
"Table
Exposure
created"
)
logger
.
debug
(
"Table
BuildingAsset
created"
)
def
insert_location
(
self
,
dataset
):
def
insert_location
(
self
,
occupancy
,
geom
,
admin_id
,
admin_name
):
"""
Inserts a dataset
in
the Location
s
table.
Inserts a dataset
to
the Location table.
Args:
dataset (tuple):
(occupancy,
point geometry in WKT,
location id,
location name)
occupancy (str):
Occupancy type of the SARA data for this location
geom (point):
Geometry (Spatialite) of the location (POINT)
admin_id (int):
ID of administrative unit of the location
admin_name (str):
Name of administrative unit of the location
Return:
Index of inserted dataset (Location
s
.id
x
)
Index of inserted dataset (Location.id)
"""
sql_statement
=
"INSERT INTO Location "
sql_statement
+=
"(occupancy, geom,
id,
name) "
sql_statement
+=
"VALUES ('%s', %s, %
s
, '%s')"
%
dataset
sql_statement
+=
"(occupancy, geom,
admin_id, admin_
name) "
sql_statement
+=
"VALUES ('%s', %s, %
d
, '%s')"
%
(
occupancy
,
geom
,
admin_id
,
admin_name
)
self
.
cursor
.
execute
(
sql_statement
)
return
self
.
cursor
.
lastrowid
def
insert_asset
(
self
,
dataset
):
def
insert_asset
(
self
,
location_id
,
taxonomy
,
number
,
structural
,
night
,
structural_per_building
,
night_per_building
,
):
"""
Inserts a dataset
in
the Asset
s
table.
Inserts a dataset
to
the Asset table.
Args:
dataset (tuple):
(location index corresponding to Locations.idx,
taxonomy,
number of buildings,
structural value of buildings (total value),
number of people inside buildings during night (total value))
Return:
Index of inserted dataset (Buildings.idx)
location_id (int):
ID of the location of SARA data (corresponds to Location.id)
taxonomy (str):
Taxonomy string of the asset
number (float):
Number of buildings represented by the asset
structural (float):
Total structural value of the asset
night (float):
Total number of people in the buildings of the asset during night
structural_per_building (float):
Structural value per building of the asset
night_per_building (float):
Number of people per building of the asset during night
"""
sql_statement
=
"INSERT INTO Asset "
sql_statement
+=
"(location_id
x
, taxonomy, number, structural, night, "
sql_statement
+=
"(location_id, taxonomy, number, structural, night, "
sql_statement
+=
"structural_per_building, night_per_building) "
sql_statement
+=
"VALUES (%s, '%s', %s, %s, %s, %f, %f)"
%
dataset
sql_statement
+=
"VALUES (%d, '%s', %f, %f, %f, %f, %f)"
%
(
location_id
,
taxonomy
,
number
,
structural
,
night
,
structural_per_building
,
night_per_building
,
)
self
.
cursor
.
execute
(
sql_statement
)
def
insert_building
(
self
,
dataset
):
def
insert_building
(
self
,
location_id
,
quadkey
,
geom
,
centroid
):
"""
Inserts a dataset
in
the Building
s
table.
Inserts a dataset
to
the Building table.
Args:
dataset (tuple):
(location_index corresponding to Locations.idx,
quadkey of the level 18 tile of the building centroid,
building_geometry,
centroid_geometry)
location_id (int):
ID of the location of SARA data this building belongs to
(corresponds to Location.id)
quadkey (str):
Quadkey of the L18 tile the building centroid is located in
geom (multipolygon):
Building geometry (Spatialite)
centroid (point):
Building centroid geometry
Return:
Index of inserted dataset (Building.id)
"""
sql_statement
=
"INSERT INTO Building "
sql_statement
+=
"(location_id
x
, quadkey, geom, centroid) "
sql_statement
+=
"VALUES (%d, %s, %s, %s)"
%
dataset
sql_statement
+=
"(location_id, quadkey, geom, centroid) "
sql_statement
+=
"VALUES (%d, %s, %s, %s)"
%
(
location_id
,
quadkey
,
geom
,
centroid
)
self
.
cursor
.
execute
(
sql_statement
)
return
self
.
cursor
.
lastrowid
def
insert_
exposure
(
self
,
datase
t
):
def
insert_
building_asset
(
self
,
building_id
,
taxonomy
,
number
,
structural
,
nigh
t
):
"""
Inserts a dataset
in
the Exposure table.
Inserts a dataset
to
the Exposure table.
Args:
dataset (tuple):
(building_index corresponding to Buildings.idx,
taxonomy,
number,
structural,
night)
building_id (int):
ID of the building asset (corresponds to Building.id)
taxonomy (str):
Taxonomy string of the asset
number (float):
Number of buildings (usually a fraction of 1) this asset represents
structural (float):
Structural value of the asset
night (float):
Number of people during night times in this asset
"""
sql_statement
=
"INSERT INTO Exposure (building_idx, taxonomy, number, "
sql_statement
+=
"structural, night) VALUES (%d, '%s', %f, %f, %f)"
%
dataset
sql_statement
=
"INSERT INTO BuildingAsset (building_id, taxonomy, number, "
sql_statement
+=
"structural, night) VALUES (%d, '%s', %f, %f, %f)"
%
(
building_id
,
taxonomy
,
number
,
structural
,
night
,
)
self
.
cursor
.
execute
(
sql_statement
)
def
read_sara_exposure
(
self
,
exposure_filepath
):
"""
Reads a SARA exposure file into the tables Location
s
and Asset
s
.
Reads a SARA exposure file into the tables Location and Asset.
Args:
exposure_filepath (str):
...
...
@@ -277,8 +321,7 @@ class CoquimboDatabase(Database):
# Create new location entry
location_count
+=
1
location
=
"GeomFromText('POINT(%s %s)', 4326)"
%
(
row
[
0
],
row
[
1
])
dataset
=
(
row
[
4
],
location
,
row
[
2
],
row
[
3
])
location_idx
=
self
.
insert_location
(
dataset
)
location_id
=
self
.
insert_location
(
row
[
4
],
location
,
int
(
row
[
2
]),
row
[
3
])
logger
.
debug
(
"Location %d created"
%
location_count
)
last_longitude
=
row
[
0
]
last_latitude
=
row
[
1
]
...
...
@@ -286,16 +329,15 @@ class CoquimboDatabase(Database):
# Add asset to current location
structural_per_building
=
float
(
row
[
7
])
/
float
(
row
[
6
])
night_per_building
=
float
(
row
[
8
])
/
float
(
row
[
6
])
dat
aset
=
(
location_id
x
,
self
.
insert_
as
s
et
(
location_id
,
row
[
5
],
row
[
6
],
row
[
7
],
row
[
8
],
float
(
row
[
6
]
)
,
float
(
row
[
7
]
)
,
float
(
row
[
8
]
)
,
structural_per_building
,
night_per_building
,
)
self
.
insert_asset
(
dataset
)
asset_count
+=
1
logger
.
debug
(
"Asset %d for location %d created"
%
(
asset_count
,
location_count
))
self
.
connection
.
commit
()
...
...
@@ -305,7 +347,7 @@ class CoquimboDatabase(Database):
def
add_voronoi_cells
(
self
):
"""
Creates Voronoi cells around each of the locations (Locations.geom) and
associates them with their respective location via the Voronoi.location_id
x
field.
associates them with their respective location via the Voronoi.location_id field.
"""
# Create the multipolygon of all Voronoi cells
...
...
@@ -325,7 +367,7 @@ class CoquimboDatabase(Database):
logger
.
debug
(
"Voronoi cells added to the Voronoi table."
)
# Associate each Voronoi cell with its respective location
sql_statement
=
"SELECT id
x
, ST_AsText(geom) FROM Location"
sql_statement
=
"SELECT id, ST_AsText(geom) FROM Location"
self
.
cursor
.
execute
(
sql_statement
)
locations
=
self
.
cursor
.
fetchall
()
for
location
in
locations
:
...
...
@@ -337,11 +379,10 @@ class CoquimboDatabase(Database):
self
.
cursor
.
execute
(
sql_statement
)
# Assign the Voronoi cell to its respective location
voronoi_cell
=
self
.
cursor
.
fetchall
()
sql_statement
=
"UPDATE Voronoi SET location_id
x
= %d "
%
index
sql_statement
=
"UPDATE Voronoi SET location_id = %d "
%
index
sql_statement
+=
"WHERE geom = GeomFromText('%s', 4326)"
%
voronoi_cell
[
0
][
0
]
self
.
cursor
.
execute
(
sql_statement
)
logger
.
debug
(
"Voronoi cell assigned to location %d."
%
index
)
self
.
connection
.
commit
()
def
add_buildings
(
self
,
buildings_filepath
):
...
...
@@ -358,13 +399,13 @@ class CoquimboDatabase(Database):
buildings
=
geopandas
.
read_file
(
buildings_filepath
)
for
index
,
building
in
buildings
.
iterrows
():
sql_statement
=
"SELECT location_id
x
, geom FROM Voronoi WHERE "
sql_statement
=
"SELECT location_id, geom FROM Voronoi WHERE "
sql_statement
+=
"within(GeomFromText('%s', 4326), "
%
building
.
geometry
.
centroid
sql_statement
+=
"voronoi.geom)"
self
.
cursor
.
execute
(
sql_statement
)
voronoi
=
self
.
cursor
.
fetchall
()
voronoi
=
self
.
cursor
.
fetchall
()
# TODO: fetchone()
try
:
location_i
ndex
=
voronoi
[
0
][
0
]
location_i
d
=
voronoi
[
0
][
0
]
centroid
=
"POINT(%f %f)"
%
(
building
.
geometry
.
centroid
.
x
,
building
.
geometry
.
centroid
.
y
,
...
...
@@ -375,12 +416,12 @@ class CoquimboDatabase(Database):
building
.
geometry
.
centroid
.
x
+
0.000001
,
building
.
geometry
.
centroid
.
y
+
0.000001
,
)
sql_statement
=
"SELECT location_id
x
, geom FROM Voronoi WHERE "
sql_statement
=
"SELECT location_id, geom FROM Voronoi WHERE "
sql_statement
+=
"within(GeomFromText('%s', 4326), "
%
centroid
sql_statement
+=
"voronoi.geom)"
self
.
cursor
.
execute
(
sql_statement
)
voronoi
=
self
.
cursor
.
fetchall
()
location_i
ndex
=
voronoi
[
0
][
0
]
voronoi
=
self
.
cursor
.
fetchall
()
# TODO: fetchone()
location_i
d
=
voronoi
[
0
][
0
]
quadkey
=
Tile
.
for_latitude_longitude
(
longitude
=
float
(
building
.
geometry
.
centroid
.
x
),
...
...
@@ -389,10 +430,11 @@ class CoquimboDatabase(Database):
).
quad_tree
building_geometry
=
"GeomFromText('%s', 4326)"
%
building
.
geometry
centroid_geometry
=
"GeomFromText('%s', 4326)"
%
centroid
dataset
=
(
location_index
,
quadkey
,
building_geometry
,
centroid_geometry
)
building_idx
=
self
.
insert_building
(
dataset
)
building_id
=
self
.
insert_building
(
location_id
,
quadkey
,
building_geometry
,
centroid_geometry
)
self
.
cursor
.
execute
(
sql_statement
)
logger
.
debug
(
"Building %d added to location %d"
%
(
building_id
x
,
location_i
ndex
))
logger
.
debug
(
"Building %d added to location %d"
%
(
building_id
,
location_i
d
))
self
.
connection
.
commit
()
def
classify_buildings
(
self
):
...
...
@@ -403,14 +445,14 @@ class CoquimboDatabase(Database):
"""
# Loop over all SARA locations (extent defined by Voronoi cells)
sql_statement
=
"SELECT id
x
FROM Location"
sql_statement
=
"SELECT id FROM Location"
self
.
cursor
.
execute
(
sql_statement
)
locations
=
self
.
cursor
.
fetchall
()
for
location
in
locations
:
# Retrieve all assets at the given location
sql_statement
=
"SELECT taxonomy, number, structural, night, "
sql_statement
+=
"structural_per_building, night_per_building FROM Asset "
sql_statement
+=
"WHERE location_id
x
= %d"
%
location
[
0
]
sql_statement
+=
"WHERE location_id = %d"
%
location
[
0
]
self
.
cursor
.
execute
(
sql_statement
)
assets
=
self
.
cursor
.
fetchall
()
...
...
@@ -425,20 +467,19 @@ class CoquimboDatabase(Database):
logger
.
debug
(
"Exposure values computed for location %d"
%
location
[
0
])
# Select all buildings in the Voronoi cell for the current location
sql_statement
=
"SELECT id
x
FROM Building WHERE location_id
x
= %d"
%
location
[
0
]
sql_statement
=
"SELECT id FROM Building WHERE location_id = %d"
%
location
[
0
]
self
.
cursor
.
execute
(
sql_statement
)
building_indices
=
self
.
cursor
.
fetchall
()
for
building_index
in
building_indices
:
# Write exposure data for the building
for
counter
in
range
(
taxonomies
.
shape
[
0
]):
dataset
=
(
self
.
insert_building_asset
(
building_index
[
0
],
taxonomies
[
counter
],
proportions
[
counter
],
exposure_values
[
counter
,
2
]
*
proportions
[
counter
],
exposure_values
[
counter
,
3
]
*
proportions
[
counter
],
)
self
.
insert_exposure
(
dataset
)
logger
.
debug
(
"Exposure values added for building %d at location %d"
%
(
building_index
[
0
],
location
[
0
])
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment