Skip to content
GitLab
Commit 45515949 authored by Danijel Schorlemmer's avatar Danijel Schorlemmer
Browse files

Changed functions for inserting datasets to tables and redesigned tables

parent 4ec7c2d5
Pipeline #20789 passed with stage
in 1 minute and 36 seconds
Hide whitespace changes
Inline Side-by-side
exposurecoquimbo/database.py
View file @ 45515949
......@@ -109,27 +109,27 @@ class CoquimboDatabase(Database):
def create_tables(self):
"""
Creates all necessary tables in the database. These are:
Location : Stores the locations of the SARA exposure data
Asset : Stores the assets per location
Voronoi : Stores the Voronoi cells around each location
Buildings: Stores the OpenStreetMap buildings
Exposure : Stores the resulting exposure assets for each building
Location: Stores the locations of the SARA exposure data
Asset : Stores the assets per location
Voronoi : Stores the Voronoi cells around each location
Building: Stores the OpenStreetMap buildings
Exposure: Stores the resulting exposure assets for each building
"""
# Create table Locations
# Create table Location
sql_statement = "CREATE TABLE Location ("
sql_statement += "idx INTEGER PRIMARY KEY AUTOINCREMENT, "
sql_statement += "occupancy TEXT, "
sql_statement += "id INTEGER, "
sql_statement += "name TEXT)"
sql_statement += "id INTEGER PRIMARY KEY AUTOINCREMENT, "
sql_statement += "occupancy TEXT, "
sql_statement += "admin_id INTEGER, "
sql_statement += "admin_name TEXT)"
self.connection.execute(sql_statement)
sql_statement = "SELECT AddGeometryColumn('Location', 'geom', 4326, 'POINT', 'XY')"
self.connection.execute(sql_statement)
logger.debug("Table Locations created")
logger.debug("Table Location created")
# Create table Asset
sql_statement = "CREATE TABLE Asset ("
sql_statement += "location_idx INTEGER, "
sql_statement += "location_id INTEGER, "
sql_statement += "taxonomy TEXT, "
sql_statement += "number REAL, "
sql_statement += "structural REAL, "
......@@ -137,119 +137,163 @@ class CoquimboDatabase(Database):
sql_statement += "structural_per_building REAL, "
sql_statement += "night_per_building REAL)"
self.connection.execute(sql_statement)
logger.debug("Table Assets created")
logger.debug("Table Asset created")
# Create table Voronoi
sql_statement = "CREATE TABLE Voronoi ("
sql_statement += "location_idx INTEGER)"
sql_statement += "location_id INTEGER)"
self.connection.execute(sql_statement)
sql_statement = "SELECT AddGeometryColumn('Voronoi', 'geom', 4326, 'POLYGON', 'XY')"
self.connection.execute(sql_statement)
logger.debug("Table Voronoi created")
# Create table Buildings
# Create table Building
sql_statement = "CREATE TABLE Building ("
sql_statement += "idx INTEGER PRIMARY KEY AUTOINCREMENT, "
sql_statement += "location_idx INTEGER, "
sql_statement += "quadkey TEXT)"
sql_statement += "id INTEGER PRIMARY KEY AUTOINCREMENT, "
sql_statement += "location_id INTEGER, "
sql_statement += "quadkey TEXT)"
self.connection.execute(sql_statement)
sql_statement = "SELECT AddGeometryColumn('Building', "
sql_statement += "'geom', 4326, 'MULTIPOLYGON', 'XY')"
self.connection.execute(sql_statement)
sql_statement = "SELECT AddGeometryColumn('Building', 'centroid', 4326, 'POINT', 'XY')"
self.connection.execute(sql_statement)
logger.debug("Table Buildings created")
logger.debug("Table Building created")
# Create table Exposure
sql_statement = "CREATE TABLE Exposure ("
sql_statement += "idx INTEGER PRIMARY KEY AUTOINCREMENT, "
sql_statement += "building_idx INTEGER, "
sql_statement += "taxonomy TEXT, "
sql_statement += "number REAL, "
sql_statement += "structural REAL, "
sql_statement += "night REAL)"
sql_statement = "CREATE TABLE BuildingAsset ("
sql_statement += "building_id INTEGER, "
sql_statement += "taxonomy TEXT, "
sql_statement += "number REAL, "
sql_statement += "structural REAL, "
sql_statement += "night REAL)"
self.connection.execute(sql_statement)
logger.debug("Table Exposure created")
logger.debug("Table BuildingAsset created")
def insert_location(self, dataset):
def insert_location(self, occupancy, geom, admin_id, admin_name):
"""
Inserts a dataset in the Locations table.
Inserts a dataset to the Location table.
Args:
dataset (tuple):
(occupancy,
point geometry in WKT,
location id,
location name)
occupancy (str):
Occupancy type of the SARA data for this location
geom (point):
Geometry (Spatialite) of the location (POINT)
admin_id (int):
ID of administrative unit of the location
admin_name (str):
Name of administrative unit of the location
Return:
Index of inserted dataset (Locations.idx)
Index of inserted dataset (Location.id)
"""
sql_statement = "INSERT INTO Location "
sql_statement += "(occupancy, geom, id, name) "
sql_statement += "VALUES ('%s', %s, %s, '%s')" % dataset
sql_statement += "(occupancy, geom, admin_id, admin_name) "
sql_statement += "VALUES ('%s', %s, %d, '%s')" % (occupancy, geom, admin_id, admin_name)
self.cursor.execute(sql_statement)
return self.cursor.lastrowid
def insert_asset(self, dataset):
def insert_asset(
self,
location_id,
taxonomy,
number,
structural,
night,
structural_per_building,
night_per_building,
):
"""
Inserts a dataset in the Assets table.
Inserts a dataset to the Asset table.
Args:
dataset (tuple):
(location index corresponding to Locations.idx,
taxonomy,
number of buildings,
structural value of buildings (total value),
number of people inside buildings during night (total value))
Return:
Index of inserted dataset (Buildings.idx)
location_id (int):
ID of the location of SARA data (corresponds to Location.id)
taxonomy (str):
Taxonomy string of the asset
number (float):
Number of buildings represented by the asset
structural (float):
Total structural value of the asset
night (float):
Total number of people in the buildings of the asset during night
structural_per_building (float):
Structural value per building of the asset
night_per_building (float):
Number of people per building of the asset during night
"""
sql_statement = "INSERT INTO Asset "
sql_statement += "(location_idx, taxonomy, number, structural, night, "
sql_statement += "(location_id, taxonomy, number, structural, night, "
sql_statement += "structural_per_building, night_per_building) "
sql_statement += "VALUES (%s, '%s', %s, %s, %s, %f, %f)" % dataset
sql_statement += "VALUES (%d, '%s', %f, %f, %f, %f, %f)" % (
location_id,
taxonomy,
number,
structural,
night,
structural_per_building,
night_per_building,
)
self.cursor.execute(sql_statement)
def insert_building(self, dataset):
def insert_building(self, location_id, quadkey, geom, centroid):
"""
Inserts a dataset in the Buildings table.
Inserts a dataset to the Building table.
Args:
dataset (tuple):
(location_index corresponding to Locations.idx,
quadkey of the level 18 tile of the building centroid,
building_geometry,
centroid_geometry)
location_id (int):
ID of the location of SARA data this building belongs to
(corresponds to Location.id)
quadkey (str):
Quadkey of the L18 tile the building centroid is located in
geom (multipolygon):
Building geometry (Spatialite)
centroid (point):
Building centroid geometry
Return:
Index of inserted dataset (Building.id)
"""
sql_statement = "INSERT INTO Building "
sql_statement += "(location_idx, quadkey, geom, centroid) "
sql_statement += "VALUES (%d, %s, %s, %s)" % dataset
sql_statement += "(location_id, quadkey, geom, centroid) "
sql_statement += "VALUES (%d, %s, %s, %s)" % (location_id, quadkey, geom, centroid)
self.cursor.execute(sql_statement)
return self.cursor.lastrowid
def insert_exposure(self, dataset):
def insert_building_asset(self, building_id, taxonomy, number, structural, night):
"""
Inserts a dataset in the Exposure table.
Inserts a dataset to the Exposure table.
Args:
dataset (tuple):
(building_index corresponding to Buildings.idx,
taxonomy,
number,
structural,
night)
building_id (int):
ID of the building asset (corresponds to Building.id)
taxonomy (str):
Taxonomy string of the asset
number (float):
Number of buildings (usually a fraction of 1) this asset represents
structural (float):
Structural value of the asset
night (float):
Number of people during night times in this asset
"""
sql_statement = "INSERT INTO Exposure (building_idx, taxonomy, number, "
sql_statement += "structural, night) VALUES (%d, '%s', %f, %f, %f)" % dataset
sql_statement = "INSERT INTO BuildingAsset (building_id, taxonomy, number, "
sql_statement += "structural, night) VALUES (%d, '%s', %f, %f, %f)" % (
building_id,
taxonomy,
number,
structural,
night,
)
self.cursor.execute(sql_statement)
def read_sara_exposure(self, exposure_filepath):
"""
Reads a SARA exposure file into the tables Locations and Assets.
Reads a SARA exposure file into the tables Location and Asset.
Args:
exposure_filepath (str):
......@@ -277,8 +321,7 @@ class CoquimboDatabase(Database):
# Create new location entry
location_count += 1
location = "GeomFromText('POINT(%s %s)', 4326)" % (row[0], row[1])
dataset = (row[4], location, row[2], row[3])
location_idx = self.insert_location(dataset)
location_id = self.insert_location(row[4], location, int(row[2]), row[3])
logger.debug("Location %d created" % location_count)
last_longitude = row[0]
last_latitude = row[1]
......@@ -286,16 +329,15 @@ class CoquimboDatabase(Database):
# Add asset to current location
structural_per_building = float(row[7]) / float(row[6])
night_per_building = float(row[8]) / float(row[6])
dataset = (
location_idx,
self.insert_asset(
location_id,
row[5],
row[6],
row[7],
row[8],
float(row[6]),
float(row[7]),
float(row[8]),
structural_per_building,
night_per_building,
)
self.insert_asset(dataset)
asset_count += 1
logger.debug("Asset %d for location %d created" % (asset_count, location_count))
self.connection.commit()
......@@ -305,7 +347,7 @@ class CoquimboDatabase(Database):
def add_voronoi_cells(self):
"""
Creates Voronoi cells around each of the locations (Locations.geom) and
associates them with their respective location via the Voronoi.location_idx field.
associates them with their respective location via the Voronoi.location_id field.
"""
# Create the multipolygon of all Voronoi cells
......@@ -325,7 +367,7 @@ class CoquimboDatabase(Database):
logger.debug("Voronoi cells added to the Voronoi table.")
# Associate each Voronoi cell with its respective location
sql_statement = "SELECT idx, ST_AsText(geom) FROM Location"
sql_statement = "SELECT id, ST_AsText(geom) FROM Location"
self.cursor.execute(sql_statement)
locations = self.cursor.fetchall()
for location in locations:
......@@ -337,11 +379,10 @@ class CoquimboDatabase(Database):
self.cursor.execute(sql_statement)
# Assign the Voronoi cell to its respective location
voronoi_cell = self.cursor.fetchall()
sql_statement = "UPDATE Voronoi SET location_idx = %d " % index
sql_statement = "UPDATE Voronoi SET location_id = %d " % index
sql_statement += "WHERE geom = GeomFromText('%s', 4326)" % voronoi_cell[0][0]
self.cursor.execute(sql_statement)
logger.debug("Voronoi cell assigned to location %d." % index)
self.connection.commit()
def add_buildings(self, buildings_filepath):
......@@ -358,13 +399,13 @@ class CoquimboDatabase(Database):
buildings = geopandas.read_file(buildings_filepath)
for index, building in buildings.iterrows():
sql_statement = "SELECT location_idx, geom FROM Voronoi WHERE "
sql_statement = "SELECT location_id, geom FROM Voronoi WHERE "
sql_statement += "within(GeomFromText('%s', 4326), " % building.geometry.centroid
sql_statement += "voronoi.geom)"
self.cursor.execute(sql_statement)
voronoi = self.cursor.fetchall()
voronoi = self.cursor.fetchall() # TODO: fetchone()
try:
location_index = voronoi[0][0]
location_id = voronoi[0][0]
centroid = "POINT(%f %f)" % (
building.geometry.centroid.x,
building.geometry.centroid.y,
......@@ -375,12 +416,12 @@ class CoquimboDatabase(Database):
building.geometry.centroid.x + 0.000001,
building.geometry.centroid.y + 0.000001,
)
sql_statement = "SELECT location_idx, geom FROM Voronoi WHERE "
sql_statement = "SELECT location_id, geom FROM Voronoi WHERE "
sql_statement += "within(GeomFromText('%s', 4326), " % centroid
sql_statement += "voronoi.geom)"
self.cursor.execute(sql_statement)
voronoi = self.cursor.fetchall()
location_index = voronoi[0][0]
voronoi = self.cursor.fetchall() # TODO: fetchone()
location_id = voronoi[0][0]
quadkey = Tile.for_latitude_longitude(
longitude=float(building.geometry.centroid.x),
......@@ -389,10 +430,11 @@ class CoquimboDatabase(Database):
).quad_tree
building_geometry = "GeomFromText('%s', 4326)" % building.geometry
centroid_geometry = "GeomFromText('%s', 4326)" % centroid
dataset = (location_index, quadkey, building_geometry, centroid_geometry)
building_idx = self.insert_building(dataset)
building_id = self.insert_building(
location_id, quadkey, building_geometry, centroid_geometry
)
self.cursor.execute(sql_statement)
logger.debug("Building %d added to location %d" % (building_idx, location_index))
logger.debug("Building %d added to location %d" % (building_id, location_id))
self.connection.commit()
def classify_buildings(self):
......@@ -403,14 +445,14 @@ class CoquimboDatabase(Database):
"""
# Loop over all SARA locations (extent defined by Voronoi cells)
sql_statement = "SELECT idx FROM Location"
sql_statement = "SELECT id FROM Location"
self.cursor.execute(sql_statement)
locations = self.cursor.fetchall()
for location in locations:
# Retrieve all assets at the given location
sql_statement = "SELECT taxonomy, number, structural, night, "
sql_statement += "structural_per_building, night_per_building FROM Asset "
sql_statement += "WHERE location_idx = %d" % location[0]
sql_statement += "WHERE location_id = %d" % location[0]
self.cursor.execute(sql_statement)
assets = self.cursor.fetchall()
......@@ -425,20 +467,19 @@ class CoquimboDatabase(Database):
logger.debug("Exposure values computed for location %d" % location[0])
# Select all buildings in the Voronoi cell for the current location
sql_statement = "SELECT idx FROM Building WHERE location_idx = %d" % location[0]
sql_statement = "SELECT id FROM Building WHERE location_id = %d" % location[0]
self.cursor.execute(sql_statement)
building_indices = self.cursor.fetchall()
for building_index in building_indices:
# Write exposure data for the building
for counter in range(taxonomies.shape[0]):
dataset = (
self.insert_building_asset(
building_index[0],
taxonomies[counter],
proportions[counter],
exposure_values[counter, 2] * proportions[counter],
exposure_values[counter, 3] * proportions[counter],
)
self.insert_exposure(dataset)
logger.debug(
"Exposure values added for building %d at location %d"
% (building_index[0], location[0])
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment