Commit 45515949 authored by Danijel Schorlemmer's avatar Danijel Schorlemmer
Browse files

Changed functions for inserting datasets to tables and redesigned tables

parent 4ec7c2d5
Pipeline #20789 passed with stage
in 1 minute and 36 seconds
...@@ -109,27 +109,27 @@ class CoquimboDatabase(Database): ...@@ -109,27 +109,27 @@ class CoquimboDatabase(Database):
def create_tables(self): def create_tables(self):
""" """
Creates all necessary tables in the database. These are: Creates all necessary tables in the database. These are:
Location : Stores the locations of the SARA exposure data Location: Stores the locations of the SARA exposure data
Asset : Stores the assets per location Asset : Stores the assets per location
Voronoi : Stores the Voronoi cells around each location Voronoi : Stores the Voronoi cells around each location
Buildings: Stores the OpenStreetMap buildings Building: Stores the OpenStreetMap buildings
Exposure : Stores the resulting exposure assets for each building Exposure: Stores the resulting exposure assets for each building
""" """
# Create table Locations # Create table Location
sql_statement = "CREATE TABLE Location (" sql_statement = "CREATE TABLE Location ("
sql_statement += "idx INTEGER PRIMARY KEY AUTOINCREMENT, " sql_statement += "id INTEGER PRIMARY KEY AUTOINCREMENT, "
sql_statement += "occupancy TEXT, " sql_statement += "occupancy TEXT, "
sql_statement += "id INTEGER, " sql_statement += "admin_id INTEGER, "
sql_statement += "name TEXT)" sql_statement += "admin_name TEXT)"
self.connection.execute(sql_statement) self.connection.execute(sql_statement)
sql_statement = "SELECT AddGeometryColumn('Location', 'geom', 4326, 'POINT', 'XY')" sql_statement = "SELECT AddGeometryColumn('Location', 'geom', 4326, 'POINT', 'XY')"
self.connection.execute(sql_statement) self.connection.execute(sql_statement)
logger.debug("Table Locations created") logger.debug("Table Location created")
# Create table Asset # Create table Asset
sql_statement = "CREATE TABLE Asset (" sql_statement = "CREATE TABLE Asset ("
sql_statement += "location_idx INTEGER, " sql_statement += "location_id INTEGER, "
sql_statement += "taxonomy TEXT, " sql_statement += "taxonomy TEXT, "
sql_statement += "number REAL, " sql_statement += "number REAL, "
sql_statement += "structural REAL, " sql_statement += "structural REAL, "
...@@ -137,119 +137,163 @@ class CoquimboDatabase(Database): ...@@ -137,119 +137,163 @@ class CoquimboDatabase(Database):
sql_statement += "structural_per_building REAL, " sql_statement += "structural_per_building REAL, "
sql_statement += "night_per_building REAL)" sql_statement += "night_per_building REAL)"
self.connection.execute(sql_statement) self.connection.execute(sql_statement)
logger.debug("Table Assets created") logger.debug("Table Asset created")
# Create table Voronoi # Create table Voronoi
sql_statement = "CREATE TABLE Voronoi (" sql_statement = "CREATE TABLE Voronoi ("
sql_statement += "location_idx INTEGER)" sql_statement += "location_id INTEGER)"
self.connection.execute(sql_statement) self.connection.execute(sql_statement)
sql_statement = "SELECT AddGeometryColumn('Voronoi', 'geom', 4326, 'POLYGON', 'XY')" sql_statement = "SELECT AddGeometryColumn('Voronoi', 'geom', 4326, 'POLYGON', 'XY')"
self.connection.execute(sql_statement) self.connection.execute(sql_statement)
logger.debug("Table Voronoi created") logger.debug("Table Voronoi created")
# Create table Buildings # Create table Building
sql_statement = "CREATE TABLE Building (" sql_statement = "CREATE TABLE Building ("
sql_statement += "idx INTEGER PRIMARY KEY AUTOINCREMENT, " sql_statement += "id INTEGER PRIMARY KEY AUTOINCREMENT, "
sql_statement += "location_idx INTEGER, " sql_statement += "location_id INTEGER, "
sql_statement += "quadkey TEXT)" sql_statement += "quadkey TEXT)"
self.connection.execute(sql_statement) self.connection.execute(sql_statement)
sql_statement = "SELECT AddGeometryColumn('Building', " sql_statement = "SELECT AddGeometryColumn('Building', "
sql_statement += "'geom', 4326, 'MULTIPOLYGON', 'XY')" sql_statement += "'geom', 4326, 'MULTIPOLYGON', 'XY')"
self.connection.execute(sql_statement) self.connection.execute(sql_statement)
sql_statement = "SELECT AddGeometryColumn('Building', 'centroid', 4326, 'POINT', 'XY')" sql_statement = "SELECT AddGeometryColumn('Building', 'centroid', 4326, 'POINT', 'XY')"
self.connection.execute(sql_statement) self.connection.execute(sql_statement)
logger.debug("Table Buildings created") logger.debug("Table Building created")
# Create table Exposure # Create table Exposure
sql_statement = "CREATE TABLE Exposure (" sql_statement = "CREATE TABLE BuildingAsset ("
sql_statement += "idx INTEGER PRIMARY KEY AUTOINCREMENT, " sql_statement += "building_id INTEGER, "
sql_statement += "building_idx INTEGER, " sql_statement += "taxonomy TEXT, "
sql_statement += "taxonomy TEXT, " sql_statement += "number REAL, "
sql_statement += "number REAL, " sql_statement += "structural REAL, "
sql_statement += "structural REAL, " sql_statement += "night REAL)"
sql_statement += "night REAL)"
self.connection.execute(sql_statement) self.connection.execute(sql_statement)
logger.debug("Table Exposure created") logger.debug("Table BuildingAsset created")
def insert_location(self, dataset): def insert_location(self, occupancy, geom, admin_id, admin_name):
""" """
Inserts a dataset in the Locations table. Inserts a dataset to the Location table.
Args: Args:
dataset (tuple): occupancy (str):
(occupancy, Occupancy type of the SARA data for this location
point geometry in WKT, geom (point):
location id, Geometry (Spatialite) of the location (POINT)
location name) admin_id (int):
ID of administrative unit of the location
admin_name (str):
Name of administrative unit of the location
Return: Return:
Index of inserted dataset (Locations.idx) Index of inserted dataset (Location.id)
""" """
sql_statement = "INSERT INTO Location " sql_statement = "INSERT INTO Location "
sql_statement += "(occupancy, geom, id, name) " sql_statement += "(occupancy, geom, admin_id, admin_name) "
sql_statement += "VALUES ('%s', %s, %s, '%s')" % dataset sql_statement += "VALUES ('%s', %s, %d, '%s')" % (occupancy, geom, admin_id, admin_name)
self.cursor.execute(sql_statement) self.cursor.execute(sql_statement)
return self.cursor.lastrowid return self.cursor.lastrowid
def insert_asset(self, dataset): def insert_asset(
self,
location_id,
taxonomy,
number,
structural,
night,
structural_per_building,
night_per_building,
):
""" """
Inserts a dataset in the Assets table. Inserts a dataset to the Asset table.
Args: Args:
dataset (tuple): location_id (int):
(location index corresponding to Locations.idx, ID of the location of SARA data (corresponds to Location.id)
taxonomy, taxonomy (str):
number of buildings, Taxonomy string of the asset
structural value of buildings (total value), number (float):
number of people inside buildings during night (total value)) Number of buildings represented by the asset
Return: structural (float):
Index of inserted dataset (Buildings.idx) Total structural value of the asset
night (float):
Total number of people in the buildings of the asset during night
structural_per_building (float):
Structural value per building of the asset
night_per_building (float):
Number of people per building of the asset during night
""" """
sql_statement = "INSERT INTO Asset " sql_statement = "INSERT INTO Asset "
sql_statement += "(location_idx, taxonomy, number, structural, night, " sql_statement += "(location_id, taxonomy, number, structural, night, "
sql_statement += "structural_per_building, night_per_building) " sql_statement += "structural_per_building, night_per_building) "
sql_statement += "VALUES (%s, '%s', %s, %s, %s, %f, %f)" % dataset sql_statement += "VALUES (%d, '%s', %f, %f, %f, %f, %f)" % (
location_id,
taxonomy,
number,
structural,
night,
structural_per_building,
night_per_building,
)
self.cursor.execute(sql_statement) self.cursor.execute(sql_statement)
def insert_building(self, dataset): def insert_building(self, location_id, quadkey, geom, centroid):
""" """
Inserts a dataset in the Buildings table. Inserts a dataset to the Building table.
Args: Args:
dataset (tuple): location_id (int):
(location_index corresponding to Locations.idx, ID of the location of SARA data this building belongs to
quadkey of the level 18 tile of the building centroid, (corresponds to Location.id)
building_geometry, quadkey (str):
centroid_geometry) Quadkey of the L18 tile the building centroid is located in
geom (multipolygon):
Building geometry (Spatialite)
centroid (point):
Building centroid geometry
Return:
Index of inserted dataset (Building.id)
""" """
sql_statement = "INSERT INTO Building " sql_statement = "INSERT INTO Building "
sql_statement += "(location_idx, quadkey, geom, centroid) " sql_statement += "(location_id, quadkey, geom, centroid) "
sql_statement += "VALUES (%d, %s, %s, %s)" % dataset sql_statement += "VALUES (%d, %s, %s, %s)" % (location_id, quadkey, geom, centroid)
self.cursor.execute(sql_statement) self.cursor.execute(sql_statement)
return self.cursor.lastrowid return self.cursor.lastrowid
def insert_exposure(self, dataset): def insert_building_asset(self, building_id, taxonomy, number, structural, night):
""" """
Inserts a dataset in the Exposure table. Inserts a dataset to the Exposure table.
Args: Args:
dataset (tuple): building_id (int):
(building_index corresponding to Buildings.idx, ID of the building asset (corresponds to Building.id)
taxonomy, taxonomy (str):
number, Taxonomy string of the asset
structural, number (float):
night) Number of buildings (usually a fraction of 1) this asset represents
structural (float):
Structural value of the asset
night (float):
Number of people during night times in this asset
""" """
sql_statement = "INSERT INTO Exposure (building_idx, taxonomy, number, " sql_statement = "INSERT INTO BuildingAsset (building_id, taxonomy, number, "
sql_statement += "structural, night) VALUES (%d, '%s', %f, %f, %f)" % dataset sql_statement += "structural, night) VALUES (%d, '%s', %f, %f, %f)" % (
building_id,
taxonomy,
number,
structural,
night,
)
self.cursor.execute(sql_statement) self.cursor.execute(sql_statement)
def read_sara_exposure(self, exposure_filepath): def read_sara_exposure(self, exposure_filepath):
""" """
Reads a SARA exposure file into the tables Locations and Assets. Reads a SARA exposure file into the tables Location and Asset.
Args: Args:
exposure_filepath (str): exposure_filepath (str):
...@@ -277,8 +321,7 @@ class CoquimboDatabase(Database): ...@@ -277,8 +321,7 @@ class CoquimboDatabase(Database):
# Create new location entry # Create new location entry
location_count += 1 location_count += 1
location = "GeomFromText('POINT(%s %s)', 4326)" % (row[0], row[1]) location = "GeomFromText('POINT(%s %s)', 4326)" % (row[0], row[1])
dataset = (row[4], location, row[2], row[3]) location_id = self.insert_location(row[4], location, int(row[2]), row[3])
location_idx = self.insert_location(dataset)
logger.debug("Location %d created" % location_count) logger.debug("Location %d created" % location_count)
last_longitude = row[0] last_longitude = row[0]
last_latitude = row[1] last_latitude = row[1]
...@@ -286,16 +329,15 @@ class CoquimboDatabase(Database): ...@@ -286,16 +329,15 @@ class CoquimboDatabase(Database):
# Add asset to current location # Add asset to current location
structural_per_building = float(row[7]) / float(row[6]) structural_per_building = float(row[7]) / float(row[6])
night_per_building = float(row[8]) / float(row[6]) night_per_building = float(row[8]) / float(row[6])
dataset = ( self.insert_asset(
location_idx, location_id,
row[5], row[5],
row[6], float(row[6]),
row[7], float(row[7]),
row[8], float(row[8]),
structural_per_building, structural_per_building,
night_per_building, night_per_building,
) )
self.insert_asset(dataset)
asset_count += 1 asset_count += 1
logger.debug("Asset %d for location %d created" % (asset_count, location_count)) logger.debug("Asset %d for location %d created" % (asset_count, location_count))
self.connection.commit() self.connection.commit()
...@@ -305,7 +347,7 @@ class CoquimboDatabase(Database): ...@@ -305,7 +347,7 @@ class CoquimboDatabase(Database):
def add_voronoi_cells(self): def add_voronoi_cells(self):
""" """
Creates Voronoi cells around each of the locations (Locations.geom) and Creates Voronoi cells around each of the locations (Locations.geom) and
associates them with their respective location via the Voronoi.location_idx field. associates them with their respective location via the Voronoi.location_id field.
""" """
# Create the multipolygon of all Voronoi cells # Create the multipolygon of all Voronoi cells
...@@ -325,7 +367,7 @@ class CoquimboDatabase(Database): ...@@ -325,7 +367,7 @@ class CoquimboDatabase(Database):
logger.debug("Voronoi cells added to the Voronoi table.") logger.debug("Voronoi cells added to the Voronoi table.")
# Associate each Voronoi cell with its respective location # Associate each Voronoi cell with its respective location
sql_statement = "SELECT idx, ST_AsText(geom) FROM Location" sql_statement = "SELECT id, ST_AsText(geom) FROM Location"
self.cursor.execute(sql_statement) self.cursor.execute(sql_statement)
locations = self.cursor.fetchall() locations = self.cursor.fetchall()
for location in locations: for location in locations:
...@@ -337,11 +379,10 @@ class CoquimboDatabase(Database): ...@@ -337,11 +379,10 @@ class CoquimboDatabase(Database):
self.cursor.execute(sql_statement) self.cursor.execute(sql_statement)
# Assign the Voronoi cell to its respective location # Assign the Voronoi cell to its respective location
voronoi_cell = self.cursor.fetchall() voronoi_cell = self.cursor.fetchall()
sql_statement = "UPDATE Voronoi SET location_idx = %d " % index sql_statement = "UPDATE Voronoi SET location_id = %d " % index
sql_statement += "WHERE geom = GeomFromText('%s', 4326)" % voronoi_cell[0][0] sql_statement += "WHERE geom = GeomFromText('%s', 4326)" % voronoi_cell[0][0]
self.cursor.execute(sql_statement) self.cursor.execute(sql_statement)
logger.debug("Voronoi cell assigned to location %d." % index) logger.debug("Voronoi cell assigned to location %d." % index)
self.connection.commit() self.connection.commit()
def add_buildings(self, buildings_filepath): def add_buildings(self, buildings_filepath):
...@@ -358,13 +399,13 @@ class CoquimboDatabase(Database): ...@@ -358,13 +399,13 @@ class CoquimboDatabase(Database):
buildings = geopandas.read_file(buildings_filepath) buildings = geopandas.read_file(buildings_filepath)
for index, building in buildings.iterrows(): for index, building in buildings.iterrows():
sql_statement = "SELECT location_idx, geom FROM Voronoi WHERE " sql_statement = "SELECT location_id, geom FROM Voronoi WHERE "
sql_statement += "within(GeomFromText('%s', 4326), " % building.geometry.centroid sql_statement += "within(GeomFromText('%s', 4326), " % building.geometry.centroid
sql_statement += "voronoi.geom)" sql_statement += "voronoi.geom)"
self.cursor.execute(sql_statement) self.cursor.execute(sql_statement)
voronoi = self.cursor.fetchall() voronoi = self.cursor.fetchall() # TODO: fetchone()
try: try:
location_index = voronoi[0][0] location_id = voronoi[0][0]
centroid = "POINT(%f %f)" % ( centroid = "POINT(%f %f)" % (
building.geometry.centroid.x, building.geometry.centroid.x,
building.geometry.centroid.y, building.geometry.centroid.y,
...@@ -375,12 +416,12 @@ class CoquimboDatabase(Database): ...@@ -375,12 +416,12 @@ class CoquimboDatabase(Database):
building.geometry.centroid.x + 0.000001, building.geometry.centroid.x + 0.000001,
building.geometry.centroid.y + 0.000001, building.geometry.centroid.y + 0.000001,
) )
sql_statement = "SELECT location_idx, geom FROM Voronoi WHERE " sql_statement = "SELECT location_id, geom FROM Voronoi WHERE "
sql_statement += "within(GeomFromText('%s', 4326), " % centroid sql_statement += "within(GeomFromText('%s', 4326), " % centroid
sql_statement += "voronoi.geom)" sql_statement += "voronoi.geom)"
self.cursor.execute(sql_statement) self.cursor.execute(sql_statement)
voronoi = self.cursor.fetchall() voronoi = self.cursor.fetchall() # TODO: fetchone()
location_index = voronoi[0][0] location_id = voronoi[0][0]
quadkey = Tile.for_latitude_longitude( quadkey = Tile.for_latitude_longitude(
longitude=float(building.geometry.centroid.x), longitude=float(building.geometry.centroid.x),
...@@ -389,10 +430,11 @@ class CoquimboDatabase(Database): ...@@ -389,10 +430,11 @@ class CoquimboDatabase(Database):
).quad_tree ).quad_tree
building_geometry = "GeomFromText('%s', 4326)" % building.geometry building_geometry = "GeomFromText('%s', 4326)" % building.geometry
centroid_geometry = "GeomFromText('%s', 4326)" % centroid centroid_geometry = "GeomFromText('%s', 4326)" % centroid
dataset = (location_index, quadkey, building_geometry, centroid_geometry) building_id = self.insert_building(
building_idx = self.insert_building(dataset) location_id, quadkey, building_geometry, centroid_geometry
)
self.cursor.execute(sql_statement) self.cursor.execute(sql_statement)
logger.debug("Building %d added to location %d" % (building_idx, location_index)) logger.debug("Building %d added to location %d" % (building_id, location_id))
self.connection.commit() self.connection.commit()
def classify_buildings(self): def classify_buildings(self):
...@@ -403,14 +445,14 @@ class CoquimboDatabase(Database): ...@@ -403,14 +445,14 @@ class CoquimboDatabase(Database):
""" """
# Loop over all SARA locations (extent defined by Voronoi cells) # Loop over all SARA locations (extent defined by Voronoi cells)
sql_statement = "SELECT idx FROM Location" sql_statement = "SELECT id FROM Location"
self.cursor.execute(sql_statement) self.cursor.execute(sql_statement)
locations = self.cursor.fetchall() locations = self.cursor.fetchall()
for location in locations: for location in locations:
# Retrieve all assets at the given location # Retrieve all assets at the given location
sql_statement = "SELECT taxonomy, number, structural, night, " sql_statement = "SELECT taxonomy, number, structural, night, "
sql_statement += "structural_per_building, night_per_building FROM Asset " sql_statement += "structural_per_building, night_per_building FROM Asset "
sql_statement += "WHERE location_idx = %d" % location[0] sql_statement += "WHERE location_id = %d" % location[0]
self.cursor.execute(sql_statement) self.cursor.execute(sql_statement)
assets = self.cursor.fetchall() assets = self.cursor.fetchall()
...@@ -425,20 +467,19 @@ class CoquimboDatabase(Database): ...@@ -425,20 +467,19 @@ class CoquimboDatabase(Database):
logger.debug("Exposure values computed for location %d" % location[0]) logger.debug("Exposure values computed for location %d" % location[0])
# Select all buildings in the Voronoi cell for the current location # Select all buildings in the Voronoi cell for the current location
sql_statement = "SELECT idx FROM Building WHERE location_idx = %d" % location[0] sql_statement = "SELECT id FROM Building WHERE location_id = %d" % location[0]
self.cursor.execute(sql_statement) self.cursor.execute(sql_statement)
building_indices = self.cursor.fetchall() building_indices = self.cursor.fetchall()
for building_index in building_indices: for building_index in building_indices:
# Write exposure data for the building # Write exposure data for the building
for counter in range(taxonomies.shape[0]): for counter in range(taxonomies.shape[0]):
dataset = ( self.insert_building_asset(
building_index[0], building_index[0],
taxonomies[counter], taxonomies[counter],
proportions[counter], proportions[counter],
exposure_values[counter, 2] * proportions[counter], exposure_values[counter, 2] * proportions[counter],
exposure_values[counter, 3] * proportions[counter], exposure_values[counter, 3] * proportions[counter],
) )
self.insert_exposure(dataset)
logger.debug( logger.debug(
"Exposure values added for building %d at location %d" "Exposure values added for building %d at location %d"
% (building_index[0], location[0]) % (building_index[0], location[0])
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment