Commit 8b42395e authored by Cecilia Nievas's avatar Cecilia Nievas
Browse files

Added feature to calculate and store buildings per data-unit tile

parent 0fc9502d
Pipeline #36180 passed with stage
in 2 minutes and 12 seconds
......@@ -119,14 +119,19 @@ class DataUnit:
Built-up area of the data-unit tile, in square metres.
fraction_data_unit_area (float):
Fraction (0.0, 1.0] that the surface area of the data-unit tile
represents with respect to the surface area of 'in_geometry' (i.e. the
represents with respect to the surface area of self.geometry (i.e. the
surface area of the summation of all defined data-unit tiles). It cannot
be zero.
fraction_data_unit_built_up_area (float):
Fraction [0.0, 1.0] that the built-up area of the data-unit tile
represents with respect to the built-up area contained by 'in_geometry'
represents with respect to the built-up area contained by self.geometry
(i.e. the summation of built-up areas of all defined data-unit tiles).
It can be zero.
aggregated_buildings (float):
Number of buildings in the data-unit tile, stemming from distributing
self.total_buildings onto the data-unit tile proportionally to
'fraction_data_unit_built_up_area', if available, or
'fraction_data_unit_area' otherwise.
"""
def __init__(
......
......@@ -228,8 +228,8 @@ class ExposureEntity:
)
data_units_ids = list(self.occupancy_cases[occupancy_case]["data_units"].keys())
data_units_geoms = [
dataunit.geometry
data_units_attrs = [
(dataunit.geometry, dataunit.total_buildings)
for dataunit in self.occupancy_cases[occupancy_case]["data_units"].values()
]
......@@ -240,7 +240,7 @@ class ExposureEntity:
db_built_up_config,
db_built_up_table,
)
all_data_unit_tiles = p.map(func, data_units_geoms)
all_data_unit_tiles = p.map(func, data_units_attrs)
p.close()
p.join()
......
......@@ -101,10 +101,20 @@ class DataUnitTilesHelper:
return data_unit_tiles, filtered_quadtiles
@staticmethod
def define_data_unit_tiles_and_attributes(db_built_up_config, db_table, in_geometry):
"""This function defines the data-unit tiles associated with 'in_geometry' and their
def define_data_unit_tiles_and_attributes(
db_built_up_config, db_table, data_unit_attributes
):
"""This function defines the data-unit tiles associated with a data unit with geometry
and total number of buildings as indicated by 'data_unit_attributes', as well as their
respective attributes. Data-unit tiles are defined as the intersection between zoom
level 18 quadtiles and 'in_geometry'.
level 18 quadtiles and the geometry of a data-unit.
The total number of buildings indicated by 'data_unit_attributes' is distributed across
all data-unit tiles proportionally to 'fraction_data_unit_built_up_area', if available,
or 'fraction_data_unit_area' otherwise. 'fraction_data_unit_built_up_area' is
unavailable when the sum of the built-up area within the data unit is zero. Both
'fraction_data_unit_built_up_area' and 'fraction_data_unit_area' are calculated by this
function.
Args:
db_built_up_config (dict):
......@@ -131,8 +141,12 @@ class DataUnitTilesHelper:
Value of the built-up area to be retrieved.
db_source_id (int):
ID of the source used to define the built-up area.
in_geometry (Shapely Polygon or MultiPolygon):
Geometry for which the associated data-unit tiles will be defined.
data_unit_attributes (tuple of (Shapely Polygon or MultiPolygon, float)):
Attributes of the data unit these data-unit tiles belong to. The first element
is the geometry for which the associated data-unit tiles will be defined
(Shapely Polygon or MultiPolygon). The second element is the total number of
buildings in the data unit (float). These attributes are passed as tuples to
allow for this method to be used together with multiprocessing.Pool.map.
Returns:
data_unit_tiles (GeoPandas GeoDataFrame without geometry):
......@@ -145,26 +159,34 @@ class DataUnitTilesHelper:
Built-up area of the data-unit tile, in square metres.
fraction_data_unit_area (float):
Fraction (0.0, 1.0] that the surface area of the data-unit tile
represents with respect to the surface area of 'in_geometry' (i.e. the
surface area of the summation of all defined data-unit tiles). It cannot
be zero.
represents with respect to the surface area of the geometry input
through 'data_unit_attributes' (i.e. the surface area of the summation
of all defined data-unit tiles). It cannot be zero.
fraction_data_unit_built_up_area (float):
Fraction [0.0, 1.0] that the built-up area of the data-unit tile
represents with respect to the built-up area contained by 'in_geometry'
(i.e. the summation of built-up areas of all defined data-unit tiles).
It can be zero.
represents with respect to the built-up area contained by the geometry
input through 'data_unit_attributes' (i.e. the summation of built-up
areas of all defined data-unit tiles). It can be zero.
aggregated_buildings (float):
Number of buildings in the data-unit tile, stemming from distributing
the total number of buildings in the data units onto the data-unit
tiles.
"""
# Split contents of data_unit_attributes
data_unit_geometry = data_unit_attributes[0]
data_unit_buildings = data_unit_attributes[1]
# Get data-unit tiles geometries, quadkeys and full associated quadtiles
data_unit_tiles, filtered_quadtiles = DataUnitTilesHelper.get_data_unit_tiles(
in_geometry
data_unit_geometry
)
# Calculate data-unit tile areas and fractions of tiles (initialise)
all_areas = numpy.zeros([data_unit_tiles.shape[0]])
all_fractions = numpy.zeros([data_unit_tiles.shape[0]])
# Tiles that are fully contained in the original geometry 'in_geometry'
# Tiles that are fully contained in the original geometry 'data_unit_geometry'
all_areas[data_unit_tiles.contained] = DataUnitTilesHelper.get_area_on_sphere(
data_unit_tiles[data_unit_tiles.contained]["lon_w"].values,
data_unit_tiles[data_unit_tiles.contained]["lat_s"].values,
......@@ -175,7 +197,7 @@ class DataUnitTilesHelper:
[len(all_areas[data_unit_tiles.contained])]
)
# Tiles that are partially contained in the original geometry 'in_geometry'
# Tiles that are partially contained in the original geometry 'data_unit_geometry'
results = DataUnitTilesHelper.get_areas_and_ratios_arbitrary_shapes(
data_unit_tiles[numpy.logical_not(data_unit_tiles.contained)]["geometry"].values,
filtered_quadtiles[numpy.logical_not(data_unit_tiles.contained)]["geometry"].values,
......@@ -200,11 +222,16 @@ class DataUnitTilesHelper:
data_unit_tiles["size_data_unit_tile_built_up_area"].values
/ data_unit_tiles["size_data_unit_tile_built_up_area"].values.sum()
)
data_unit_tiles["aggregated_buildings"] = (
data_unit_buildings * data_unit_tiles["fraction_data_unit_built_up_area"]
)
else:
data_unit_tiles["fraction_data_unit_built_up_area"] = numpy.nan * numpy.ones_like(
all_areas
)
# Writing of NaNs to the database and posterior decision-making handled separately
data_unit_tiles["aggregated_buildings"] = (
data_unit_buildings * data_unit_tiles["fraction_data_unit_area"]
)
# Discard unnecessary columns of 'data_unit_tiles' (free up memory)
data_unit_tiles = data_unit_tiles.drop(
......@@ -613,6 +640,8 @@ class DataUnitTilesHelper:
Fraction [0.0, 1.0] that the built-up area of the data-unit tile
represents with respect to the built-up area contained in the data unit.
It can be zero.
aggregated_buildings (float):
Number of buildings in the data-unit tile.
aggregated_source_id (int):
ID of the source of the aggregated exposure model.
occupancy_case (str):
......@@ -641,8 +670,8 @@ class DataUnitTilesHelper:
sql_commands["update"] = "UPDATE %s"
sql_commands["update"] += " SET (size_data_unit_tile_area,"
sql_commands["update"] += " size_data_unit_tile_built_up_area, fraction_data_unit_area,"
sql_commands["update"] += " fraction_data_unit_built_up_area) ="
sql_commands["update"] += " ('%s','%s','%s','%s')"
sql_commands["update"] += " fraction_data_unit_built_up_area, aggregated_buildings) ="
sql_commands["update"] += " ('%s','%s','%s','%s','%s')"
sql_commands["update"] += " WHERE (quadkey='%s' AND aggregated_source_id='%s'"
sql_commands["update"] += " AND occupancy_case='%s' AND data_unit_id='%s');"
......@@ -650,8 +679,8 @@ class DataUnitTilesHelper:
sql_commands["insert"] += " %s(quadkey, aggregated_source_id, occupancy_case,"
sql_commands["insert"] += " exposure_entity, data_unit_id, size_data_unit_tile_area,"
sql_commands["insert"] += " size_data_unit_tile_built_up_area, fraction_data_unit_area,"
sql_commands["insert"] += " fraction_data_unit_built_up_area)"
sql_commands["insert"] += " VALUES('%s','%s','%s','%s','%s','%s','%s','%s','%s');"
sql_commands["insert"] += " fraction_data_unit_built_up_area, aggregated_buildings)"
sql_commands["insert"] += " VALUES('%s','%s','%s','%s','%s','%s','%s','%s','%s','%s');"
db_gde_tiles = Database(**db_data_unit_tiles_config)
db_gde_tiles.create_connection_and_cursor()
......@@ -680,6 +709,7 @@ class DataUnitTilesHelper:
],
data_unit.data_unit_tiles["fraction_data_unit_area"].values[i],
data_unit.data_unit_tiles["fraction_data_unit_built_up_area"].values[i],
data_unit.data_unit_tiles["aggregated_buildings"].values[i],
data_unit.data_unit_tiles["quadkey"].values[i],
str(aggregated_source_id),
occupancy_case,
......@@ -702,6 +732,7 @@ class DataUnitTilesHelper:
],
data_unit.data_unit_tiles["fraction_data_unit_area"].values[i],
data_unit.data_unit_tiles["fraction_data_unit_built_up_area"].values[i],
data_unit.data_unit_tiles["aggregated_buildings"].values[i],
)
)
else: # More than one entries found, this is an error
......
quadkey,type,lon_w,lat_s,lon_e,lat_n,contained,size_data_unit_tile_area,fraction_data_unit_area,size_data_unit_tile_built_up_area,fraction_data_unit_built_up_area
120222331000133202,MultiPolygon,4.9994,42.0026,5.000153,42.0032,False,2784.421,0.068256627491064,393.720042735683,0.091
120222331000133203,MultiPolygon,5.000153,42.0026,5.001526,42.0032,False,5154.434,0.126354556823582,1355.42193743533,0.082
120222331000133212,Polygon,5.001526,42.002366,5.002899,42.0032,False,7544.606,0.184946659039292,4331.24437901172,0.237
120222331000133213,Polygon,5.002899,42.002366,5.0037,42.0032,False,6144.446,0.150623473160473,2828.98618958169,0.141
120222331000133220,Polygon,4.9994,42.0015,5.000153,42.0018,False,2077.655,0.050931135553836,333.852842868529,0.022
120222331000133221,Polygon,5.000153,42.0015,5.001526,42.0018,False,3791.219,0.092937031799445,360.852144117784,0.067
120222331000133230,Polygon,5.001526,42.0015,5.002899,42.002366,False,6913.142,0.169467102240224,1566.1305332532,0.083
120222331000133231,Polygon,5.002899,42.0015,5.0037,42.002366,False,6383.493,0.156483413892085,2101.1230827478,0.037
quadkey,type,lon_w,lat_s,lon_e,lat_n,contained,size_data_unit_tile_area,fraction_data_unit_area,size_data_unit_tile_built_up_area,fraction_data_unit_built_up_area,aggregated_buildings
120222331000133202,MultiPolygon,4.9994,42.0026,5.000153,42.0032,False,2784.421,0.068256627491064,393.720042735683,0.02967,44.50043913
120222331000133203,MultiPolygon,5.000153,42.0026,5.001526,42.0032,False,5154.434,0.126354556823582,1355.42193743533,0.10213,153.1973608
120222331000133212,Polygon,5.001526,42.002366,5.002899,42.0032,False,7544.606,0.184946659039292,4331.24437901172,0.32636,489.5414404
120222331000133213,Polygon,5.002899,42.002366,5.0037,42.0032,False,6144.446,0.150623473160473,2828.98618958169,0.21317,319.747826
120222331000133220,Polygon,4.9994,42.0015,5.000153,42.0018,False,2077.655,0.050931135553836,333.852842868529,0.02516,37.7339137
120222331000133221,Polygon,5.000153,42.0015,5.001526,42.0018,False,3791.219,0.092937031799445,360.852144117784,0.02719,40.78552558
120222331000133230,Polygon,5.001526,42.0015,5.002899,42.002366,False,6913.142,0.169467102240224,1566.1305332532,0.11801,177.0128236
120222331000133231,Polygon,5.002899,42.0015,5.0037,42.002366,False,6383.493,0.156483413892085,2101.1230827478,0.15832,237.4806708
......@@ -55,6 +55,7 @@ CREATE TABLE data_unit_tiles
size_data_unit_tile_built_up_area FLOAT,
fraction_data_unit_area FLOAT,
fraction_data_unit_built_up_area FLOAT,
aggregated_buildings FLOAT,
PRIMARY KEY (quadkey, aggregated_source_id, occupancy_case, data_unit_id)
);
......@@ -66,7 +67,8 @@ INSERT INTO data_unit_tiles(quadkey,
size_data_unit_tile_area,
size_data_unit_tile_built_up_area,
fraction_data_unit_area,
fraction_data_unit_built_up_area)
fraction_data_unit_built_up_area,
aggregated_buildings)
VALUES (
'120222331000133202',
2,
......@@ -76,7 +78,8 @@ VALUES (
2532.671,
287.720,
0.0572566,
0.086
0.086,
42.7
);
CREATE TABLE data_units
......
......@@ -281,7 +281,7 @@ def test_define_data_unit_tiles_and_attributes(test_db):
# Run the function to test
returned_data_unit_tiles = DataUnitTilesHelper.define_data_unit_tiles_and_attributes(
db_built_up_config, "obm_built_area_assessments", in_polygon
db_built_up_config, "obm_built_area_assessments", (in_polygon, 1500.0)
)
assert returned_data_unit_tiles.shape[0] == expected_results.shape[0]
......@@ -306,6 +306,14 @@ def test_define_data_unit_tiles_and_attributes(test_db):
0,
)
assert round(
returned_data_unit_tiles["fraction_data_unit_built_up_area"].values[i], 5
) == round(expected_results["fraction_data_unit_built_up_area"].values[which[0]], 5)
assert round(returned_data_unit_tiles["aggregated_buildings"].values[i], 2) == round(
expected_results["aggregated_buildings"].values[which[0]], 2
)
def test_determine_if_areas_are_the_same():
# Read quadtiles from file
......@@ -452,6 +460,7 @@ def test_write_data_unit_tiles_to_database(test_db):
"fraction_data_unit_area",
"size_data_unit_tile_built_up_area",
"fraction_data_unit_built_up_area",
"aggregated_buildings",
],
dtype={
"quadkey": str,
......@@ -459,6 +468,7 @@ def test_write_data_unit_tiles_to_database(test_db):
"fraction_data_unit_area": float,
"size_data_unit_tile_built_up_area": float,
"fraction_data_unit_built_up_area": float,
"aggregated_buildings": float,
},
)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment