Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Dynamic Exposure
Global Dynamic Exposure
exposure-japan
Commits
12fcff2e
Commit
12fcff2e
authored
Apr 12, 2021
by
Danijel Schorlemmer
Committed by
Danijel Schorlemmer
Aug 09, 2021
Browse files
Implemented the import of dwelling floorspace and sizes
parent
6b1a2a65
Pipeline
#26417
passed with stage
in 4 minutes and 53 seconds
Changes
2
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
exposurejapan/database.py
View file @
12fcff2e
...
...
@@ -74,11 +74,19 @@ class JapanDatabase(SpatialiteDatabase):
construction material and number of stories for each district
HouseholdData : Stores different parameters describing the household numbers,
household members, and household spaces for each district
DwellingFloorspace : Stores the floorspace per dwelling depending on the building,
dwelling, tenure types and construction material for each
district
DwellingDistribution: Stores the number of dwellings of different sizes depending on
the building, dwelling, tenure types and construction material
for each district
BuildingType : Stores the different types of buildings
DwellingType : Stores the different types of dwellings
TenureType : Stores the different types of tenures
ConstructionMaterial: Stores the construction-material types
StoryNumber : Stores the classifications of numbers of stories
StoryNumber : Stores the number (e.g. 1, 2, 3, 4, ...) and classifications
(1-2, 3-5, ...) of numbers of stories.
DwellingSizeType : Stores the different types of dwelling sizes
"""
# Create table District
...
...
@@ -132,35 +140,72 @@ class JapanDatabase(SpatialiteDatabase):
self
.
connection
.
execute
(
sql_statement
)
logger
.
debug
(
"Table HouseholdData created"
)
# Create table DwellingFloorspace
sql_statement
=
"CREATE TABLE DwellingFloorspace ("
sql_statement
+=
"id INTEGER PRIMARY KEY, "
sql_statement
+=
"district_id INTEGER, "
sql_statement
+=
"building_type_id INTEGER, "
sql_statement
+=
"dwelling_type_id INTEGER, "
sql_statement
+=
"tenure_type_id INTEGER, "
sql_statement
+=
"construction_material_id INTEGER, "
sql_statement
+=
"floorspace REAL)"
self
.
connection
.
execute
(
sql_statement
)
logger
.
debug
(
"Table DwellingFloorspace created"
)
# Create table DwellingDistribution
sql_statement
=
"CREATE TABLE DwellingDistribution ("
sql_statement
+=
"id INTEGER PRIMARY KEY, "
sql_statement
+=
"district_id INTEGER, "
sql_statement
+=
"building_type_id INTEGER, "
sql_statement
+=
"dwelling_type_id INTEGER, "
sql_statement
+=
"tenure_type_id INTEGER, "
sql_statement
+=
"construction_material_id INTEGER, "
sql_statement
+=
"dwelling_size_type_id INTEGER, "
sql_statement
+=
"number_dwelling REAL)"
self
.
connection
.
execute
(
sql_statement
)
logger
.
debug
(
"Table DwellingDistribution created"
)
# Create table BuildingType
sql_statement
=
"CREATE TABLE BuildingType ("
sql_statement
+=
"id INTEGER PRIMARY KEY, "
sql_statement
+=
"description TEXT)"
self
.
connection
.
execute
(
sql_statement
)
logger
.
debug
(
"Table BuildingType created"
)
# Create table DwellingType
sql_statement
=
"CREATE TABLE DwellingType ("
sql_statement
+=
"id INTEGER PRIMARY KEY, "
sql_statement
+=
"description TEXT)"
self
.
connection
.
execute
(
sql_statement
)
logger
.
debug
(
"Table DwellingType created"
)
# Create table TenureType
sql_statement
=
"CREATE TABLE TenureType ("
sql_statement
+=
"id INTEGER PRIMARY KEY, "
sql_statement
+=
"description TEXT)"
self
.
connection
.
execute
(
sql_statement
)
logger
.
debug
(
"Table TenureType created"
)
# Create table ConstructionMaterial
sql_statement
=
"CREATE TABLE ConstructionMaterial ("
sql_statement
+=
"id INTEGER PRIMARY KEY, "
sql_statement
+=
"description TEXT)"
self
.
connection
.
execute
(
sql_statement
)
logger
.
debug
(
"Table ConstructionMaterial created"
)
# Create table Number
Stories
# Create table
Story
Number
sql_statement
=
"CREATE TABLE StoryNumber ("
sql_statement
+=
"id INTEGER PRIMARY KEY, "
sql_statement
+=
"description TEXT)"
self
.
connection
.
execute
(
sql_statement
)
logger
.
debug
(
"Table StoryNumber created"
)
# Create table DwellingSizeType
sql_statement
=
"CREATE TABLE DwellingSizeType ("
sql_statement
+=
"id INTEGER PRIMARY KEY, "
sql_statement
+=
"description TEXT)"
self
.
connection
.
execute
(
sql_statement
)
logger
.
debug
(
"Table DwellingSizeType created"
)
def
insert_district
(
self
,
admin_id
,
admin_name
,
geom
):
"""
...
...
@@ -319,6 +364,90 @@ class JapanDatabase(SpatialiteDatabase):
)
self
.
cursor
.
execute
(
sql_statement
)
def
insert_dwelling_floorspace
(
self
,
district_id
,
building_type_id
,
dwelling_type_id
,
tenure_type_id
,
construction_material_id
,
floorspace
,
):
"""
Inserts a full dataset to the DwellingFloorspace table.
Args:
district_id (int):
ID of the district. Corresponds to District.id
building_type_id (int):
ID of the building type. Corresponds to BuildingType.id
dwelling_type_id (int):
ID of the dwelling type. Corresponds to DwellingType.id
tenure_type_id (int):
ID of the tenure type. Corresponds to TenureType.id
construction_material_id (int):
ID of the construction material. Corresponds to ConstructionMaterial.id
floorspace (float):
Size of the floorspace in square meters
"""
sql_statement
=
"INSERT INTO DwellingFloorspace "
sql_statement
+=
"(district_id, building_type_id, dwelling_type_id, tenure_type_id, "
sql_statement
+=
"construction_material_id, floorspace) "
sql_statement
+=
"VALUES (%d, %d, %d, %d, %d, %f)"
%
(
district_id
,
building_type_id
,
dwelling_type_id
,
tenure_type_id
,
construction_material_id
,
floorspace
,
)
self
.
cursor
.
execute
(
sql_statement
)
def
insert_dwelling_distribution
(
self
,
district_id
,
building_type_id
,
dwelling_type_id
,
tenure_type_id
,
construction_material_id
,
dwelling_size_type_id
,
number_dwelling
,
):
"""
Inserts a full dataset to the DwellingDistribution table.
Args:
district_id (int):
ID of the district. Corresponds to District.id
building_type_id (int):
ID of the building type. Corresponds to BuildingType.id
dwelling_type_id (int):
ID of the dwelling type. Corresponds to DwellingType.id
tenure_type_id (int):
ID of the tenure type. Corresponds to TenureType.id
construction_material_id (int):
ID of the construction material. Corresponds to ConstructionMaterial.id
dwelling_size_type_id (int):
ID of the dwelling-size type. Corresponds to DwellingSizeType.id
number_dwelling (float):
Number of dwellings for the combination of the other parameters
"""
sql_statement
=
"INSERT INTO DwellingDistribution "
sql_statement
+=
"(district_id, building_type_id, dwelling_type_id, tenure_type_id, "
sql_statement
+=
"construction_material_id, dwelling_size_type_id, number_dwelling) "
sql_statement
+=
"VALUES (%d, %d, %d, %d, %d, %d, %f)"
%
(
district_id
,
building_type_id
,
dwelling_type_id
,
tenure_type_id
,
construction_material_id
,
dwelling_size_type_id
,
number_dwelling
,
)
self
.
cursor
.
execute
(
sql_statement
)
def
insert_building_type
(
self
,
building_type_id
,
description
):
"""
Inserts a building-type description to the BuildingType table.
...
...
@@ -399,9 +528,25 @@ class JapanDatabase(SpatialiteDatabase):
sql_statement
+=
"VALUES (%d, '%s')"
%
(
story_number_id
,
description
)
self
.
cursor
.
execute
(
sql_statement
)
def
insert_dwelling_size_type
(
self
,
dwelling_size_type_id
,
description
):
"""
Inserts a Dwelling-size description to the DwellingSizeType table.
Args:
dwelling_size_type_id (int):
ID of the dwelling-size type entry
description (str):
Description of the dwelling-size type entry
"""
sql_statement
=
"INSERT INTO DwellingSizeType "
sql_statement
+=
"(id, description) "
sql_statement
+=
"VALUES (%d, '%s')"
%
(
dwelling_size_type_id
,
description
)
self
.
cursor
.
execute
(
sql_statement
)
def
read_districts_and_boundaries
(
self
,
district_boundary_filepath
):
"""
Imports all districts and boundaries from a prepared geopackage file
Imports all districts and boundaries from a prepared geopackage file
.
('estat_bound_municipal`). The function opens this file as Spatialite database
and copies all necessary district and boundary information into the `District` table.
...
...
@@ -416,7 +561,7 @@ class JapanDatabase(SpatialiteDatabase):
sql_statement
=
"SELECT key_code_ward, "
sql_statement
+=
"CITY_NAME, "
sql_statement
+=
"AsWKT(CastAutomagic(geom)) "
sql_statement
+=
"FROM
estat_bound_municipal
"
sql_statement
+=
"FROM
Boundary
"
boundary_db
.
cursor
.
execute
(
sql_statement
)
for
row
in
boundary_db
.
cursor
:
self
.
insert_district
(
...
...
@@ -429,7 +574,11 @@ class JapanDatabase(SpatialiteDatabase):
logger
.
info
(
"Districts and boundaries added"
)
def
import_exposure_data
(
self
,
dwelling_numbers_filepath
,
building_numbers_filepath
,
household_numbers_filepath
self
,
dwelling_numbers_filepath
,
building_numbers_filepath
,
household_numbers_filepath
,
dwelling_sizes_filepath
,
):
"""
Imports all exposure data from the Excel files provided by E-Stat, Japan. The following
...
...
@@ -443,6 +592,8 @@ class JapanDatabase(SpatialiteDatabase):
File path to the file of number of buildings
household_numbers_filepath (str):
File path to the file of numbers of households
dwelling_sizes_filepath (str):
File path to the file of dwelling sizes
"""
# Tables for building classifications
...
...
@@ -457,6 +608,15 @@ class JapanDatabase(SpatialiteDatabase):
"102_Wooden and fire-proofed"
,
"2_Non-wooden"
,
]
dwelling_size_type_list
=
[
"0_Total"
,
"1_29m2 and under"
,
"2_30 to 49m2"
,
"3_50 to 69m2"
,
"4_70 to 99m2"
,
"5_100 to 149m2"
,
"6_150m2 and over"
,
]
# Read numbers of dwellings
dwelling_numbers_input
=
pandas
.
read_excel
(
...
...
@@ -614,6 +774,83 @@ class JapanDatabase(SpatialiteDatabase):
)
logger
.
info
(
"Number of households added"
)
# Read numbers of dwellings by size
dwelling_sizes_input
=
pandas
.
read_excel
(
dwelling_sizes_filepath
,
header
=
10
,
usecols
=
[
4
,
5
,
7
,
9
,
11
,
13
,
14
,
15
,
16
,
17
,
18
,
19
,
20
,
21
],
)
for
index
,
row
in
dwelling_sizes_input
.
iterrows
():
admin_id
=
int
((
row
[
"Area classification"
].
split
(
"_"
))[
0
])
# Identify district_id based on admin_id from the District table
sql_statement
=
"SELECT id FROM District "
sql_statement
+=
"WHERE admin_id = %d"
%
admin_id
self
.
cursor
.
execute
(
sql_statement
)
result
=
self
.
cursor
.
fetchone
()
if
result
is
None
:
# Only data for which a district exist matter
continue
district_id
=
result
[
0
]
# Get ID of building type, dwelling type, and tenure type
building_type_id
=
add_element_and_get_index
(
row
[
"Type of building"
],
building_type_list
)
dwelling_type_id
=
add_element_and_get_index
(
row
[
"Type of dwelling"
],
dwelling_type_list
)
tenure_type_id
=
add_element_and_get_index
(
row
[
"Tenure of dwelling"
],
tenure_type_list
)
construction_material_id
=
add_element_and_get_index
(
row
[
"Construction material"
],
construction_material_list
)
# Prepare and insert the data to the DwellingFloorspace table
floorspace
=
float
(
str
(
row
[
13
]).
replace
(
"-"
,
"0"
))
self
.
insert_dwelling_floorspace
(
district_id
,
building_type_id
,
dwelling_type_id
,
tenure_type_id
,
construction_material_id
,
floorspace
,
)
logger
.
debug
(
"Floorspace for district %s, type %s, %s, %s, %s added"
%
(
admin_id
,
building_type_list
[
building_type_id
],
dwelling_type_list
[
dwelling_type_id
],
tenure_type_list
[
tenure_type_id
],
construction_material_list
[
construction_material_id
],
)
)
# Prepare and insert the data to the DwellingDistribution table
for
dwelling_size_type_id
in
range
(
7
):
number_dwelling
=
float
(
str
(
row
[
6
+
dwelling_size_type_id
]).
replace
(
"-"
,
"0"
))
self
.
insert_dwelling_distribution
(
district_id
,
building_type_id
,
dwelling_type_id
,
tenure_type_id
,
construction_material_id
,
dwelling_size_type_id
,
number_dwelling
,
)
logger
.
debug
(
"Dwelling numbers by size for district %s, type %s, %s, %s, %s added"
%
(
admin_id
,
building_type_list
[
building_type_id
],
dwelling_type_list
[
dwelling_type_id
],
tenure_type_list
[
tenure_type_id
],
construction_material_list
[
construction_material_id
],
)
)
logger
.
info
(
"Number and sizes of dwellings added"
)
# Add the building types to the database
for
building_type_id
,
building_type
in
enumerate
(
building_type_list
):
self
.
insert_building_type
(
building_type_id
,
building_type
)
...
...
@@ -639,6 +876,11 @@ class JapanDatabase(SpatialiteDatabase):
construction_material_list
):
self
.
insert_construction_material
(
construction_material_id
,
construction_material
)
logger
.
info
(
"construction-material types added"
)
logger
.
info
(
"Construction-material types added"
)
# Add the types of dwelling sizes to the database
for
dwelling_size_type_id
,
dwelling_size_type
in
enumerate
(
dwelling_size_type_list
):
self
.
insert_dwelling_size_type
(
dwelling_size_type_id
,
dwelling_size_type
)
logger
.
info
(
"Dwelling-size types added"
)
self
.
connection
.
commit
()
exposurejapan/exposurejapan.py
View file @
12fcff2e
...
...
@@ -42,8 +42,10 @@ def main():
logger
.
warning
(
"Spatialite extension cannot be loaded. Exiting ..."
)
exit
()
db
.
create_tables
()
db
.
read_districts_and_boundaries
(
"data/estat_bound_municipal.gpkg"
)
db
.
import_exposure_data
(
"data/e008_3e.xlsx"
,
"data/e039_3e.xlsx"
,
"data/e011_2e.xlsx"
)
db
.
read_districts_and_boundaries
(
"data/Boundary.gpkg"
)
db
.
import_exposure_data
(
"data/e008_3e.xlsx"
,
"data/e039_3e.xlsx"
,
"data/e011_2e.xlsx"
,
"data/e014e.xlsx"
)
# Leave the program
sys
.
exit
()
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment