Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Dynamic Exposure
Global Dynamic Exposure
exposure-japan
Commits
6b1a2a65
Commit
6b1a2a65
authored
Apr 09, 2021
by
Danijel Schorlemmer
Browse files
Implemented the import of the numbers of households and related parameters
parent
1b383f67
Pipeline
#21762
passed with stage
in 1 minute and 30 seconds
Changes
2
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
exposurejapan/database.py
View file @
6b1a2a65
...
...
@@ -72,7 +72,11 @@ class JapanDatabase(SpatialiteDatabase):
construction material and number of stories for each district
BuildingNumber : Stores the number of buildings depending on building types,
construction material and number of stories for each district
HouseholdData : Stores different parameters describing the household numbers,
household members, and household spaces for each district
BuildingType : Stores the different types of buildings
DwellingType : Stores the different types of dwellings
TenureType : Stores the different types of tenures
ConstructionMaterial: Stores the construction-material types
StoryNumber : Stores the classifications of numbers of stories
"""
...
...
@@ -110,12 +114,42 @@ class JapanDatabase(SpatialiteDatabase):
self
.
connection
.
execute
(
sql_statement
)
logger
.
debug
(
"Table BuildingNumber created"
)
# Create table HouseholdData
sql_statement
=
"CREATE TABLE HouseholdData ("
sql_statement
+=
"id INTEGER PRIMARY KEY AUTOINCREMENT, "
sql_statement
+=
"district_id INTEGER, "
sql_statement
+=
"building_type_id INTEGER, "
sql_statement
+=
"dwelling_type_id INTEGER, "
sql_statement
+=
"tenure_type_id INTEGER, "
sql_statement
+=
"number_dwelling REAL, "
sql_statement
+=
"number_household REAL, "
sql_statement
+=
"number_household_member REAL, "
sql_statement
+=
"rooms_per_dwelling REAL, "
sql_statement
+=
"tatami_per_dwelling REAL, "
sql_statement
+=
"floorspace_per_dwelling REAL, "
sql_statement
+=
"tatami_per_person REAL, "
sql_statement
+=
"person_per_room REAL)"
self
.
connection
.
execute
(
sql_statement
)
logger
.
debug
(
"Table HouseholdData created"
)
# Create table BuildingType
sql_statement
=
"CREATE TABLE BuildingType ("
sql_statement
+=
"id INTEGER PRIMARY KEY, "
sql_statement
+=
"description TEXT)"
self
.
connection
.
execute
(
sql_statement
)
# Create table DwellingType
sql_statement
=
"CREATE TABLE DwellingType ("
sql_statement
+=
"id INTEGER PRIMARY KEY, "
sql_statement
+=
"description TEXT)"
self
.
connection
.
execute
(
sql_statement
)
# Create table TenureType
sql_statement
=
"CREATE TABLE TenureType ("
sql_statement
+=
"id INTEGER PRIMARY KEY, "
sql_statement
+=
"description TEXT)"
self
.
connection
.
execute
(
sql_statement
)
# Create table ConstructionMaterial
sql_statement
=
"CREATE TABLE ConstructionMaterial ("
sql_statement
+=
"id INTEGER PRIMARY KEY, "
...
...
@@ -218,6 +252,73 @@ class JapanDatabase(SpatialiteDatabase):
)
self
.
cursor
.
execute
(
sql_statement
)
def
insert_household_data
(
self
,
district_id
,
building_type_id
,
dwelling_type_id
,
tenure_type_id
,
number_dwelling
,
number_household
,
number_household_member
,
rooms_per_dwelling
,
tatami_per_dwelling
,
floorspace_per_dwelling
,
tatami_per_person
,
person_per_room
,
):
"""
Inserts a full dataset to the HouseholdData table.
Args:
district_id (int):
ID of the district. Corresponds to District.id
building_type_id (int):
ID of the building type. Corresponds to BuildingType.id
dwelling_type_id (int):
ID of the dwelling type. Corresponds to DwellingType.id
tenure_type_id (int):
ID of the tenure type. Corresponds to TenureType.id
number_dwelling (float):
Number of dwellings for the combination of the other parameters
number_household (float):
Number of households for the combination of the other parameters
number_household_member (float):
Number of household members for the combination of the other parameters
rooms_per_dwelling (float):
Number of rooms per dwelling for the combination of the other parameters
tatami_per_dwelling (float):
Number of tatami mats per dwelling for the combination of the other parameters
floorspace_per_dwelling (float):
Size of floorspace (in square meters) per dwelling for the combination of the
other parameters
tatami_per_person (float):
Number of tatami mats per person for the combination of the other parameters
person_per_room (float):
Number of persons per room for the combination of the other parameters
"""
sql_statement
=
"INSERT INTO HouseholdData "
sql_statement
+=
"(district_id, building_type_id, dwelling_type_id, tenure_type_id, "
sql_statement
+=
"number_dwelling, number_household, number_household_member, "
sql_statement
+=
"rooms_per_dwelling, tatami_per_dwelling, floorspace_per_dwelling, "
sql_statement
+=
"tatami_per_person, person_per_room) "
sql_statement
+=
"VALUES (%d, %d, %d, %d, %f, %f, %f, %f, %f, %f, %f, %f)"
%
(
district_id
,
building_type_id
,
dwelling_type_id
,
tenure_type_id
,
number_dwelling
,
number_household
,
number_household_member
,
rooms_per_dwelling
,
tatami_per_dwelling
,
floorspace_per_dwelling
,
tatami_per_person
,
person_per_room
,
)
self
.
cursor
.
execute
(
sql_statement
)
def
insert_building_type
(
self
,
building_type_id
,
description
):
"""
Inserts a building-type description to the BuildingType table.
...
...
@@ -234,6 +335,38 @@ class JapanDatabase(SpatialiteDatabase):
sql_statement
+=
"VALUES (%d, '%s')"
%
(
building_type_id
,
description
)
self
.
cursor
.
execute
(
sql_statement
)
def
insert_dwelling_type
(
self
,
dwelling_type_id
,
description
):
"""
Inserts a dwelling-type description to the DwellingType table.
Args:
dwelling_type_id (int):
ID of the dwelling type
description (str):
Description of the dwelling type
"""
sql_statement
=
"INSERT INTO DwellingType "
sql_statement
+=
"(id, description) "
sql_statement
+=
"VALUES (%d, '%s')"
%
(
dwelling_type_id
,
description
)
self
.
cursor
.
execute
(
sql_statement
)
def
insert_tenure_type
(
self
,
tenure_type_id
,
description
):
"""
Inserts a tenure-type description to the TenureType table.
Args:
tenure_type_id (int):
ID of the tenure type
description (str):
Description of the tenure type
"""
sql_statement
=
"INSERT INTO TenureType "
sql_statement
+=
"(id, description) "
sql_statement
+=
"VALUES (%d, '%s')"
%
(
tenure_type_id
,
description
)
self
.
cursor
.
execute
(
sql_statement
)
def
insert_construction_material
(
self
,
construction_material_id
,
description
):
"""
Inserts a construction-material description to the ConstructionMaterial table.
...
...
@@ -295,7 +428,9 @@ class JapanDatabase(SpatialiteDatabase):
self
.
connection
.
commit
()
logger
.
info
(
"Districts and boundaries added"
)
def
import_exposure_data
(
self
,
dwelling_numbers_filepath
,
building_numbers_filepath
):
def
import_exposure_data
(
self
,
dwelling_numbers_filepath
,
building_numbers_filepath
,
household_numbers_filepath
):
"""
Imports all exposure data from the Excel files provided by E-Stat, Japan. The following
files are needed:
...
...
@@ -306,11 +441,15 @@ class JapanDatabase(SpatialiteDatabase):
File path to the file of number of dwellings
building_numbers_filepath (str):
File path to the file of number of buildings
household_numbers_filepath (str):
File path to the file of numbers of households
"""
# Tables for building classifications
building_type_list
=
[]
story_number_list
=
[]
dwelling_type_list
=
[]
tenure_type_list
=
[]
construction_material_list
=
[
"0_Total"
,
"1_Wooden"
,
...
...
@@ -410,19 +549,96 @@ class JapanDatabase(SpatialiteDatabase):
)
logger
.
info
(
"Number of buildings added"
)
# Read numbers of households
household_numbers_input
=
pandas
.
read_excel
(
household_numbers_filepath
,
header
=
7
,
usecols
=
[
4
,
5
,
7
,
9
,
11
,
12
,
13
,
14
,
15
,
16
,
17
,
18
,
19
],
)
for
index
,
row
in
household_numbers_input
.
iterrows
():
admin_id
=
int
((
row
[
"Area classification"
].
split
(
"_"
))[
0
])
# Identify district_id based on admin_id from the District table
sql_statement
=
"SELECT id FROM District "
sql_statement
+=
"WHERE admin_id = %d"
%
admin_id
self
.
cursor
.
execute
(
sql_statement
)
result
=
self
.
cursor
.
fetchone
()
if
result
is
None
:
# Only data for which a district exist matter
continue
district_id
=
result
[
0
]
# Get ID of building type, dwelling type, and tenure type
building_type_id
=
add_element_and_get_index
(
row
[
"Type of building"
],
building_type_list
)
dwelling_type_id
=
add_element_and_get_index
(
row
[
"Type of dwelling"
],
dwelling_type_list
)
tenure_type_id
=
add_element_and_get_index
(
row
[
"Tenure of dwelling"
],
tenure_type_list
)
# Read the relevant values from the input data
number_dwelling
=
float
(
str
(
row
[
5
]).
replace
(
"-"
,
"0"
))
number_household
=
float
(
str
(
row
[
6
]).
replace
(
"-"
,
"0"
))
number_household_member
=
float
(
str
(
row
[
7
]).
replace
(
"-"
,
"0"
))
rooms_per_dwelling
=
float
(
str
(
row
[
8
]).
replace
(
"-"
,
"0"
))
tatami_per_dwelling
=
float
(
str
(
row
[
9
]).
replace
(
"-"
,
"0"
))
floorspace_per_dwelling
=
float
(
str
(
row
[
10
]).
replace
(
"-"
,
"0"
))
tatami_per_person
=
float
(
str
(
row
[
11
]).
replace
(
"-"
,
"0"
))
person_per_room
=
float
(
str
(
row
[
12
]).
replace
(
"-"
,
"0"
))
# Insert household numbers and related values
self
.
insert_household_data
(
district_id
,
building_type_id
,
dwelling_type_id
,
tenure_type_id
,
number_dwelling
,
number_household
,
number_household_member
,
rooms_per_dwelling
,
tatami_per_dwelling
,
floorspace_per_dwelling
,
tatami_per_person
,
person_per_room
,
)
logger
.
debug
(
"Household number for district %s, type %s, %s, %s added"
%
(
admin_id
,
building_type_list
[
building_type_id
],
dwelling_type_list
[
dwelling_type_id
],
tenure_type_list
[
tenure_type_id
],
)
)
logger
.
info
(
"Number of households added"
)
# Add the building types to the database
for
building_type
in
building_type_list
:
building_type_id
=
building_type_list
.
index
(
building_type
)
for
building_type_id
,
building_type
in
enumerate
(
building_type_list
):
self
.
insert_building_type
(
building_type_id
,
building_type
)
logger
.
info
(
"Building types added"
)
# Add the dwelling types to the database
for
dwelling_type_id
,
dwelling_type
in
enumerate
(
dwelling_type_list
):
self
.
insert_dwelling_type
(
dwelling_type_id
,
dwelling_type
)
logger
.
info
(
"Dwelling types added"
)
# Add the tenure types to the database
for
tenure_type_id
,
tenure_type
in
enumerate
(
tenure_type_list
):
self
.
insert_tenure_type
(
tenure_type_id
,
tenure_type
)
logger
.
info
(
"Tenure types added"
)
# Add the number of stories to the database
for
story_number
in
story_number_list
:
story_number_id
=
story_number_list
.
index
(
story_number
)
for
story_number_id
,
story_number
in
enumerate
(
story_number_list
):
self
.
insert_story_number
(
story_number_id
,
story_number
)
logger
.
info
(
"Number-of-story types added"
)
# Add the number of stories to the database
for
construction_material
in
construction_material_list
:
construction_material_id
=
construction_material_list
.
index
(
construction_material
)
# Add the types of construction material to the database
for
construction_material_id
,
construction_material
in
enumerate
(
construction_material_list
):
self
.
insert_construction_material
(
construction_material_id
,
construction_material
)
logger
.
info
(
"construction-material types added"
)
self
.
connection
.
commit
()
exposurejapan/exposurejapan.py
View file @
6b1a2a65
...
...
@@ -43,7 +43,7 @@ def main():
exit
()
db
.
create_tables
()
db
.
read_districts_and_boundaries
(
"data/estat_bound_municipal.gpkg"
)
db
.
import_exposure_data
(
"data/e008_3e.xlsx"
,
"data/e039_3e.xlsx"
)
db
.
import_exposure_data
(
"data/e008_3e.xlsx"
,
"data/e039_3e.xlsx"
,
"data/e011_2e.xlsx"
)
# Leave the program
sys
.
exit
()
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment