Commit 0c59e097 authored by shinde's avatar shinde
Browse files

Population table added

parent 12fcff2e
Pipeline #27877 passed with stage
in 4 minutes and 55 seconds
......@@ -165,6 +165,17 @@ class JapanDatabase(SpatialiteDatabase):
self.connection.execute(sql_statement)
logger.debug("Table DwellingDistribution created")
# Create table PopulationDistribution
sql_statement = "CREATE TABLE PopulationDistribution ("
sql_statement += "id INTEGER PRIMARY KEY, "
sql_statement += "district_id INTEGER, "
sql_statement += "total INTEGER, "
sql_statement += "male INTEGER, "
sql_statement += "female INTEGER, "
sql_statement += "number_household INTEGER)"
self.connection.execute(sql_statement)
logger.debug("Table PopulationDistribution created")
# Create table BuildingType
sql_statement = "CREATE TABLE BuildingType ("
sql_statement += "id INTEGER PRIMARY KEY, "
......@@ -448,6 +459,37 @@ class JapanDatabase(SpatialiteDatabase):
)
self.cursor.execute(sql_statement)
def insert_population_distribution(
self, district_id, total, male, female, number_household
):
"""
Inserts a full dataset to the PopulationDistribution table.
Args:
district_id (int):
ID of the district. Corresponds to District.id
total (int):
Total population of the district
male (int):
Male population of the district
female (int):
Female population of the district
number_household (int):
Total number of households of the district
"""
sql_statement = "INSERT INTO PopulationDistribution "
sql_statement += "(district_id, total, male, female, "
sql_statement += "number_household) "
sql_statement += "VALUES (%d, %d, %d, %d, %d)" % (
district_id,
total,
male,
female,
number_household,
)
self.cursor.execute(sql_statement)
def insert_building_type(self, building_type_id, description):
"""
Inserts a building-type description to the BuildingType table.
......@@ -573,12 +615,21 @@ class JapanDatabase(SpatialiteDatabase):
self.connection.commit()
logger.info("Districts and boundaries added")
def read_district_id(self, admin_id):
# Identify district_id based on admin_id from the District table
sql_statement = "SELECT id FROM District "
sql_statement += "WHERE admin_id = %d" % admin_id
self.cursor.execute(sql_statement)
result = self.cursor.fetchone()
return result
def import_exposure_data(
self,
dwelling_numbers_filepath,
building_numbers_filepath,
household_numbers_filepath,
dwelling_sizes_filepath,
population_distribution_filepath,
):
"""
Imports all exposure data from the Excel files provided by E-Stat, Japan. The following
......@@ -594,6 +645,8 @@ class JapanDatabase(SpatialiteDatabase):
File path to the file of numbers of households
dwelling_sizes_filepath (str):
File path to the file of dwelling sizes
population_distribution_filepath (str):
File path to the path of population distribution
"""
# Tables for building classifications
......@@ -626,13 +679,10 @@ class JapanDatabase(SpatialiteDatabase):
admin_id = int((row["Area classification"].split("_"))[0])
# Identify district_id based on admin_id from the District table
sql_statement = "SELECT id FROM District "
sql_statement += "WHERE admin_id = %d" % admin_id
self.cursor.execute(sql_statement)
result = self.cursor.fetchone()
if result is None: # Only data for which a district exist matter
district_id_result = self.read_district_id(admin_id)
if district_id_result is None: # Only data for which a district exist matter
continue
district_id = result[0]
district_id = district_id_result[0]
# Get ID of building type and number of stories
building_type_id = add_element_and_get_index(
......@@ -646,11 +696,7 @@ class JapanDatabase(SpatialiteDatabase):
for material_id in range(5):
dwelling_number = float(str(row[4 + material_id]).replace("-", "0"))
self.insert_dwelling_number(
district_id,
building_type_id,
material_id,
story_number_id,
dwelling_number,
district_id, building_type_id, material_id, story_number_id, dwelling_number
)
logger.debug(
"Dwelling number for district %s, type %s, %s added"
......@@ -670,13 +716,10 @@ class JapanDatabase(SpatialiteDatabase):
admin_id = int((row["Area classification"].split("_"))[0])
# Identify district_id based on admin_id from the District table
sql_statement = "SELECT id FROM District "
sql_statement += "WHERE admin_id = %d" % admin_id
self.cursor.execute(sql_statement)
result = self.cursor.fetchone()
if result is None: # Only data for which a district exist matter
district_id_result = self.read_district_id(admin_id)
if district_id_result is None: # Only data for which a district exist matter
continue
district_id = result[0]
district_id = district_id_result[0]
# Get ID of building type and number of stories
building_type_id = add_element_and_get_index(
......@@ -693,11 +736,7 @@ class JapanDatabase(SpatialiteDatabase):
if material_id == 2:
material_id = 4
self.insert_building_number(
district_id,
building_type_id,
material_id,
story_number_id,
building_number,
district_id, building_type_id, material_id, story_number_id, building_number
)
logger.debug(
"Building number for district %s, type %s, %s added"
......@@ -719,13 +758,10 @@ class JapanDatabase(SpatialiteDatabase):
admin_id = int((row["Area classification"].split("_"))[0])
# Identify district_id based on admin_id from the District table
sql_statement = "SELECT id FROM District "
sql_statement += "WHERE admin_id = %d" % admin_id
self.cursor.execute(sql_statement)
result = self.cursor.fetchone()
if result is None: # Only data for which a district exist matter
district_id_result = self.read_district_id(admin_id)
if district_id_result is None: # Only data for which a district exist matter
continue
district_id = result[0]
district_id = district_id_result[0]
# Get ID of building type, dwelling type, and tenure type
building_type_id = add_element_and_get_index(
......@@ -784,13 +820,10 @@ class JapanDatabase(SpatialiteDatabase):
admin_id = int((row["Area classification"].split("_"))[0])
# Identify district_id based on admin_id from the District table
sql_statement = "SELECT id FROM District "
sql_statement += "WHERE admin_id = %d" % admin_id
self.cursor.execute(sql_statement)
result = self.cursor.fetchone()
if result is None: # Only data for which a district exist matter
district_id_result = self.read_district_id(admin_id)
if district_id_result is None: # Only data for which a district exist matter
continue
district_id = result[0]
district_id = district_id_result[0]
# Get ID of building type, dwelling type, and tenure type
building_type_id = add_element_and_get_index(
......@@ -851,6 +884,30 @@ class JapanDatabase(SpatialiteDatabase):
)
logger.info("Number and sizes of dwellings added")
# Read PopulationDistribution
population_distribution_input = pandas.read_csv(
population_distribution_filepath, usecols=[2, 9, 10, 11, 12]
)
for index, row in population_distribution_input.iterrows():
admin_id = int(row["KEY_CODE"])
# Identify district_id based on admin_id from the District table
district_id_result = self.read_district_id(admin_id)
if district_id_result is None: # Only data for which a district exist matter
continue
district_id = district_id_result[0]
# Prepare and insert the data to the PopulationDistribution table
total = int(row[1])
male = int(row[2])
female = int(row[3])
number_household = int(row[4])
self.insert_population_distribution(
district_id, total, male, female, number_household
)
logger.debug("PopulationDistribution for district %s added" % admin_id)
logger.info("Population data added")
# Add the building types to the database
for building_type_id, building_type in enumerate(building_type_list):
self.insert_building_type(building_type_id, building_type)
......
......@@ -44,7 +44,11 @@ def main():
db.create_tables()
db.read_districts_and_boundaries("data/Boundary.gpkg")
db.import_exposure_data(
"data/e008_3e.xlsx", "data/e039_3e.xlsx", "data/e011_2e.xlsx", "data/e014e.xlsx"
"data/e008_3e.xlsx",
"data/e039_3e.xlsx",
"data/e011_2e.xlsx",
"data/e014e.xlsx",
"data/population_sub_municipal.csv",
)
# Leave the program
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment