From e20e8c7461becc0d746b21c6d7a45341ecfe7c6a Mon Sep 17 00:00:00 2001 From: Danijel Schorlemmer Date: Fri, 9 Apr 2021 16:31:07 +0200 Subject: [PATCH] Implemented the import of the number of buildings --- exposurejapan/database.py | 173 +++++++++++++++++++++++++++------ exposurejapan/exposurejapan.py | 2 +- 2 files changed, 144 insertions(+), 31 deletions(-) diff --git a/exposurejapan/database.py b/exposurejapan/database.py index ca94d50..ed9f6cf 100644 --- a/exposurejapan/database.py +++ b/exposurejapan/database.py @@ -46,12 +46,14 @@ class JapanDatabase(SpatialiteDatabase): def create_tables(self): """ Creates all necessary tables in the database. These are: - District: Stores the districts with their IDs, names and geometries - DwellingNumber: Stores the number of dwellings depending on building types, - construction material and number of stories for each district - BuildingType: Stores the different types of buildings + District : Stores the districts with their IDs, names and geometries + DwellingNumber : Stores the number of dwellings depending on building types, + construction material and number of stories for each district + BuildingNumber : Stores the number of buildings depending on building types, + construction material and number of stories for each district + BuildingType : Stores the different types of buildings ConstructionMaterial: Stores the construction-material types - NumberStories: Stores the classifications of numbers of stories + StoryNumber : Stores the classifications of numbers of stories """ # Create table District @@ -71,11 +73,22 @@ class JapanDatabase(SpatialiteDatabase): sql_statement += "district_id INTEGER, " sql_statement += "building_type_id INTEGER, " sql_statement += "construction_material_id INTEGER, " - sql_statement += "number_stories_id INTEGER, " + sql_statement += "story_number_id INTEGER, " sql_statement += "number_dwelling REAL)" self.connection.execute(sql_statement) logger.debug("Table DwellingNumber created") + # Create table BuildingNumber + sql_statement = "CREATE TABLE BuildingNumber (" + sql_statement += "id INTEGER PRIMARY KEY AUTOINCREMENT, " + sql_statement += "district_id INTEGER, " + sql_statement += "building_type_id INTEGER, " + sql_statement += "construction_material_id INTEGER, " + sql_statement += "story_number_id INTEGER, " + sql_statement += "number_building REAL)" + self.connection.execute(sql_statement) + logger.debug("Table BuildingNumber created") + # Create table BuildingType sql_statement = "CREATE TABLE BuildingType (" sql_statement += "id INTEGER PRIMARY KEY, " @@ -89,7 +102,7 @@ class JapanDatabase(SpatialiteDatabase): self.connection.execute(sql_statement) # Create table NumberStories - sql_statement = "CREATE TABLE NumberStories (" + sql_statement = "CREATE TABLE StoryNumber (" sql_statement += "id INTEGER PRIMARY KEY, " sql_statement += "description TEXT)" self.connection.execute(sql_statement) @@ -117,11 +130,11 @@ class JapanDatabase(SpatialiteDatabase): district_id, building_type_id, construction_material_id, - number_stories_id, + story_number_id, number_dwelling, ): """ - Inserts a full dataset to the BuildingNumber table. + Inserts a full dataset to the DwellingNumber table. Args: district_id (int): @@ -130,24 +143,60 @@ class JapanDatabase(SpatialiteDatabase): ID of the building type. Corresponds to BuildingType.id construction_material_id (int): ID of the construction material. Corresponds to ConstructionMaterial.id - number_stories_id (int): + story_number_id (int): ID of the number of stories type. Corresponds to NumberStories.id number_dwelling (float): - Number of buildings for the combination of the other parameters + Number of dwellings for the combination of the other parameters """ sql_statement = "INSERT INTO DwellingNumber " sql_statement += "(district_id, building_type_id, construction_material_id, " - sql_statement += "number_stories_id, number_dwelling) " + sql_statement += "story_number_id, number_dwelling) " sql_statement += "VALUES (%d, %d, %d, %d, %f)" % ( district_id, building_type_id, construction_material_id, - number_stories_id, + story_number_id, number_dwelling, ) self.cursor.execute(sql_statement) + def insert_building_number( + self, + district_id, + building_type_id, + construction_material_id, + story_number_id, + number_building, + ): + """ + Inserts a full dataset to the BuildingNumber table. + + Args: + district_id (int): + ID of the district. Corresponds to District.id + building_type_id (int): + ID of the building type. Corresponds to BuildingType.id + construction_material_id (int): + ID of the construction material. Corresponds to ConstructionMaterial.id + story_number_id (int): + ID of the number of stories type. Corresponds to NumberStories.id + number_building (float): + Number of buildings for the combination of the other parameters + """ + + sql_statement = "INSERT INTO BuildingNumber " + sql_statement += "(district_id, building_type_id, construction_material_id, " + sql_statement += "story_number_id, number_building) " + sql_statement += "VALUES (%d, %d, %d, %d, %f)" % ( + district_id, + building_type_id, + construction_material_id, + story_number_id, + number_building, + ) + self.cursor.execute(sql_statement) + def insert_building_type(self, building_type_id, description): """ Inserts a building-type description to the BuildingType table. @@ -180,20 +229,20 @@ class JapanDatabase(SpatialiteDatabase): sql_statement += "VALUES (%d, '%s')" % (construction_material_id, description) self.cursor.execute(sql_statement) - def insert_number_stories(self, number_stories_id, description): + def insert_story_number(self, story_number_id, description): """ - Inserts a story-number description to the NumberStories table. + Inserts a story-number description to the StoryNumber table. Args: - number_stories_id (int): + story_number_id (int): ID of the number of stories entry description (str): Description of the number of stories entry """ - sql_statement = "INSERT INTO NumberStories " + sql_statement = "INSERT INTO StoryNumber " sql_statement += "(id, description) " - sql_statement += "VALUES (%d, '%s')" % (number_stories_id, description) + sql_statement += "VALUES (%d, '%s')" % (story_number_id, description) self.cursor.execute(sql_statement) def read_districts_and_boundaries(self, district_boundary_filepath): @@ -225,7 +274,7 @@ class JapanDatabase(SpatialiteDatabase): self.connection.commit() logger.info("Districts and boundaries added") - def import_exposure_data(self, dwelling_numbers_filepath): + def import_exposure_data(self, dwelling_numbers_filepath, building_numbers_filepath): """ Imports all exposure data from the Excel files provided by E-Stat, Japan. The following files are needed: @@ -233,11 +282,14 @@ class JapanDatabase(SpatialiteDatabase): Args: dwelling_numbers_filepath (str): - File path to the file of number_of_dwellings) + File path to the file of number of dwellings + building_numbers_filepath (str): + File path to the file of number of buildings """ + # Tables for building classifications building_type_list = [] - number_stories_list = [] + story_number_list = [] construction_material_list = [ "0_Total", "1_Wooden", @@ -246,6 +298,7 @@ class JapanDatabase(SpatialiteDatabase): "2_Non-wooden", ] + # Read numbers of dwellings dwelling_numbers_input = pandas.read_excel( dwelling_numbers_filepath, header=10, usecols=[4, 5, 7, 9, 10, 11, 12, 13, 14] ) @@ -268,21 +321,81 @@ class JapanDatabase(SpatialiteDatabase): building_type_id = building_type_list.index(building_type) # Get ID of number of stories - number_stories = row["Stories of building"] - if number_stories not in number_stories_list: - number_stories_list.append(number_stories) - number_stories_id = number_stories_list.index(number_stories) + story_number = row["Stories of building"] + if story_number not in story_number_list: + story_number_list.append(story_number) + story_number_id = story_number_list.index(story_number) - # Insert building numbers for each construction material + # Insert dwelling numbers for each construction material for material_id in range(5): dwelling_number = float(str(row[4 + material_id]).replace("-", "0")) self.insert_dwelling_number( district_id, building_type_id, material_id, - number_stories_id, + story_number_id, dwelling_number, ) + logger.debug( + "Dwelling number for district %s, type %s, %s added" + % ( + admin_id, + building_type_list[building_type_id], + story_number_list[story_number_id], + ) + ) + logger.info("Number of dwellings added") + + # Read numbers of buildings + building_numbers_input = pandas.read_excel( + building_numbers_filepath, header=10, usecols=[4, 5, 7, 9, 10, 11, 12] + ) + for index, row in building_numbers_input.iterrows(): + admin_id = int((row["Area classification"].split("_"))[0]) + + # Identify district_id based on admin_id from the District table + sql_statement = "SELECT id FROM District " + sql_statement += "WHERE admin_id = %d" % admin_id + self.cursor.execute(sql_statement) + result = self.cursor.fetchone() + if result is None: # Only data for which a district exist matter + continue + district_id = result[0] + + # Get ID of building type + building_type = row["Type of building"] + if building_type not in building_type_list: + building_type_list.append(building_type) + building_type_id = building_type_list.index(building_type) + + # Get ID of number of stories + story_number = row["Stories of building"] + if story_number not in story_number_list: + story_number_list.append(story_number) + story_number_id = story_number_list.index(story_number) + + # Insert building numbers for each construction material + for material_index in range(3): + building_number = float(str(row[4 + material_index]).replace("-", "0")) + material_id = material_index # Map the column index to the material id + if material_id == 2: + material_id = 4 + self.insert_building_number( + district_id, + building_type_id, + material_id, + story_number_id, + building_number, + ) + logger.debug( + "Building number for district %s, type %s, %s added" + % ( + admin_id, + building_type_list[building_type_id], + story_number_list[story_number_id], + ) + ) + logger.info("Number of buildings added") # Add the building types to the database for building_type in building_type_list: @@ -290,9 +403,9 @@ class JapanDatabase(SpatialiteDatabase): self.insert_building_type(building_type_id, building_type) # Add the number of stories to the database - for number_stories in number_stories_list: - number_stories_id = number_stories_list.index(number_stories) - self.insert_number_stories(number_stories_id, number_stories) + for story_number in story_number_list: + story_number_id = story_number_list.index(story_number) + self.insert_story_number(story_number_id, story_number) # Add the number of stories to the database for construction_material in construction_material_list: diff --git a/exposurejapan/exposurejapan.py b/exposurejapan/exposurejapan.py index 7961d95..6f682ab 100644 --- a/exposurejapan/exposurejapan.py +++ b/exposurejapan/exposurejapan.py @@ -43,7 +43,7 @@ def main(): exit() db.create_tables() db.read_districts_and_boundaries("data/estat_bound_municipal.gpkg") - db.import_exposure_data("data/e008_3e.xlsx") + db.import_exposure_data("data/e008_3e.xlsx", "data/e039_3e.xlsx") # Leave the program sys.exit() -- GitLab