Implemented the import of dwelling floorspace and sizes

exposurejapan/database.py

@@ -74,11 +74,19 @@ class JapanDatabase(SpatialiteDatabase):
                                   construction material and number of stories for each district
             HouseholdData       : Stores different parameters describing the household numbers,
                                   household members, and household spaces for each district
+            DwellingFloorspace  : Stores the floorspace per dwelling depending on the building,
+                                  dwelling, tenure types and construction material for each
+                                  district
+            DwellingDistribution: Stores the number of dwellings of different sizes depending on
+                                  the building, dwelling, tenure types and construction material
+                                  for each district
             BuildingType        : Stores the different types of buildings
             DwellingType        : Stores the different types of dwellings
             TenureType          : Stores the different types of tenures
             ConstructionMaterial: Stores the construction-material types
-            StoryNumber         : Stores the classifications of numbers of stories
+            StoryNumber         : Stores the number (e.g. 1, 2, 3, 4, ...) and classifications
+                                  (1-2, 3-5, ...) of numbers of stories.
+            DwellingSizeType    : Stores the different types of dwelling sizes
         """
 
         # Create table District
@@ -132,35 +140,72 @@ class JapanDatabase(SpatialiteDatabase):
         self.connection.execute(sql_statement)
         logger.debug("Table HouseholdData created")
 
+        # Create table DwellingFloorspace
+        sql_statement = "CREATE TABLE DwellingFloorspace ("
+        sql_statement += "id                        INTEGER PRIMARY KEY, "
+        sql_statement += "district_id               INTEGER, "
+        sql_statement += "building_type_id          INTEGER, "
+        sql_statement += "dwelling_type_id          INTEGER, "
+        sql_statement += "tenure_type_id            INTEGER, "
+        sql_statement += "construction_material_id  INTEGER, "
+        sql_statement += "floorspace                REAL)"
+        self.connection.execute(sql_statement)
+        logger.debug("Table DwellingFloorspace created")
+
+        # Create table DwellingDistribution
+        sql_statement = "CREATE TABLE DwellingDistribution ("
+        sql_statement += "id                        INTEGER PRIMARY KEY, "
+        sql_statement += "district_id               INTEGER, "
+        sql_statement += "building_type_id          INTEGER, "
+        sql_statement += "dwelling_type_id          INTEGER, "
+        sql_statement += "tenure_type_id            INTEGER, "
+        sql_statement += "construction_material_id  INTEGER, "
+        sql_statement += "dwelling_size_type_id     INTEGER, "
+        sql_statement += "number_dwelling           REAL)"
+        self.connection.execute(sql_statement)
+        logger.debug("Table DwellingDistribution created")
+
         # Create table BuildingType
         sql_statement = "CREATE TABLE BuildingType ("
         sql_statement += "id           INTEGER PRIMARY KEY, "
         sql_statement += "description  TEXT)"
         self.connection.execute(sql_statement)
+        logger.debug("Table BuildingType created")
 
         # Create table DwellingType
         sql_statement = "CREATE TABLE DwellingType ("
         sql_statement += "id           INTEGER PRIMARY KEY, "
         sql_statement += "description  TEXT)"
         self.connection.execute(sql_statement)
+        logger.debug("Table DwellingType created")
 
         # Create table TenureType
         sql_statement = "CREATE TABLE TenureType ("
         sql_statement += "id           INTEGER PRIMARY KEY, "
         sql_statement += "description  TEXT)"
         self.connection.execute(sql_statement)
+        logger.debug("Table TenureType created")
 
         # Create table ConstructionMaterial
         sql_statement = "CREATE TABLE ConstructionMaterial ("
         sql_statement += "id           INTEGER PRIMARY KEY, "
         sql_statement += "description  TEXT)"
         self.connection.execute(sql_statement)
+        logger.debug("Table ConstructionMaterial created")
 
-        # Create table NumberStories
+        # Create table StoryNumber
         sql_statement = "CREATE TABLE StoryNumber ("
         sql_statement += "id           INTEGER PRIMARY KEY, "
         sql_statement += "description  TEXT)"
         self.connection.execute(sql_statement)
+        logger.debug("Table StoryNumber created")
+
+        # Create table DwellingSizeType
+        sql_statement = "CREATE TABLE DwellingSizeType ("
+        sql_statement += "id           INTEGER PRIMARY KEY, "
+        sql_statement += "description  TEXT)"
+        self.connection.execute(sql_statement)
+        logger.debug("Table DwellingSizeType created")
 
     def insert_district(self, admin_id, admin_name, geom):
         """
@@ -319,6 +364,90 @@ class JapanDatabase(SpatialiteDatabase):
         )
         self.cursor.execute(sql_statement)
 
+    def insert_dwelling_floorspace(
+        self,
+        district_id,
+        building_type_id,
+        dwelling_type_id,
+        tenure_type_id,
+        construction_material_id,
+        floorspace,
+    ):
+        """
+        Inserts a full dataset to the DwellingFloorspace table.
+
+        Args:
+            district_id (int):
+                ID of the district. Corresponds to District.id
+            building_type_id (int):
+                ID of the building type. Corresponds to BuildingType.id
+            dwelling_type_id (int):
+                ID of the dwelling type. Corresponds to DwellingType.id
+            tenure_type_id (int):
+                ID of the tenure type. Corresponds to TenureType.id
+            construction_material_id (int):
+                ID of the construction material. Corresponds to ConstructionMaterial.id
+            floorspace (float):
+                Size of the floorspace in square meters
+        """
+
+        sql_statement = "INSERT INTO DwellingFloorspace "
+        sql_statement += "(district_id, building_type_id, dwelling_type_id, tenure_type_id, "
+        sql_statement += "construction_material_id, floorspace) "
+        sql_statement += "VALUES (%d, %d, %d, %d, %d, %f)" % (
+            district_id,
+            building_type_id,
+            dwelling_type_id,
+            tenure_type_id,
+            construction_material_id,
+            floorspace,
+        )
+        self.cursor.execute(sql_statement)
+
+    def insert_dwelling_distribution(
+        self,
+        district_id,
+        building_type_id,
+        dwelling_type_id,
+        tenure_type_id,
+        construction_material_id,
+        dwelling_size_type_id,
+        number_dwelling,
+    ):
+        """
+        Inserts a full dataset to the DwellingDistribution table.
+
+        Args:
+            district_id (int):
+                ID of the district. Corresponds to District.id
+            building_type_id (int):
+                ID of the building type. Corresponds to BuildingType.id
+            dwelling_type_id (int):
+                ID of the dwelling type. Corresponds to DwellingType.id
+            tenure_type_id (int):
+                ID of the tenure type. Corresponds to TenureType.id
+            construction_material_id (int):
+                ID of the construction material. Corresponds to ConstructionMaterial.id
+            dwelling_size_type_id (int):
+                ID of the dwelling-size type. Corresponds to DwellingSizeType.id
+            number_dwelling (float):
+                Number of dwellings for the combination of the other parameters
+        """
+
+        sql_statement = "INSERT INTO DwellingDistribution "
+        sql_statement += "(district_id, building_type_id, dwelling_type_id, tenure_type_id, "
+        sql_statement += "construction_material_id, dwelling_size_type_id, number_dwelling) "
+        sql_statement += "VALUES (%d, %d, %d, %d, %d, %d, %f)" % (
+            district_id,
+            building_type_id,
+            dwelling_type_id,
+            tenure_type_id,
+            construction_material_id,
+            dwelling_size_type_id,
+            number_dwelling,
+        )
+        self.cursor.execute(sql_statement)
+
     def insert_building_type(self, building_type_id, description):
         """
         Inserts a building-type description to the BuildingType table.
@@ -399,9 +528,25 @@ class JapanDatabase(SpatialiteDatabase):
         sql_statement += "VALUES (%d, '%s')" % (story_number_id, description)
         self.cursor.execute(sql_statement)
 
+    def insert_dwelling_size_type(self, dwelling_size_type_id, description):
+        """
+        Inserts a Dwelling-size description to the DwellingSizeType table.
+
+        Args:
+            dwelling_size_type_id (int):
+                ID of the dwelling-size type entry
+            description (str):
+                Description of the dwelling-size type entry
+        """
+
+        sql_statement = "INSERT INTO DwellingSizeType "
+        sql_statement += "(id, description) "
+        sql_statement += "VALUES (%d, '%s')" % (dwelling_size_type_id, description)
+        self.cursor.execute(sql_statement)
+
     def read_districts_and_boundaries(self, district_boundary_filepath):
         """
-        Imports all districts and boundaries from a prepared geopackage file
+        Imports all districts and boundaries from a prepared geopackage file.
         ('estat_bound_municipal`). The function opens this file as Spatialite database
         and copies all necessary district and boundary information into the `District` table.
 
@@ -416,7 +561,7 @@ class JapanDatabase(SpatialiteDatabase):
         sql_statement = "SELECT key_code_ward, "
         sql_statement += "CITY_NAME, "
         sql_statement += "AsWKT(CastAutomagic(geom)) "
-        sql_statement += "FROM estat_bound_municipal"
+        sql_statement += "FROM Boundary"
         boundary_db.cursor.execute(sql_statement)
         for row in boundary_db.cursor:
             self.insert_district(
@@ -429,7 +574,11 @@ class JapanDatabase(SpatialiteDatabase):
         logger.info("Districts and boundaries added")
 
     def import_exposure_data(
-        self, dwelling_numbers_filepath, building_numbers_filepath, household_numbers_filepath
+        self,
+        dwelling_numbers_filepath,
+        building_numbers_filepath,
+        household_numbers_filepath,
+        dwelling_sizes_filepath,
     ):
         """
         Imports all exposure data from the Excel files provided by E-Stat, Japan. The following
@@ -443,6 +592,8 @@ class JapanDatabase(SpatialiteDatabase):
                 File path to the file of number of buildings
             household_numbers_filepath (str):
                 File path to the file of numbers of households
+            dwelling_sizes_filepath (str):
+                File path to the file of dwelling sizes
         """
 
         # Tables for building classifications
@@ -457,6 +608,15 @@ class JapanDatabase(SpatialiteDatabase):
             "102_Wooden and fire-proofed",
             "2_Non-wooden",
         ]
+        dwelling_size_type_list = [
+            "0_Total",
+            "1_29m2 and under",
+            "2_30 to 49m2",
+            "3_50 to 69m2",
+            "4_70 to 99m2",
+            "5_100 to 149m2",
+            "6_150m2 and over",
+        ]
 
         # Read numbers of dwellings
         dwelling_numbers_input = pandas.read_excel(
@@ -614,6 +774,83 @@ class JapanDatabase(SpatialiteDatabase):
             )
         logger.info("Number of households added")
 
+        # Read numbers of dwellings by size
+        dwelling_sizes_input = pandas.read_excel(
+            dwelling_sizes_filepath,
+            header=10,
+            usecols=[4, 5, 7, 9, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21],
+        )
+        for index, row in dwelling_sizes_input.iterrows():
+            admin_id = int((row["Area classification"].split("_"))[0])
+
+            # Identify district_id based on admin_id from the District table
+            sql_statement = "SELECT id FROM District "
+            sql_statement += "WHERE admin_id = %d" % admin_id
+            self.cursor.execute(sql_statement)
+            result = self.cursor.fetchone()
+            if result is None:  # Only data for which a district exist matter
+                continue
+            district_id = result[0]
+
+            # Get ID of building type, dwelling type, and tenure type
+            building_type_id = add_element_and_get_index(
+                row["Type of building"], building_type_list
+            )
+            dwelling_type_id = add_element_and_get_index(
+                row["Type of dwelling"], dwelling_type_list
+            )
+            tenure_type_id = add_element_and_get_index(
+                row["Tenure of dwelling"], tenure_type_list
+            )
+            construction_material_id = add_element_and_get_index(
+                row["Construction material"], construction_material_list
+            )
+
+            # Prepare and insert the data to the DwellingFloorspace table
+            floorspace = float(str(row[13]).replace("-", "0"))
+            self.insert_dwelling_floorspace(
+                district_id,
+                building_type_id,
+                dwelling_type_id,
+                tenure_type_id,
+                construction_material_id,
+                floorspace,
+            )
+            logger.debug(
+                "Floorspace for district %s, type %s, %s, %s, %s added"
+                % (
+                    admin_id,
+                    building_type_list[building_type_id],
+                    dwelling_type_list[dwelling_type_id],
+                    tenure_type_list[tenure_type_id],
+                    construction_material_list[construction_material_id],
+                )
+            )
+
+            # Prepare and insert the data to the DwellingDistribution table
+            for dwelling_size_type_id in range(7):
+                number_dwelling = float(str(row[6 + dwelling_size_type_id]).replace("-", "0"))
+                self.insert_dwelling_distribution(
+                    district_id,
+                    building_type_id,
+                    dwelling_type_id,
+                    tenure_type_id,
+                    construction_material_id,
+                    dwelling_size_type_id,
+                    number_dwelling,
+                )
+            logger.debug(
+                "Dwelling numbers by size for district %s, type %s, %s, %s, %s added"
+                % (
+                    admin_id,
+                    building_type_list[building_type_id],
+                    dwelling_type_list[dwelling_type_id],
+                    tenure_type_list[tenure_type_id],
+                    construction_material_list[construction_material_id],
+                )
+            )
+        logger.info("Number and sizes of dwellings added")
+
         # Add the building types to the database
         for building_type_id, building_type in enumerate(building_type_list):
             self.insert_building_type(building_type_id, building_type)
@@ -639,6 +876,11 @@ class JapanDatabase(SpatialiteDatabase):
             construction_material_list
         ):
             self.insert_construction_material(construction_material_id, construction_material)
-        logger.info("construction-material types added")
+        logger.info("Construction-material types added")
+
+        # Add the types of dwelling sizes to the database
+        for dwelling_size_type_id, dwelling_size_type in enumerate(dwelling_size_type_list):
+            self.insert_dwelling_size_type(dwelling_size_type_id, dwelling_size_type)
+        logger.info("Dwelling-size types added")
 
         self.connection.commit()

exposurejapan/exposurejapan.py

@@ -42,8 +42,10 @@ def main():
         logger.warning("Spatialite extension cannot be loaded. Exiting ...")
         exit()
     db.create_tables()
-    db.read_districts_and_boundaries("data/estat_bound_municipal.gpkg")
-    db.import_exposure_data("data/e008_3e.xlsx", "data/e039_3e.xlsx", "data/e011_2e.xlsx")
+    db.read_districts_and_boundaries("data/Boundary.gpkg")
+    db.import_exposure_data(
+        "data/e008_3e.xlsx", "data/e039_3e.xlsx", "data/e011_2e.xlsx", "data/e014e.xlsx"
+    )
 
     # Leave the program
     sys.exit()