diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b32a166506354b75b643f234c1416739a855e416..242d499b71c8d77dadb5bf2f6be82c6406464631 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: python:3-buster +image: python:3-bullseye # Make pip cache the installed dependencies variables: diff --git a/exposurejapan/constants.py b/exposurejapan/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..0f0cf2c7b7c04f4befa7ec074adfaacc3f446ecc --- /dev/null +++ b/exposurejapan/constants.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2021: +# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or (at +# your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero +# General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. + + +# Constant numbers for Building attributes for distributing household data +DISTRICT_ID = 0 +BUILDING_TYPE_ID = 1 +CONSTRUCTION_MATERIAL_ID = 2 +NUMBER_DWELLING = 3 + +# Constant number for `total` value for building attributes +TOTAL = 0 + +# Constant numbers for construction material types. +WOODEN = 1 +WOODEN_EXCLUDING_WOODEN_AND_FIRE_PROOFED = 2 +WOODEN_AND_FIRE_PROOFED = 3 +NON_WOODEN = 4 diff --git a/exposurejapan/database.py b/exposurejapan/database.py index fd02f271518e4092d0e507f886c084de3b958512..9f12ffe23f5a11f30d021c9add35806d6e0297c3 100644 --- a/exposurejapan/database.py +++ b/exposurejapan/database.py @@ -19,7 +19,8 @@ import logging from exposurelib.database import SpatialiteDatabase import pandas - +import numpy +from exposurejapan import constants # Initialize log logger = logging.getLogger(__name__) @@ -365,7 +366,7 @@ class JapanDatabase(SpatialiteDatabase): sql_statement += "rooms_per_dwelling, tatami_per_dwelling, floorspace_per_dwelling, " sql_statement += "tatami_per_person, person_per_room) " - if construction_material_id == 0: + if construction_material_id == constants.TOTAL: sql_statement += "VALUES (%d, %d, %d, %d, %d, %f, %f, %f, %f, %f, %f, %f, %f)" % ( district_id, building_type_id, @@ -381,6 +382,20 @@ class JapanDatabase(SpatialiteDatabase): tatami_per_person, person_per_room, ) + elif construction_material_id != constants.TOTAL and ( + tenure_type_id != constants.TOTAL or dwelling_type_id != constants.TOTAL + ): + sql_statement += ( + "VALUES (%d, %d, %d, %d, %d, NULL, NULL, NULL, NULL, " + "NULL, NULL, NULL, NULL)" + % ( + district_id, + building_type_id, + dwelling_type_id, + tenure_type_id, + construction_material_id, + ) + ) else: sql_statement += ( "VALUES (%d, %d, %d, %d, %d, NULL, %f, %f, NULL, NULL, NULL, NULL, NULL)" @@ -653,6 +668,87 @@ class JapanDatabase(SpatialiteDatabase): result = self.cursor.fetchone() return result + def create_list_with_dwelling_data(self): + """ + Create an empty list and store the required building attributes from + the DwellingNumbers table in the list. + """ + + # Create an empty list to add dwelling attributes + dwelling_attributes = [] + + # Select the building_type_id and construction_material_id from DwellingNumber table + sql_statement = "SELECT district_id, building_type_id, " + sql_statement += "construction_material_id, " + sql_statement += "number_dwelling " + sql_statement += ( + "FROM DwellingNumber WHERE (construction_material_id != %d " + "AND construction_material_id != %d " + "AND construction_material_id != %d" + ")" + % ( + constants.TOTAL, + constants.WOODEN_EXCLUDING_WOODEN_AND_FIRE_PROOFED, + constants.WOODEN_AND_FIRE_PROOFED, + ) + ) + self.cursor.execute(sql_statement) + + # Store the DwellingNumber attributes to the dwelling_attributes list + for row in self.cursor: + dwelling_attributes.append(row) + return dwelling_attributes + + @staticmethod + def calculate_dwelling_numbers_proportion(dwelling_numbers): + """ + Calculates the proportion of dwelling_numbers for + wooden and non-wooden construction material. + + Args: + dwelling_numbers (list): + List of building_type_id, construction_material_id and its + corresponding number_dwelling for each district_id. + """ + + # Create empty lists for wooden, non-wooden and a list to store building attributes + # only. + wooden = [] + non_wooden = [] + proportion = [] + # Store dwelling-number values into wooden, non-wooden and numbers + for row in dwelling_numbers: + if row[constants.CONSTRUCTION_MATERIAL_ID] == constants.WOODEN: + wooden.append(row[constants.NUMBER_DWELLING]) + else: # Construction_material non-wooden + non_wooden.append(row[constants.NUMBER_DWELLING]) + + # Create a tuple with district_id, building_type_id and construction_material_id + attributes = ( + row[constants.DISTRICT_ID], + row[constants.BUILDING_TYPE_ID], + row[constants.CONSTRUCTION_MATERIAL_ID], + ) + add_element_and_get_index(attributes, proportion) + # Sum the wooden and non-wooden dwelling numbers + sum_wooden = sum(filter(None, wooden)) + sum_non_wooden = sum(filter(None, non_wooden)) + sum_wooden_non_wooden = sum_wooden + sum_non_wooden + # Calculate proportion of dwellings numbers + # and append numbers to store the proportions + for row in proportion: + if row[constants.CONSTRUCTION_MATERIAL_ID] == constants.WOODEN: + prop = numpy.true_divide(sum_wooden, sum_wooden_non_wooden) + index = proportion.index(row) + proportion[index] = row + (prop,) + numpy.seterr(divide="ignore", invalid="ignore") + else: + prop = numpy.true_divide(sum_non_wooden, sum_wooden_non_wooden) + index = proportion.index(row) + proportion[index] = row + (prop,) + numpy.seterr(divide="ignore", invalid="ignore") + return proportion + def import_exposure_data( self, dwelling_numbers_filepath, @@ -795,6 +891,10 @@ class JapanDatabase(SpatialiteDatabase): ) logger.info("Number of buildings added") + # List with dwelling attributes + dwelling_attributes = self.create_list_with_dwelling_data() + logger.info(dwelling_attributes) + # Read numbers of households household_numbers_input = pandas.read_excel( household_numbers_filepath, @@ -821,7 +921,20 @@ class JapanDatabase(SpatialiteDatabase): row["Tenure of dwelling"], tenure_type_list ) - # Insert building numbers for total, wooden and non-wooden construction material + # Create a temporary list to store dwelling attributes + dwelling_numbers = [] + + # Add number of dwellings into dwelling_numbers for combination of district_id, + # building_type_id and construction_material_id + for dwell in dwelling_attributes: + if ( + dwell[constants.DISTRICT_ID] == district_id + and dwell[constants.BUILDING_TYPE_ID] == building_type_id + ): + dwelling_numbers.append(dwell) + logger.info(dwelling_numbers) + + # Insert household data for each total, wooden and non-wooden construction material for construction_material_index in range(3): number_dwelling = float(str(row[5]).replace("-", "0")) rooms_per_dwelling = float(str(row[8]).replace("-", "0")) @@ -839,6 +952,25 @@ class JapanDatabase(SpatialiteDatabase): if construction_material_id == 2: construction_material_id = 4 + # Calculate the number_household and number_household_member + # for proportion of wooden and non-wooden dwelling numbers + for attr in JapanDatabase.calculate_dwelling_numbers_proportion( + dwelling_numbers + ): + if not all(numpy.isfinite(attr)): + continue + if ( + attr[constants.CONSTRUCTION_MATERIAL_ID] == construction_material_id + and tenure_type_id == constants.TOTAL + and dwelling_type_id == constants.TOTAL + ): + number_household = attr[constants.NUMBER_DWELLING] * number_household + number_household_member = ( + attr[constants.NUMBER_DWELLING] * number_household_member + ) + else: + continue + # Insert household numbers and related values self.insert_household_data( district_id,