Skip to content
GitLab
Commit 2c60a4f5 authored by Tara Evaz Zadeh's avatar Tara Evaz Zadeh
Browse files

Edited funcs to avoid opening and closing files repeatedly in for loops

parent 1c3ecd7b
Pipeline #18683 passed with stage
in 1 minute and 19 seconds
Hide whitespace changes
Inline Side-by-side
losscalculator/add_cellids_to_OBM_exposure_files.py
View file @ 2c60a4f5
#!/usr/bin/env python3
# Copyright (c) 2020:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
# Copyright (c) 2020-2021:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
......@@ -33,39 +33,35 @@ def AddRespectiveCellIdToOBMExposureFiles(exposureFile_path, OSMSource_path, Res
exposureSource = pd.read_csv(exposureFile_path)
originids = exposureSource.origin_id
def append_list_as_row(file_name, list_of_elem):
with open(file_name, "a+", newline="") as write_obj:
csv_writer = csv.writer(write_obj)
csv_writer.writerow(list_of_elem)
with open(Result_path, "a+", newline="") as write_obj:
csv_writer = csv.writer(write_obj)
title = [
"id",
"lon",
"lat",
"taxonomy",
"number",
"structural",
"night",
"occupancy",
"admin_name",
"admin_ID",
"origin_id",
"respectiveCellId",
]
csv_writer.writerow(title)
OSMIdToCellId = {}
for MappingItem in OSMSource:
if MappingItem[0] in OSMIdToCellId:
# Ignore the additional geometry to keep everything unambiguous
pass
OSMIdToCellId[MappingItem[0]] = MappingItem[2]
title = [
"id",
"lon",
"lat",
"taxonomy",
"number",
"structural",
"night",
"occupancy",
"admin_name",
"admin_ID",
"origin_id",
"respectiveCellId",
]
append_list_as_row(Result_path, title)
OSMIdToCellId = {}
for MappingItem in OSMSource:
if MappingItem[0] in OSMIdToCellId:
# Ignore the additional geometry to keep everything unambiguous
pass
OSMIdToCellId[MappingItem[0]] = MappingItem[2]
for asset in range(exposureSource.shape[0]):
exposureArr = list(exposureSource.loc[asset])
origin_id = originids[asset]
respectiveCellId = OSMIdToCellId[origin_id]
exposureArr.append(respectiveCellId)
append_list_as_row(Result_path, exposureArr)
for asset_from_model in range(exposureSource.shape[0]):
asset_for_calculation = list(exposureSource.loc[asset_from_model])
origin_id = originids[asset_from_model]
respectiveCellId = OSMIdToCellId[origin_id]
asset_for_calculation.append(respectiveCellId)
csv_writer.writerow(asset_for_calculation)
print("Execution time of the script", (datetime.datetime.now() - startTime))
losscalculator/damage_calculator.py
View file @ 2c60a4f5
#!/usr/bin/env python3
# Copyright (c) 2020:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
# Copyright (c) 2020-2021:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
......@@ -19,11 +19,11 @@
import numpy as np
import pandas as pd
import csv
import losslib
import os
import argparse
from damage_calculator_tile_version import damageCalculator_TileVersion
import csv
def get_exposure_per_tile(
......@@ -46,84 +46,77 @@ def get_exposure_per_tile(
ground_motion_field, lons_whole_area, lats_whole_area, interpolation_method
)
all_cell_ids = np.loadtxt(cellIdSource_filepath, dtype="str", skiprows=1)
def write_asset_to_file(file_name, list_of_elem):
"""This function writes a list (list_of_elem) to a file (file_name)"""
# Open the file to write the lists to. a+ gives the possibility of reading and
# writing if file exists and creating it if it doesn't.
with open(file_name, "a+", newline="") as write_obj:
csv_writer = csv.writer(write_obj)
csv_writer.writerow(list_of_elem)
if exposure_type == "building":
title = [
"geometry",
"",
"origin_id",
"",
"RespectiveCellid",
"",
"asset_id",
"",
"lon",
"",
"lat",
"",
"taxonomy",
"",
"gm_value",
"",
"PoEs",
"",
"PoOs",
"",
"tot_num_buildings",
"",
"structural_no_damage",
"",
"structural_slight",
"",
"structural_moderate",
"",
"structural_extensive",
"",
"structural_complete",
]
tile_id = "respectiveCellId"
else:
title = [
"geometry",
"",
"origin_id",
"",
"asset_id",
"",
"lon",
"",
"lat",
"",
"taxonomy",
"",
"gm_value",
"",
"PoEs",
"",
"PoOs",
"",
"tot_num_buildings",
"",
"structural_no_damage",
"",
"structural_slight",
"",
"structural_moderate",
"",
"structural_extensive",
"",
"structural_complete",
]
tile_id = "origin_id"
write_asset_to_file(result_filepath, title)
with open(result_filepath, "a+", newline="") as write_obj:
csv_writer = csv.writer(write_obj)
if exposure_type == "building":
title = [
"geometry",
"",
"origin_id",
"",
"RespectiveCellid",
"",
"asset_id",
"",
"lon",
"",
"lat",
"",
"taxonomy",
"",
"gm_value",
"",
"PoEs",
"",
"PoOs",
"",
"num_buildings",
"",
"structural_no_damage",
"",
"structural_slight",
"",
"structural_moderate",
"",
"structural_extensive",
"",
"structural_complete",
]
tile_id = "respectiveCellId"
else:
title = [
"geometry",
"",
"origin_id",
"",
"asset_id",
"",
"lon",
"",
"lat",
"",
"taxonomy",
"",
"gm_value",
"",
"PoEs",
"",
"PoOs",
"",
"num_buildings",
"",
"structural_no_damage",
"",
"structural_slight",
"",
"structural_moderate",
"",
"structural_extensive",
"",
"structural_complete",
]
tile_id = "origin_id"
csv_writer.writerow(title)
for i in range(all_cell_ids.shape[0]):
cell_id = all_cell_ids[i]
......
losscalculator/damage_calculator_tile_version.py
View file @ 2c60a4f5
#!/usr/bin/env python3
# Copyright (c) 2020:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
# Copyright (c) 2020-2021:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
......@@ -16,7 +16,6 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see http://www.gnu.org/licenses/.
import numpy as np
import csv
import datetime
......@@ -24,10 +23,10 @@ import losslib
def damageCalculator_TileVersion(
fullGroundMotionField,
Result_path,
fragilityFileDir,
exposures,
full_ground_motion_field,
result_filepath,
fragility_pathname,
exposure,
taxonomy_conversion_path,
shakemap_path,
geometry_source_path,
......@@ -36,12 +35,12 @@ def damageCalculator_TileVersion(
):
"""
Returns a file "Result_path" including damage results of a scenario earthquake using the
Returns a file "result_filepath" including damage results of a scenario earthquake using the
"losslib" functions.
Input:
------
- fullGroundMotionField: (Numpy nd-array of shape(m,n); m records with long,
- full_ground_motion_field: (Numpy nd-array of shape(m,n); m records with long,
lat, and n-2 gm-types)
contains locations and the ground-motion values of each location
in the format of lon, lat, gmValueofType1, gmValueofType2, ...,
......@@ -54,7 +53,7 @@ def damageCalculator_TileVersion(
As an example:
[23.6875,38.3402777778,'PGA','SA(0.3)','SA(0.6)','SA(1.0)']
Example extract:
>>> fullGroundMotionField
>>> full_ground_motion_field
array([[
[23.687 , 38.340, 0.179, 0.362 , 0.252,
0.152],
......@@ -66,22 +65,22 @@ def damageCalculator_TileVersion(
0.149]
]])
- Result_path: (str)
- result_filepath: (str)
Result file address.
Example extract:
>>> Result_path
>>> result_filepath
"/home/TileCalculations/DamageResult.csv"
- fragilityFileDir: (str)
- fragility_pathname: (str)
Address to the directory including all the fragility functions.
Example extract:
>>> fragilityFileDir
>>> fragility_pathname
"/home/TileCalculations/Fragilities"
- exposures: (pandas.DataFrame(series))
- exposure: (pandas.DataFrame(series))
informaion of the assets of the region of interest under hazard.
Example extract:
>>> exposures
>>> exposure
id lon lat ... admin_name admin_ID origin_id
0 GDE_Ind_0 23.687500 38.340278 ... Oropos GR_3514913 cell_2410244527
1 GDE_Ind_1 23.687500 38.340278 ... Oropos GR_3514913 cell_2410244527
......@@ -120,11 +119,11 @@ def damageCalculator_TileVersion(
Output:
-------
- Result_path: (arrays written to file)
- result_filepath: (arrays written to file)
file containing the damage elements for each asset of the exposure.
Example extract of the result file:
geometry,,origin_id,,asset_id,,lon,,lat,,taxonomy,,gmfValue,,PoEs,,PoOs,,
tot_num_buildings,,structural_No-damage,,structural_Slight,,structural_Moderate,,
num_buildings,,structural_No-damage,,structural_Slight,,structural_Moderate,,
structural_Extensive,,structural_Complete
"POLYGON ((23.68611111111113 38.3388888888889, 23.6888888888889 38.3388888888889,
23.6888888888889 38.34166666666667, 23.68611111111113 38.34166666666667,
......@@ -149,119 +148,116 @@ def damageCalculator_TileVersion(
# Skip the header
next(geometry_source, None)
# Read each column of the input "exposure"
taxonomies = exposures.taxonomy
tot_num_buildings_row = exposures.number
lons = exposures.lon
lats = exposures.lat
assetids = exposures.id
originids = exposures.origin_id
# Prepare Result File
def append_list_as_row(file_name, list_of_elem):
with open(file_name, "a+", newline="") as write_obj:
csv_writer = csv.writer(write_obj)
csv_writer.writerow(list_of_elem)
# Read each column of the exposure model
taxonomies = exposure.taxonomy
tot_num_buildings = exposure.number
lons = exposure.lon
lats = exposure.lat
asset_ids = exposure.id
origin_ids = exposure.origin_id
# Begin Computation
# Define a dictionary with keys as the ground-motion type and value as the column
# number of the ground-motion type in the shakemap file.
gmDict = {"PGA": 2, "SA(0.3)": 3, "SA(0.6)": 4, "SA(1.0)": 5, "SA(1)": 5}
# Calling the function "Taxonomy_to_Fragility" to get a dictionary with keys as the
# taxonomy and the values as both the fragility function name (excluding ".csv" part)
# and column of ground-motion_type in ground-motion_field file.
# taxonomy and the values as both the fragility function name (excluding the ".csv" part)
# and the column number of the respective ground-motion type in `ground-motion-field` file.
taxonomyToFragilityMap = losslib.Taxonomy_to_Fragility(
gmDict, taxonomyToFragilitySource, fragilityFileDir
gmDict, taxonomyToFragilitySource, fragility_pathname
)
# Calling the function "origin_id_to_geometry" to get a dictionary with keys as the
# origin_id and the value as the respective polygon.
origin_id_to_geometry_map = losslib.origin_id_to_geometry(geometry_source, exposureType)
# Define number of columns that contain the data in the fragiliy function files.
# Define number of columns that contain the data in the fragility function files.
cls = range(1, 101)
# Just a trick to have multiple commas between each result element, since we do
# not want a single comma as the delimiter due to having ploygons as a result element.
# not want a single comma as the delimiter due to having geometries as a result element.
a = [0, 2, 4, 6, 8]
# Looping through each line of the exposure file to do the computations line by line.
for asset in range(exposures.shape[0]):
taxonomy = taxonomies.iloc[asset]
fragilityFileName = taxonomyToFragilityMap[taxonomy][0] + ".csv"
tot_num_buildings = tot_num_buildings_row.iloc[asset]
lon = lons.iloc[asset]
lat = lats.iloc[asset]
asset_id = assetids.iloc[asset]
origin_id = originids.iloc[asset]
# Since for the OBM exposure files we also need to know the cell-id that the
# building is located in, referred as "RespectiveCellid" and the polygon of
# the buiding.
if exposureType == "OBM":
[geometry, RespectiveCellid] = origin_id_to_geometry_map[origin_id]
else:
geometry = origin_id_to_geometry_map[origin_id]
# Looping through each line of the exposure file to run the computations asset by asset.
with open(result_filepath, "a+", newline="") as write_obj:
csv_writer = csv.writer(write_obj)
for asset in range(exposure.shape[0]):
taxonomy = taxonomies.iloc[asset]
fragilityFileName = taxonomyToFragilityMap[taxonomy][0] + ".csv"
num_buildings = tot_num_buildings.iloc[asset]
lon = lons.iloc[asset]
lat = lats.iloc[asset]
asset_id = asset_ids.iloc[asset]
origin_id = origin_ids.iloc[asset]
# In case of `exposureType == "OBM"`, not only the geometry, but also the
# respective cell ID (in which the asset is located), is extracted
# using `origin_id_to_geometry_map`
if exposureType == "OBM":
[geometry, respective_cell_id] = origin_id_to_geometry_map[origin_id]
else:
geometry = origin_id_to_geometry_map[origin_id]
# Read fragility functions as numpy arrays.
fragility_function = np.loadtxt(
fragilityFileDir + "/" + fragilityFileName, delimiter=",", usecols=cls
)
# Achieve the ground-motion value from the fullGroundMotionField. Please note
# that the "fullGroundMotionField" contains many identical lines (since so many
# assets have same locations (same location leads to same ground-motion value))
# because we want it to have same number of lines and same orders as the exposure file,
# so that in this line we read the same line number as the exposure file.
gm_value = fullGroundMotionField[asset, taxonomyToFragilityMap[taxonomy][1]]
# Achieve Probabilities of exceedance and occurance.
[PoEs, PoOs] = losslib.get_PoEs(fragility_function, gm_value)
# Read fragility functions as numpy arrays.
fragility_function = np.loadtxt(
fragility_pathname + "/" + fragilityFileName, delimiter=",", usecols=cls
)
# Computing the ground-motion values from `full_ground_motion_field`. Please
# note that `full_ground_motion_field` contains many identical lines (because
# many of the assets have same locations and same location leads to the same
# ground-motion value). This way, `full_ground_motion_field` has the same number
# and order of assets as in the exposure model for simpler processing.
gm_value = full_ground_motion_field[asset, taxonomyToFragilityMap[taxonomy][1]]
# Computing probabilities of exceedance and occurrence.
[PoEs, PoOs] = losslib.get_PoEs(fragility_function, gm_value)
# Compute damage by assets
dmg_by_asset = [i * tot_num_buildings for i in PoOs]
for h in a:
dmg_by_asset.insert(h, "")
# Append results
if exposureType == "OBM":
arr0 = [
geometry,
"",
origin_id,
"",
"cell_" + RespectiveCellid,
"",
asset_id,
"",
lon,
"",
lat,
"",
taxonomy,
"",
gm_value,
"",
PoEs,
"",
PoOs,
"",
tot_num_buildings,
]
else:
arr0 = [
geometry,
"",
origin_id,
"",
asset_id,
"",
lon,
"",
lat,
"",
taxonomy,
"",
gm_value,
"",
PoEs,
"",
PoOs,
"",
tot_num_buildings,
]
arr0.extend(dmg_by_asset)
append_list_as_row(Result_path, arr0)
# Compute damage by assets
dmg_by_asset = [i * num_buildings for i in PoOs]
for h in a:
dmg_by_asset.insert(h, "")
# Append results
if exposureType == "OBM":
asset = [
geometry,
"",
origin_id,
"",
"cell_" + respective_cell_id,
"",
asset_id,
"",
lon,
"",
lat,
"",
taxonomy,
"",
gm_value,
"",
PoEs,
"",
PoOs,
"",
num_buildings,
]
else:
asset = [
geometry,
"",
origin_id,
"",
asset_id,
"",
lon,
"",
lat,
"",
taxonomy,
"",
gm_value,
"",
PoEs,
"",
PoOs,
"",
num_buildings,
]
asset.extend(dmg_by_asset)
csv_writer.writerow(asset)
print("Execution time of the script", (datetime.datetime.now() - startTime))
losscalculator/losslib.py
View file @ 2c60a4f5
#!/usr/bin/env python3
# Copyright (c) 2020:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
# Copyright (c) 2020-2021:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment