Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Dynamic Exposure
Global Dynamic Exposure
losscalculator
Commits
2c60a4f5
Commit
2c60a4f5
authored
Jan 21, 2021
by
Tara Evaz Zadeh
Browse files
Edited funcs to avoid opening and closing files repeatedly in for loops
parent
1c3ecd7b
Pipeline
#18683
passed with stage
in 1 minute and 19 seconds
Changes
4
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
losscalculator/add_cellids_to_OBM_exposure_files.py
View file @
2c60a4f5
#!/usr/bin/env python3
# Copyright (c) 2020:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
# Copyright (c) 2020
-2021
:
#
Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
...
...
@@ -33,39 +33,35 @@ def AddRespectiveCellIdToOBMExposureFiles(exposureFile_path, OSMSource_path, Res
exposureSource
=
pd
.
read_csv
(
exposureFile_path
)
originids
=
exposureSource
.
origin_id
def
append_list_as_row
(
file_name
,
list_of_elem
):
with
open
(
file_name
,
"a+"
,
newline
=
""
)
as
write_obj
:
csv_writer
=
csv
.
writer
(
write_obj
)
csv_writer
.
writerow
(
list_of_elem
)
with
open
(
Result_path
,
"a+"
,
newline
=
""
)
as
write_obj
:
csv_writer
=
csv
.
writer
(
write_obj
)
title
=
[
"id"
,
"lon"
,
"lat"
,
"taxonomy"
,
"number"
,
"structural"
,
"night"
,
"occupancy"
,
"admin_name"
,
"admin_ID"
,
"origin_id"
,
"respectiveCellId"
,
]
csv_writer
.
writerow
(
title
)
OSMIdToCellId
=
{}
for
MappingItem
in
OSMSource
:
if
MappingItem
[
0
]
in
OSMIdToCellId
:
# Ignore the additional geometry to keep everything unambiguous
pass
OSMIdToCellId
[
MappingItem
[
0
]]
=
MappingItem
[
2
]
title
=
[
"id"
,
"lon"
,
"lat"
,
"taxonomy"
,
"number"
,
"structural"
,
"night"
,
"occupancy"
,
"admin_name"
,
"admin_ID"
,
"origin_id"
,
"respectiveCellId"
,
]
append_list_as_row
(
Result_path
,
title
)
OSMIdToCellId
=
{}
for
MappingItem
in
OSMSource
:
if
MappingItem
[
0
]
in
OSMIdToCellId
:
# Ignore the additional geometry to keep everything unambiguous
pass
OSMIdToCellId
[
MappingItem
[
0
]]
=
MappingItem
[
2
]
for
asset
in
range
(
exposureSource
.
shape
[
0
]):
exposureArr
=
list
(
exposureSource
.
loc
[
asset
])
origin_id
=
originids
[
asset
]
respectiveCellId
=
OSMIdToCellId
[
origin_id
]
exposureArr
.
append
(
respectiveCellId
)
append_list_as_row
(
Result_path
,
exposureArr
)
for
asset_from_model
in
range
(
exposureSource
.
shape
[
0
]):
asset_for_calculation
=
list
(
exposureSource
.
loc
[
asset_from_model
])
origin_id
=
originids
[
asset_from_model
]
respectiveCellId
=
OSMIdToCellId
[
origin_id
]
asset_for_calculation
.
append
(
respectiveCellId
)
csv_writer
.
writerow
(
asset_for_calculation
)
print
(
"Execution time of the script"
,
(
datetime
.
datetime
.
now
()
-
startTime
))
losscalculator/damage_calculator.py
View file @
2c60a4f5
#!/usr/bin/env python3
# Copyright (c) 2020:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
# Copyright (c) 2020
-2021
:
#
Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
...
...
@@ -19,11 +19,11 @@
import
numpy
as
np
import
pandas
as
pd
import
csv
import
losslib
import
os
import
argparse
from
damage_calculator_tile_version
import
damageCalculator_TileVersion
import
csv
def
get_exposure_per_tile
(
...
...
@@ -46,84 +46,77 @@ def get_exposure_per_tile(
ground_motion_field
,
lons_whole_area
,
lats_whole_area
,
interpolation_method
)
all_cell_ids
=
np
.
loadtxt
(
cellIdSource_filepath
,
dtype
=
"str"
,
skiprows
=
1
)
def
write_asset_to_file
(
file_name
,
list_of_elem
):
"""This function writes a list (list_of_elem) to a file (file_name)"""
# Open the file to write the lists to. a+ gives the possibility of reading and
# writing if file exists and creating it if it doesn't.
with
open
(
file_name
,
"a+"
,
newline
=
""
)
as
write_obj
:
csv_writer
=
csv
.
writer
(
write_obj
)
csv_writer
.
writerow
(
list_of_elem
)
if
exposure_type
==
"building"
:
title
=
[
"geometry"
,
""
,
"origin_id"
,
""
,
"RespectiveCellid"
,
""
,
"asset_id"
,
""
,
"lon"
,
""
,
"lat"
,
""
,
"taxonomy"
,
""
,
"gm_value"
,
""
,
"PoEs"
,
""
,
"PoOs"
,
""
,
"tot_num_buildings"
,
""
,
"structural_no_damage"
,
""
,
"structural_slight"
,
""
,
"structural_moderate"
,
""
,
"structural_extensive"
,
""
,
"structural_complete"
,
]
tile_id
=
"respectiveCellId"
else
:
title
=
[
"geometry"
,
""
,
"origin_id"
,
""
,
"asset_id"
,
""
,
"lon"
,
""
,
"lat"
,
""
,
"taxonomy"
,
""
,
"gm_value"
,
""
,
"PoEs"
,
""
,
"PoOs"
,
""
,
"tot_num_buildings"
,
""
,
"structural_no_damage"
,
""
,
"structural_slight"
,
""
,
"structural_moderate"
,
""
,
"structural_extensive"
,
""
,
"structural_complete"
,
]
tile_id
=
"origin_id"
write_asset_to_file
(
result_filepath
,
title
)
with
open
(
result_filepath
,
"a+"
,
newline
=
""
)
as
write_obj
:
csv_writer
=
csv
.
writer
(
write_obj
)
if
exposure_type
==
"building"
:
title
=
[
"geometry"
,
""
,
"origin_id"
,
""
,
"RespectiveCellid"
,
""
,
"asset_id"
,
""
,
"lon"
,
""
,
"lat"
,
""
,
"taxonomy"
,
""
,
"gm_value"
,
""
,
"PoEs"
,
""
,
"PoOs"
,
""
,
"num_buildings"
,
""
,
"structural_no_damage"
,
""
,
"structural_slight"
,
""
,
"structural_moderate"
,
""
,
"structural_extensive"
,
""
,
"structural_complete"
,
]
tile_id
=
"respectiveCellId"
else
:
title
=
[
"geometry"
,
""
,
"origin_id"
,
""
,
"asset_id"
,
""
,
"lon"
,
""
,
"lat"
,
""
,
"taxonomy"
,
""
,
"gm_value"
,
""
,
"PoEs"
,
""
,
"PoOs"
,
""
,
"num_buildings"
,
""
,
"structural_no_damage"
,
""
,
"structural_slight"
,
""
,
"structural_moderate"
,
""
,
"structural_extensive"
,
""
,
"structural_complete"
,
]
tile_id
=
"origin_id"
csv_writer
.
writerow
(
title
)
for
i
in
range
(
all_cell_ids
.
shape
[
0
]):
cell_id
=
all_cell_ids
[
i
]
...
...
losscalculator/damage_calculator_tile_version.py
View file @
2c60a4f5
#!/usr/bin/env python3
# Copyright (c) 2020:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
# Copyright (c) 2020
-2021
:
#
Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
...
...
@@ -16,7 +16,6 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see http://www.gnu.org/licenses/.
import
numpy
as
np
import
csv
import
datetime
...
...
@@ -24,10 +23,10 @@ import losslib
def
damageCalculator_TileVersion
(
full
G
round
M
otion
F
ield
,
R
esult_path
,
fragility
FileDir
,
exposure
s
,
full
_g
round
_m
otion
_f
ield
,
r
esult_
file
path
,
fragility
_pathname
,
exposure
,
taxonomy_conversion_path
,
shakemap_path
,
geometry_source_path
,
...
...
@@ -36,12 +35,12 @@ def damageCalculator_TileVersion(
):
"""
Returns a file "
R
esult_path" including damage results of a scenario earthquake using the
Returns a file "
r
esult_
file
path" including damage results of a scenario earthquake using the
"losslib" functions.
Input:
------
- full
G
round
M
otion
F
ield: (Numpy nd-array of shape(m,n); m records with long,
- full
_g
round
_m
otion
_f
ield: (Numpy nd-array of shape(m,n); m records with long,
lat, and n-2 gm-types)
contains locations and the ground-motion values of each location
in the format of lon, lat, gmValueofType1, gmValueofType2, ...,
...
...
@@ -54,7 +53,7 @@ def damageCalculator_TileVersion(
As an example:
[23.6875,38.3402777778,'PGA','SA(0.3)','SA(0.6)','SA(1.0)']
Example extract:
>>> full
G
round
M
otion
F
ield
>>> full
_g
round
_m
otion
_f
ield
array([[
[23.687 , 38.340, 0.179, 0.362 , 0.252,
0.152],
...
...
@@ -66,22 +65,22 @@ def damageCalculator_TileVersion(
0.149]
]])
-
R
esult_path: (str)
-
r
esult_
file
path: (str)
Result file address.
Example extract:
>>>
R
esult_path
>>>
r
esult_
file
path
"/home/TileCalculations/DamageResult.csv"
- fragility
FileDir
: (str)
- fragility
_pathname
: (str)
Address to the directory including all the fragility functions.
Example extract:
>>> fragility
FileDir
>>> fragility
_pathname
"/home/TileCalculations/Fragilities"
- exposure
s
: (pandas.DataFrame(series))
- exposure: (pandas.DataFrame(series))
informaion of the assets of the region of interest under hazard.
Example extract:
>>> exposure
s
>>> exposure
id lon lat ... admin_name admin_ID origin_id
0 GDE_Ind_0 23.687500 38.340278 ... Oropos GR_3514913 cell_2410244527
1 GDE_Ind_1 23.687500 38.340278 ... Oropos GR_3514913 cell_2410244527
...
...
@@ -120,11 +119,11 @@ def damageCalculator_TileVersion(
Output:
-------
-
R
esult_path: (arrays written to file)
-
r
esult_
file
path: (arrays written to file)
file containing the damage elements for each asset of the exposure.
Example extract of the result file:
geometry,,origin_id,,asset_id,,lon,,lat,,taxonomy,,gmfValue,,PoEs,,PoOs,,
tot_
num_buildings,,structural_No-damage,,structural_Slight,,structural_Moderate,,
num_buildings,,structural_No-damage,,structural_Slight,,structural_Moderate,,
structural_Extensive,,structural_Complete
"POLYGON ((23.68611111111113 38.3388888888889, 23.6888888888889 38.3388888888889,
23.6888888888889 38.34166666666667, 23.68611111111113 38.34166666666667,
...
...
@@ -149,119 +148,116 @@ def damageCalculator_TileVersion(
# Skip the header
next
(
geometry_source
,
None
)
# Read each column of the input "exposure"
taxonomies
=
exposures
.
taxonomy
tot_num_buildings_row
=
exposures
.
number
lons
=
exposures
.
lon
lats
=
exposures
.
lat
assetids
=
exposures
.
id
originids
=
exposures
.
origin_id
# Prepare Result File
def
append_list_as_row
(
file_name
,
list_of_elem
):
with
open
(
file_name
,
"a+"
,
newline
=
""
)
as
write_obj
:
csv_writer
=
csv
.
writer
(
write_obj
)
csv_writer
.
writerow
(
list_of_elem
)
# Read each column of the exposure model
taxonomies
=
exposure
.
taxonomy
tot_num_buildings
=
exposure
.
number
lons
=
exposure
.
lon
lats
=
exposure
.
lat
asset_ids
=
exposure
.
id
origin_ids
=
exposure
.
origin_id
# Begin Computation
# Define a dictionary with keys as the ground-motion type and value as the column
# number of the ground-motion type in the shakemap file.
gmDict
=
{
"PGA"
:
2
,
"SA(0.3)"
:
3
,
"SA(0.6)"
:
4
,
"SA(1.0)"
:
5
,
"SA(1)"
:
5
}
# Calling the function "Taxonomy_to_Fragility" to get a dictionary with keys as the
# taxonomy and the values as both the fragility function name (excluding ".csv" part)
# and column
of
ground-motion
_
type in ground-motion
_
field file.
# taxonomy and the values as both the fragility function name (excluding
the
".csv" part)
# and
the
column
number of the respective
ground-motion
type in
`
ground-motion
-
field
`
file.
taxonomyToFragilityMap
=
losslib
.
Taxonomy_to_Fragility
(
gmDict
,
taxonomyToFragilitySource
,
fragility
FileDir
gmDict
,
taxonomyToFragilitySource
,
fragility
_pathname
)
# Calling the function "origin_id_to_geometry" to get a dictionary with keys as the
# origin_id and the value as the respective polygon.
origin_id_to_geometry_map
=
losslib
.
origin_id_to_geometry
(
geometry_source
,
exposureType
)
# Define number of columns that contain the data in the fragiliy function files.
# Define number of columns that contain the data in the fragili
t
y function files.
cls
=
range
(
1
,
101
)
# Just a trick to have multiple commas between each result element, since we do
# not want a single comma as the delimiter due to having
ploygon
s as a result element.
# not want a single comma as the delimiter due to having
geometrie
s as a result element.
a
=
[
0
,
2
,
4
,
6
,
8
]
# Looping through each line of the exposure file to do the computations line by line.
for
asset
in
range
(
exposures
.
shape
[
0
]):
taxonomy
=
taxonomies
.
iloc
[
asset
]
fragilityFileName
=
taxonomyToFragilityMap
[
taxonomy
][
0
]
+
".csv"
tot_num_buildings
=
tot_num_buildings_row
.
iloc
[
asset
]
lon
=
lons
.
iloc
[
asset
]
lat
=
lats
.
iloc
[
asset
]
asset_id
=
assetids
.
iloc
[
asset
]
origin_id
=
originids
.
iloc
[
asset
]
# Since for the OBM exposure files we also need to know the cell-id that the
# building is located in, referred as "RespectiveCellid" and the polygon of
# the buiding.
if
exposureType
==
"OBM"
:
[
geometry
,
RespectiveCellid
]
=
origin_id_to_geometry_map
[
origin_id
]
else
:
geometry
=
origin_id_to_geometry_map
[
origin_id
]
# Looping through each line of the exposure file to run the computations asset by asset.
with
open
(
result_filepath
,
"a+"
,
newline
=
""
)
as
write_obj
:
csv_writer
=
csv
.
writer
(
write_obj
)
for
asset
in
range
(
exposure
.
shape
[
0
]):
taxonomy
=
taxonomies
.
iloc
[
asset
]
fragilityFileName
=
taxonomyToFragilityMap
[
taxonomy
][
0
]
+
".csv"
num_buildings
=
tot_num_buildings
.
iloc
[
asset
]
lon
=
lons
.
iloc
[
asset
]
lat
=
lats
.
iloc
[
asset
]
asset_id
=
asset_ids
.
iloc
[
asset
]
origin_id
=
origin_ids
.
iloc
[
asset
]
# In case of `exposureType == "OBM"`, not only the geometry, but also the
# respective cell ID (in which the asset is located), is extracted
# using `origin_id_to_geometry_map`
if
exposureType
==
"OBM"
:
[
geometry
,
respective_cell_id
]
=
origin_id_to_geometry_map
[
origin_id
]
else
:
geometry
=
origin_id_to_geometry_map
[
origin_id
]
# Read fragility functions as numpy arrays.
fragility_function
=
np
.
loadtxt
(
fragility
FileDir
+
"/"
+
fragilityFileName
,
delimiter
=
","
,
usecols
=
cls
)
# Achieve
the ground-motion value from
the
full
G
round
M
otion
F
ield. Please
note
# that the "
full
G
round
M
otion
F
ield
"
contains many identical lines (
since so many
#
assets have same locations
(
same location leads to same
ground-motion value))
# because we want it to have same number of lines and same orders as the exposure file,
# so that in this line we read the same line number as the exposure file
.
gm_value
=
full
G
round
M
otion
F
ield
[
asset
,
taxonomyToFragilityMap
[
taxonomy
][
1
]]
# Achieve P
robabilities of exceedance and occur
a
nce.
[
PoEs
,
PoOs
]
=
losslib
.
get_PoEs
(
fragility_function
,
gm_value
)
# Read fragility functions as numpy arrays.
fragility_function
=
np
.
loadtxt
(
fragility
_pathname
+
"/"
+
fragilityFileName
,
delimiter
=
","
,
usecols
=
cls
)
# Computing
the ground-motion value
s
from
`
full
_g
round
_m
otion
_f
ield
`
. Please
# note that `
full
_g
round
_m
otion
_f
ield
`
contains many identical lines (
because
# many of the
assets have same locations
and
same location leads to
the
same
# ground-motion value). This way, `full_ground_motion_field` has the same number
# and order of assets as in the exposure model for simpler processing
.
gm_value
=
full
_g
round
_m
otion
_f
ield
[
asset
,
taxonomyToFragilityMap
[
taxonomy
][
1
]]
# Computing p
robabilities of exceedance and occur
re
nce.
[
PoEs
,
PoOs
]
=
losslib
.
get_PoEs
(
fragility_function
,
gm_value
)
# Compute damage by assets
dmg_by_asset
=
[
i
*
tot_
num_buildings
for
i
in
PoOs
]
for
h
in
a
:
dmg_by_asset
.
insert
(
h
,
""
)
# Append results
if
exposureType
==
"OBM"
:
arr0
=
[
geometry
,
""
,
origin_id
,
""
,
"cell_"
+
R
espective
C
ellid
,
""
,
asset_id
,
""
,
lon
,
""
,
lat
,
""
,
taxonomy
,
""
,
gm_value
,
""
,
PoEs
,
""
,
PoOs
,
""
,
tot_
num_buildings
,
]
else
:
arr0
=
[
geometry
,
""
,
origin_id
,
""
,
asset_id
,
""
,
lon
,
""
,
lat
,
""
,
taxonomy
,
""
,
gm_value
,
""
,
PoEs
,
""
,
PoOs
,
""
,
tot_
num_buildings
,
]
arr0
.
extend
(
dmg_by_asset
)
append_list_as_row
(
Result_path
,
arr0
)
# Compute damage by assets
dmg_by_asset
=
[
i
*
num_buildings
for
i
in
PoOs
]
for
h
in
a
:
dmg_by_asset
.
insert
(
h
,
""
)
# Append results
if
exposureType
==
"OBM"
:
asset
=
[
geometry
,
""
,
origin_id
,
""
,
"cell_"
+
r
espective
_c
ell
_
id
,
""
,
asset_id
,
""
,
lon
,
""
,
lat
,
""
,
taxonomy
,
""
,
gm_value
,
""
,
PoEs
,
""
,
PoOs
,
""
,
num_buildings
,
]
else
:
asset
=
[
geometry
,
""
,
origin_id
,
""
,
asset_id
,
""
,
lon
,
""
,
lat
,
""
,
taxonomy
,
""
,
gm_value
,
""
,
PoEs
,
""
,
PoOs
,
""
,
num_buildings
,
]
asset
.
extend
(
dmg_by_asset
)
csv_writer
.
writerow
(
asset
)
print
(
"Execution time of the script"
,
(
datetime
.
datetime
.
now
()
-
startTime
))
losscalculator/losslib.py
View file @
2c60a4f5
#!/usr/bin/env python3
# Copyright (c) 2020:
# Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
# Copyright (c) 2020
-2021
:
#
Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment