From 0c085d566a892045c11e2ca7b293ff042466113b Mon Sep 17 00:00:00 2001 From: Cecilia Nievas <cnievas@gfz-potsdam.de> Date: Sat, 21 Jan 2023 19:21:12 +0100 Subject: [PATCH] Renamed to EHRE --- .gitlab-ci.yml => .gitlab-ci.yml_ignore | 0 Makefile | 2 +- README.md | 58 +++++---- docs/01_General_Concepts.md | 16 +-- docs/{02_GDE_Model.md => 02_EHRE_Model.md} | 31 ++--- docs/03_Configuration.md | 27 ++--- docs/04_Processing_Logic.md | 112 +++++++++--------- docs/05_Querying_Results.md | 49 ++++---- docs/README.md | 24 ++-- ...ithm_01.png => ehre_core_algorithm_01.png} | Bin ...gorithm_02_process_data_unit_overview.png} | Bin ... ehre_core_algorithm_03_OBM_buildings.png} | Bin ...rithm_04_OBM_buildings_assign_classes.png} | Bin ...core_algorithm_05_remainder_buildings.png} | Bin ...hm_06_remainder_buildings_calculation.png} | Bin ...l_overview.png => ehre_model_overview.png} | Bin {gdecore => ehrecore}/__init__.py | 0 {gdecore => ehrecore}/configuration.py | 6 +- {gdecore => ehrecore}/database_queries.py | 2 +- {gdecore => ehrecore}/database_storage.py | 2 +- gdecore/gdecore.py => ehrecore/ehrecore.py | 10 +- {gdecore => ehrecore}/occupancy_cases.py | 0 {gdecore => ehrecore}/processor.py | 4 +- setup.py | 10 +- tests/conftest.py | 2 +- tests/test_configuration.py | 2 +- tests/test_database_queries.py | 4 +- tests/test_database_storage.py | 6 +- tests/{test_gdecore.py => test_ehrecore.py} | 0 tests/test_processor.py | 6 +- 30 files changed, 185 insertions(+), 188 deletions(-) rename .gitlab-ci.yml => .gitlab-ci.yml_ignore (100%) rename docs/{02_GDE_Model.md => 02_EHRE_Model.md} (75%) rename docs/images/{gde_core_algorithm_01.png => ehre_core_algorithm_01.png} (100%) rename docs/images/{gde_core_algorithm_02_process_data_unit_overview.png => ehre_core_algorithm_02_process_data_unit_overview.png} (100%) rename docs/images/{gde_core_algorithm_03_OBM_buildings.png => ehre_core_algorithm_03_OBM_buildings.png} (100%) rename docs/images/{gde_core_algorithm_04_OBM_buildings_assign_classes.png => ehre_core_algorithm_04_OBM_buildings_assign_classes.png} (100%) rename docs/images/{gde_core_algorithm_05_remainder_buildings.png => ehre_core_algorithm_05_remainder_buildings.png} (100%) rename docs/images/{gde_core_algorithm_06_remainder_buildings_calculation.png => ehre_core_algorithm_06_remainder_buildings_calculation.png} (100%) rename docs/images/{gde_model_overview.png => ehre_model_overview.png} (100%) rename {gdecore => ehrecore}/__init__.py (100%) rename {gdecore => ehrecore}/configuration.py (98%) rename {gdecore => ehrecore}/database_queries.py (99%) rename {gdecore => ehrecore}/database_storage.py (99%) rename gdecore/gdecore.py => ehrecore/ehrecore.py (96%) rename {gdecore => ehrecore}/occupancy_cases.py (100%) rename {gdecore => ehrecore}/processor.py (99%) rename tests/{test_gdecore.py => test_ehrecore.py} (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml_ignore similarity index 100% rename from .gitlab-ci.yml rename to .gitlab-ci.yml_ignore diff --git a/Makefile b/Makefile index 254c04f..5b9b673 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -SOURCES=gdecore tests setup.py +SOURCES=ehrecore tests setup.py LENGTH=96 check: $(SOURCES) diff --git a/README.md b/README.md index 1734958..b549309 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,19 @@ -# gde-core +# ehre-core -Creates the Global Dynamic Exposure (GDE) model. +Creates the European High-Resolution Exposure (EHRE) model. ## Description -`gde-core` creates the Global Dynamic Exposure (GDE) model by bringing +`ehre-core` creates the European High-Resolution Exposure (EHRE) model by bringing together: - aggregated exposure models distributed on zoom level 18 tiles (output -of [gde-importer](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/gde-importer)), +of [ehre-importer](https://git.gfz-potsdam.de/ehre/ehre-software/ehre-importer)), - OpenBuildingMap (OBM) buildings (output of -[rabotnik-obm](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/rabotnik-obm)), +[rabotnik-obm](https://git.gfz-potsdam.de/ehre/obm-used-by-ehre/rabotnik-obm)), - automatically-calculated completeness of OpenBuildingMap (OBM) (output of -[obmgapanalysis](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/obmgapanalysis)). +[obmgapanalysis](https://git.gfz-potsdam.de/ehre/obm-used-by-ehre/obmgapanalysis)). -## Installing gde-core +## Installing ehre-core ### Software dependencies @@ -24,13 +24,13 @@ of [gde-importer](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposu - `numpy` - `shapely` - `geopandas` -- `gde-importer` +- `ehre-importer` ### Install ```bash -git clone https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/gde-core.git -cd gde-core +git clone https://git.gfz-potsdam.de/ehre/ehre-software/ehre-core.git +cd ehre-core pip3 install -e . ``` @@ -38,26 +38,26 @@ pip3 install -e . ### Run the previous steps -`gde-core` requires that the following software be run in advance for the geographic area and +`ehre-core` requires that the following software be run in advance for the geographic area and input aggregated exposure models of interest: -1. [rabotnik-obm](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/rabotnik-obm): +1. [rabotnik-obm](https://git.gfz-potsdam.de/ehre/obm-used-by-ehre/rabotnik-obm): processing of [OpenStreetMap (OSM)](https://www.openstreetmap.org) data to generate the `OpenBuildingMap` (OBM). -2. [obmgapanalysis](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/obmgapanalysis): +2. [obmgapanalysis](https://git.gfz-potsdam.de/ehre/obm-used-by-ehre/obmgapanalysis): estimation of completeness of `OpenBuildingMap` by means of an automatic comparison between remote-sensing-derived built-up area (from the [Global Human Settlement Layer](https://data.jrc.ec.europa.eu/dataset/jrc-ghsl-10007)) and OBM-derived built-up area. -3. [gde-importer](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/gde-importer): +3. [ehre-importer](https://git.gfz-potsdam.de/ehre/ehre-software/ehre-importer): import of aggregated exposure models and their distribution on zoom level 18 tiles. This requisite implies as well that the database tables that are output of these software exist and are populated with relevant data. The databases associated with these software are: -- [GDE Tiles](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/database-gdetiles) -- [OBM Tiles](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/database-obmtiles) -- [OBM Buildings](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/database-obmbuildings) +- [EHRE Tiles](https://git.gfz-potsdam.de/ehre/databases/database-ehretiles) +- [OBM Tiles](https://git.gfz-potsdam.de/ehre/databases/database-obmtiles) +- [OBM Buildings](https://git.gfz-potsdam.de/ehre/databases/database-obmbuildings) ### Configuration @@ -66,13 +66,11 @@ necessary parameters. Required parameters are: - `model_name`: Name of the input aggregated exposure model to be processed. It needs to have been imported by -[gde-importer](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/gde-importer) -already. +[ehre-importer](https://git.gfz-potsdam.de/ehre/ehre-software/ehre-importer) already. - `exposure_entities_to_run`: List of names of exposure entities for which the code will be run. Currently supported options: - "all": The list of names associated with `model_name` will be retrieved from the - [GDE Tiles](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/database-gdetiles) -database. + [EHRE Tiles](https://git.gfz-potsdam.de/ehre/databases/database-ehretiles) database. - A comma-space-separated list of entity names: This list of names will be used. - A full path to a .txt or .csv file: The list of names will be retrieved from the indicated .txt/.csv file. @@ -82,30 +80,30 @@ and 3-character codes. When running `model_name=esrm20`, "ISO3" is the preferred (comma and space). They need to exist for the indicated `exposure format`. Currently supported values: residential, commercial, industrial. - `database_gde_tiles`: Credentials for the -[GDE Tiles](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/database-gdetiles) -database where information on the GDE tiles is stored. +[EHRE Tiles](https://git.gfz-potsdam.de/ehre/databases/database-ehretiles) database where +information on the EHRE tiles is stored. - `database_obm_buildings`: Credentials for the -[OBM Buildings](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/database-obmbuildings) -database where information on the OBM buildings is stored. +[OBM Buildings](https://git.gfz-potsdam.de/ehre/databases/database-obmbuildings) database where +information on the OBM buildings is stored. - `database_completeness`: Credentials for the -[OBM Tiles](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/database-obmtiles) -database where information on the OSM-completeness of tiles is stored. +[OBM Tiles](https://git.gfz-potsdam.de/ehre/databases/database-obmtiles) database where +information on the OSM-completeness of tiles is stored. - `number_cores`: Number of cores used for parallelisation. Further details on the meaning and use of these parameters can be found in the [documentation](docs/04_Configuration.md). -## Running gde-core +## Running ehre-core From the working directory (where you placed `config.yml`), run the code by typing: ``` -gdecore +ehrecore ``` ## Copyright and copyleft -Copyright (C) 2022 +Copyright (C) 2022-2023 * Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ diff --git a/docs/01_General_Concepts.md b/docs/01_General_Concepts.md index c99b0f3..57da56e 100644 --- a/docs/01_General_Concepts.md +++ b/docs/01_General_Concepts.md @@ -20,12 +20,12 @@ class) or group of buildings (or building classes). Similarly to the case of the calculation, the building classes need to be meaningful in terms of representing the expected behaviour of the buildings and outcome (in terms of losses) when subject to the hazard. -The [Global Dynamic Exposure](02_GDE_Model.md) (GDE) model classifies buildings using the -[GEM Building Taxonomy v3.0](https://github.com/gem/gem_taxonomy) (Silva et al., 2022). The GEM -Building Taxonomy v3.0 is a faceted taxonomy, which means that it characterises buildings by -means of individual relevant attributes, such as construction material, type of lateral -load-resisting system, number of storeys, expected ductility, etc. Building classes arise from -the combination of the different possible values of these attributes. For example: +The [European High-Resolution Exposure](02_EHRE_Model.md) (EHRE) model classifies buildings +using the [GEM Building Taxonomy v3.0](https://github.com/gem/gem_taxonomy) (Silva et al., +2022). The GEM Building Taxonomy v3.0 is a faceted taxonomy, which means that it characterises +buildings by means of individual relevant attributes, such as construction material, type of +lateral load-resisting system, number of storeys, expected ductility, etc. Building classes +arise from the combination of the different possible values of these attributes. For example: - `CR/LFINF+CDL/HBET:2-4` represents a 2- to 4-storey (`HBET:2`) reinforced concrete (`CR`) infilled frame (`LFINF`) with low seismic code design level (`CDL`), @@ -36,5 +36,5 @@ It is not uncommon for aggregated exposure models to be associated with a specif case. Usual occupancy cases include residential, commercial and industrial buildings, for example. Other occupancy cases may be educational, agricultural or governmental buildings, as well as many others. The total number of buildings and distribution of building classes -described above are thus associated with a specific occupancy case in the GDE model (and, -consequently, the `gde-core`). +described above are thus associated with a specific occupancy case in the EHRE model (and, +consequently, the `ehre-core`). diff --git a/docs/02_GDE_Model.md b/docs/02_EHRE_Model.md similarity index 75% rename from docs/02_GDE_Model.md rename to docs/02_EHRE_Model.md index 181e9a6..a1395bf 100644 --- a/docs/02_GDE_Model.md +++ b/docs/02_EHRE_Model.md @@ -1,36 +1,37 @@ -# The Global Dynamic Exposure (GDE) Model +# The European High-Resolution Exposure (EHRE) Model -The Global Dynamic Exposure (GDE) model is a high-resolution exposure model that is generated by -means of combining: +The European High-Resolution Exposure (EHRE) Model is a high-resolution exposure model that is +generated by means of combining: 1. Aggregated exposure models. 2. Data on individual buildings from [OpenStreetMap](https://www.openstreetmap.org). 3. Remote sensing-derived built-up areas from the Global Human Settlement Layer (GHSL; Corbane et al., 2018). -Fig. 2.1 gives an overview of how these elements are processed and combined to create the GDE +Fig. 2.1 gives an overview of how these elements are processed and combined to create the EHRE model. -<img src="images/gde_model_overview.png" width=75%> +<img src="images/ehre_model_overview.png" width=75%> -Fig. 2.1 Conceptual overview of OpenBuildingMap (OBM) and the Global Dynamic Exposure (GDE) model. +Fig. 2.1 Conceptual overview of OpenBuildingMap (OBM) and the European High-Resolution Exposure +(EHRE) Model. Aggregated exposure models (such as that of the European Seismic Risk Model 2020, ESRM20, Crowley et al., 2020) are distributed onto a grid of zoom-level 18 quadtiles proportionally to the built-up area expected in each tile. This task is carried out by the -[gde-importer](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/gde-importer/-/tree/master/), +[ehre-importer](https://git.gfz-potsdam.de/ehre/ehre-software/ehre-importer), which also processes data on building classes, their attributes (e.g. people per building, -replacement cost per building) and other relevant exposure parameters. Currently, `gde-importer` +replacement cost per building) and other relevant exposure parameters. Currently, `ehre-importer` can only import the format of the exposure model of the European Seismic Risk Model 2020 (ESRM20) (Crowley et al., 2020), but further formats can be implemented in the future. The expected built-up areas in each tile are an output of -[obmgapanalysis](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/obmgapanalysis), +[obmgapanalysis](https://git.gfz-potsdam.de/ehre/obm-used-by-ehre/obmgapanalysis), which processes the [Global Human Settlement Layer (GHSL) built-up area layer with multi-temporal analysis and global coverage](https://data.jrc.ec.europa.eu/dataset/jrc-ghsl-10007) (Corbane et al., 2018) for this purpose. Independently from this, -[rabotnik-obm](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/rabotnik-obm) +[rabotnik-obm](https://git.gfz-potsdam.de/ehre/obm-used-by-ehre/rabotnik-obm) retrieves buildings from [OpenStreetMap](https://www.openstreetmap.org) together with some additional attributes, and assigns them occupancy types as per the [GEM Building Taxonomy v3.0](https://github.com/gem/gem_taxonomy) (Silva et al., 2022), the @@ -40,22 +41,22 @@ OBM buildings are assigned building classes as per the aggregated exposure model of their geographic location and their occupancy type. Only occupancy types that belong to occupancy cases covered by the imported aggregated exposure models can be assigned building classes. This is done by the -[gde-core](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/gde-core), as +[ehre-core](https://git.gfz-potsdam.de/ehre/ehre-software/ehre-core), as explained [here](04_Processing_Logic.md#retrieve-obm-buildings-and-assign-building-classes-and-probabilities-to-them). As `OpenStreetMap` is not yet complete everywhere, that is, as not all buildings that exist in reality are represented in `OpenStreetMap`, the final number of buildings in each tile of the -GDE model is calculated by combining numbers of buildings from the aggregated exposure model and +EHRE model is calculated by combining numbers of buildings from the aggregated exposure model and OpenBuildingMap, taking into account the completeness of the tile. The completeness of the tile is determined by -[obmgapanalysis](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/obmgapanalysis) in +[obmgapanalysis](https://git.gfz-potsdam.de/ehre/obm-used-by-ehre/obmgapanalysis) in an automatic fashion, by calculating the ratio between the area of building footprints from `OpenStreetMap` and the expected built-up area as per GHSL, and using a threshold to decide whether the tile is or not complete. The threshold was selected by analysing the built-up ratios obtained for the region of Attica (Greece), for which a manual assessment of completeness was carried out. If a tile is incomplete, -[gde-core](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/gde-core) compares +[ehre-core](https://git.gfz-potsdam.de/ehre/ehre-software/ehre-core) compares the number of buildings in the tile as per the distribution of the aggregated exposure model onto the grid against the number of OBM buildings. If the former is larger than the latter, a number of so-called "remainder" buildings is calculated as the difference between the two and @@ -65,6 +66,6 @@ so-called "aggregated" buildings, no remainder buildings are assigned to the til is complete, no remainder buildings are assigned to the tile either. Further details can be found [here](04_Processing_Logic.md#calculate-remainder-buildings-in-data-unit-tiles). -The resulting GDE model is thus a combination of individual OBM buildings (re-named as GDE +The resulting EHRE model is thus a combination of individual OBM buildings (re-named as EHRE buildings once building classes have been assigned to them) and remainder buildings defined in zoom-level 18 quadtiles. diff --git a/docs/03_Configuration.md b/docs/03_Configuration.md index 3107004..364872f 100644 --- a/docs/03_Configuration.md +++ b/docs/03_Configuration.md @@ -3,23 +3,23 @@ User-configurable parameters need to be provided in a file named `config.yml`, located in the working directory. The file [config_example.yml](../config_example.yml) in this repository can be used as a starting point. Currently all parameters shown in the example file are mandatory to -run `gde-core` (i.e. none of them is optional). +run `ehre-core` (i.e. none of them is optional). ## General parameters - `model_name` (required): Name of the input aggregated exposure model to be processed. It needs to have been imported by -[gde-importer](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/gde-importer) +[ehre-importer](https://git.gfz-potsdam.de/ehre/ehre-software/ehre-importer) already and thus exist in the `aggregated_sources` table of the database specified under `database_gde_tiles` in the configuration file, which needs to follow the structure of the -[GDE Tiles database](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/database-gdetiles). +[EHRE Tiles database](https://git.gfz-potsdam.de/ehre/databases/database-ehretiles). ## Parameters that control what cases are run An input aggregated exposure model may cover different exposure entities and different occupancy -cases. These parameters allow to control which are run when calling `gde-core` (they need to +cases. These parameters allow to control which are run when calling `ehre-core` (they need to have been processed already by -[gde-importer](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/gde-importer)): +[ehre-importer](https://git.gfz-potsdam.de/ehre/ehre-software/ehre-importer)): - `occupancies_to_run` (required): List of occupancies for which the code will be run, separated by ", " (comma and space). They need to exist for the indicated `model_name`. Currently @@ -28,8 +28,7 @@ supported values: residential, commercial, industrial. will be run. Currently supported options: - "all": The list of 3-character codes of all exposure entities associated with `model_name` will be retrieved from the `data_units` table of the - [GDE Tiles](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/database-gdetiles) -database. + [EHRE Tiles](https://git.gfz-potsdam.de/ehre/databases/database-ehretiles) database. - A comma-space-separated list of entity names: This list of names will be transformed into a list of their associated 3-character codes, by means of what is specified in `exposure_entities_code` (see below). @@ -40,19 +39,19 @@ database. ## Parameters needed to access databases - `database_gde_tiles` (required): Credentials for the -[GDE Tiles](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/database-gdetiles) -database where information on the GDE tiles is stored. +[EHRE Tiles](https://git.gfz-potsdam.de/ehre/databases/database-ehretiles) database where +information on the EHRE tiles is stored. - `database_obm_buildings` (required): Credentials for the -[OBM Buildings](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/database-obmbuildings) +[OBM buildings](https://git.gfz-potsdam.de/ehre/databases/database-obmbuildings) database where information on the OBM buildings is stored. - `database_completeness` (required): Credentials for the -[OBM Tiles](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/database-obmtiles) +[OBM tiles](https://git.gfz-potsdam.de/ehre/databases/database-obmtiles) database where information on the OSM-completeness of tiles is stored. Apart from the parameters listed below, the `sourceid` from which the completeness values will be read needs to be indicated within a nested structure. This parameter will be sought for in the `source_id` column of the `obm_built_area_assessments` table of the `OBM Tiles` database. -The `gde-core` assumes that these databases contain the tables indicated in each corresponding +The `ehre-core` assumes that these databases contain the tables indicated in each corresponding link. In order to connect to the databases, some or all of the following parameters may be required (as nested elements of `database_gde_tiles`, `database_obm_buildings` and `database_completeness`): @@ -69,11 +68,11 @@ under `exposure_entities_to_run` get converted into their 3-character codes. The which `exposure_entities_code` is not used is when `exposure_entities_to_run` is "all", as the 3-character codes get directly retrieved from the database in this case. In all other cases, the user will be specifying names (e.g. "Greece", "United_Kingdom") and: - - if `exposure_entities_code` is "ISO3", the `gde-core` will retrieve the corresponding + - if `exposure_entities_code` is "ISO3", the `ehre-core` will retrieve the corresponding alpha-3 [ISO 3166 country codes](https://www.iso.org/iso-3166-country-codes.html), using the [iso3166 library](https://github.com/deactivated/python-iso3166) (this only works if the listed exposure entities are countries); - - if `exposure_entities_code` is a dictionary, the `gde-core` will seek for the name of the + - if `exposure_entities_code` is a dictionary, the `ehre-core` will seek for the name of the exposure entities within the keys of the dictionary and take as codes the values associated with those keys. - `number_cores` (required): Number of cores used for parallelisation. diff --git a/docs/04_Processing_Logic.md b/docs/04_Processing_Logic.md index 97c0453..146eb38 100644 --- a/docs/04_Processing_Logic.md +++ b/docs/04_Processing_Logic.md @@ -1,21 +1,21 @@ # Processing Logic -When running `gdecore`, it is the `main()` function in -[gdecore.py](../gdecore/gdecore.py) that is called and the program is set to run. -The processing logic described herein is the one adopted at present for the `gde-core`, but +When running `ehrecore`, it is the `main()` function in +[ehrecore.py](../ehrecore/ehrecore.py) that is called and the program is set to run. +The processing logic described herein is the one adopted at present for the `ehre-core`, but the modularity of the code allows for it to be easily reorganised or modified in the future if needed. In other words, there are aspects of this processing logic that do not need to follow the order in which they are currently carried out. -For details on the geographic units used by `gde-core` (e.g. "exposure entity", "data unit", -"data-unit tile"), please refer to the documentation of the `gde-importer` -[here](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/gde-importer/-/blob/master/docs/02_Organisation_Geographic_Space.md). +For details on the geographic units used by `ehre-core` (e.g. "exposure entity", "data unit", +"data-unit tile"), please refer to the documentation of the `ehre-importer` +[here](https://git.gfz-potsdam.de/ehre/ehre-software/ehre-importer/-/blob/master/docs/02_Organisation_Geographic_Space.md). ## Reading the Configuration File -The first task carried out by the `gde-core` is reading and interpreting the +The first task carried out by the `ehre-core` is reading and interpreting the [configuration file](03_Configuration.md). This is done by means of the -[Configuration](../gdecore/configuration.py#L29) class, which verifies that all the +[Configuration](../ehrecore/configuration.py#L29) class, which verifies that all the required (i.e. non-optional) parameters are available and that their values/types are as expected by the program. If a problem is found that does not allow the program to run, it will raise an error and stop. @@ -25,14 +25,14 @@ raise an error and stop. The `model_name` of the aggregated exposure model specified by the user in the configuration file is sought for in the `aggregated_sources` table of the database specified under `database_gde_tiles` in the configuration file, which needs to follow the structure of the -[GDE Tiles database](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/database-gdetiles). +[EHRE Tiles database](https://git.gfz-potsdam.de/ehre/databases/database-ehretiles). If `model_name` is not found, the program stops running. If found, its ID (`aggregated_source_id`) and format (`aggregated_source_format`) are retrieved. ## Initialise Occupancy Cases Class The format of the aggregated exposure model is relevant to initialise the correct occupancy -cases sub-class of `OccupancyCases` from [occupancy_cases.py](../gdecore/occupancy_cases.py). +cases sub-class of `OccupancyCases` from [occupancy_cases.py](../ehrecore/occupancy_cases.py). The `OccupancyCases` class represents the grouping of occupancy types (as per the GEM Building Taxonomy v3.0) into occupancy cases. Occupancy cases are broader groups of occupancy types and are relevant to connect individual OBM buildings with aggregated exposure models, as the latter @@ -42,20 +42,20 @@ for "COM"), "COM1", "COM2", "COM3", "COM5" and "RES3" occupancy types. ## Interpreting the Exposure Entities to Run -The `gde-core` offers different ways in which the user can specify which exposure entity/ies to +The `ehre-core` offers different ways in which the user can specify which exposure entity/ies to run (see documentation on the [configuration file](03_Configuration.md)). The -`interpret_exposure_entities_to_run` method of the [Configuration](../gdecore/configuration.py) +`interpret_exposure_entities_to_run` method of the [Configuration](../ehrecore/configuration.py) class is called to interpret the input provided by the user and update the `exposure_entities_to_run` attribute of the configuration object accordingly. -If the user specifies to run `all` exposure entities, `gde-core` retrieves the list of +If the user specifies to run `all` exposure entities, `ehre-core` retrieves the list of 3-character codes of all exposure entities associated with `aggregated_source_id` in the -[GDE Tiles database](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/database-gdetiles). +[EHRE Tiles database](https://git.gfz-potsdam.de/ehre/databases/database-ehretiles). -If the user specifies a path to a `.txt` or `.csv`, `gde-core` retrieves the list of 3-character +If the user specifies a path to a `.txt` or `.csv`, `ehre-core` retrieves the list of 3-character codes of the exposure entities listed in the indicated `.txt`/`.csv` file. -If the user specifies a list with one or more names of exposure entities, `gde-core` retrieves +If the user specifies a list with one or more names of exposure entities, `ehre-core` retrieves the 3-character codes associated with these exposure entities either from a nested structure that the user indicates in the configuration file or by retrieving the corresponding alpha-3 [ISO 3166 country codes](https://www.iso.org/iso-3166-country-codes.html), using the @@ -72,27 +72,27 @@ attribute of the Configuration class. All data unit IDs associated with the exposure entity, occupancy case and aggregated source ID are retrieved, together with their geometries, from the `data_units` table of the -[GDE Tiles database](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/database-gdetiles). +[EHRE Tiles database](https://git.gfz-potsdam.de/ehre/databases/database-ehretiles). Data units for which no geometry is available (which should not occur unless an unexpected error -occurred while running the `gde-importer`) are logged with a warning, as they cannot be +occurred while running the `ehre-importer`) are logged with a warning, as they cannot be processed. If no data units with geometries are retrieved (i.e. they don't exist in the database), a warning is logged and the rest of the code is not run for this combination of exposure entity and occupancy case (i.e. the code moves on to the next occupancy case). -<img src="images/gde_core_algorithm_01.png" width=75%> +<img src="images/ehre_core_algorithm_01.png" width=75%> Fig. 4.1 Flowchart showing the processing logic at the highest level. -### Deleting Existing Entries from GDE Buildings Database Table +### Deleting Existing Entries from EHRE Buildings Database Table The `delete_old_database_entries` method of the `DatabaseStorage` class is called to delete all entries associated with the exposure entity and occupancy case (and aggregated_source_id) being -run, from the `gde_buildings` table of the -[GDE Tiles database](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/database-gdetiles). +run, from the `ehre_buildings` table of the +[EHRE Tiles database](https://git.gfz-potsdam.de/ehre/databases/database-ehretiles). -This step is relevant because, even though the method that writes data to the `gde_buildings` +This step is relevant because, even though the method that writes data to the `ehre_buildings` table does delete existing entries for a specific combination of `data_unit_id`, `occupacy_case` and `aggregated_source_id` before proceeding to write the new entries, the actual data unit IDs associated with an exposure entity may change for different versions of the aggregated exposure @@ -106,13 +106,13 @@ Note that if no new data unit IDs with geometries are identified in the previous ### Process Data Units in Parallel Data units are processed using the `process_data_unit` method of the -[GDEProcessor](../gdecore/processor.py#L34) class. Each data unit is processed by a core, using +[GDEProcessor](../ehrecore/processor.py#L34) class. Each data unit is processed by a core, using as many cores as specified by the user in the configuration file. Each of the subtitles that follow refer to a task carried out or called by the `process_data_unit` method, as shown in Fig. 4.2. -<img src="images/gde_core_algorithm_02_process_data_unit_overview.png" width=75%> +<img src="images/ehre_core_algorithm_02_process_data_unit_overview.png" width=75%> Fig. 4.2 Overview of the main tasks carried out by `GDEProcessor.process_data_unit()`. @@ -120,7 +120,7 @@ Fig. 4.2 Overview of the main tasks carried out by `GDEProcessor.process_data_un The building classes associated with the data unit ID are retrieved from the `data_units_buildings` table of the -[GDE Tiles database](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/database-gdetiles). +[EHRE Tiles database](https://git.gfz-potsdam.de/ehre/databases/database-ehretiles). For each building class, the proportion of buildings it represents within the data unit and the minimum and maximum number of storeys it covers are retrieved. @@ -135,14 +135,14 @@ As show in Fig. 4.3, the following tasks are carried out in order to assign buil all OpenBuildingMap (OBM) buildings that belong to the data unit and occupancy case being processed: -<img src="images/gde_core_algorithm_03_OBM_buildings.png" width=75%> +<img src="images/ehre_core_algorithm_03_OBM_buildings.png" width=75%> Fig. 4.3 Flowchart of tasks associated with retrieving OBM buildings and assigning building classes to them. 1. **Retrieve OBM buildings**: The `get_OBM_buildings_in_data_unit_by_occupancy_types` method of `DatabaseQueries` retrieves from the -[OBM buildings database](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/database-obmbuildings) +[OBM buildings database](https://git.gfz-potsdam.de/ehre/databases/database-obmbuildings) all OBM building parts whose centroids fall within the boundaries of the data unit and whose occupancy type belongs to the occupancy case being processed (see [here](#initialise-occupancy-cases-class)). @@ -161,59 +161,59 @@ incompatible with the indicated number are discarded. If the building is commerc "COM1", "COM2", "COM3", "COM5" or "RES3", only building classes associated with these types are kept. This stems from the fact the ESRM20 exposure model (Crowley et al., 2020) has different sub-types of commercial classes and the logic might need revision when other aggregated exposure -models are incorporated to the GDE model in the future. If discarding potential building classes +models are incorporated to the EHRE model in the future. If discarding potential building classes as per these criteria lead to no building class being compatible with the building, a warning is logged and all initial building classes are assigned to the building. This is done because assigning no building classes to a building leads to the impossibility of considering the building within a damage/loss assessment. -5. **Store building classes of GDE (OBM) buildings**: At this stage, OBM buildings that have -been assigned building classes become so-called GDE buildings. Their details are stored in the -`gde_buildings` table of the -[GDE Tiles database](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/database-gdetiles). +5. **Store building classes of EHRE (OBM) buildings**: At this stage, OBM buildings that have +been assigned building classes become so-called EHRE buildings. Their details are stored in the +`ehre_buildings` table of the +[EHRE Tiles database](https://git.gfz-potsdam.de/ehre/databases/database-ehretiles). -<img src="images/gde_core_algorithm_04_OBM_buildings_assign_classes.png" width=75%> +<img src="images/ehre_core_algorithm_04_OBM_buildings_assign_classes.png" width=75%> Fig. 4.4 Algorithm used to assign building classes to individual OBM buildings. Step number 2 should become part of -[rabotnik-obm](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/rabotnik-obm/-/tree/master) +[rabotnik-obm](https://git.gfz-potsdam.de/ehre/obm-used-by-ehre/rabotnik-obm) in the future. As explained -[here](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/rabotnik-obm/-/blob/master/docs/general_overview.md), +[here](https://git.gfz-potsdam.de/ehre/obm-used-by-ehre/rabotnik-obm/-/blob/master/docs/general_overview.md), `rabotnik-obm` currently processes the IDs of individual building polygons without gathering together building parts that make up a relation in [OpenStreetMap](https://www.openstreetmap.org) (OSM). Relations are often used in OSM to represent complex building geometries, like vertical irregularities (e.g. a building whose plan comprises partly 3 storeys and partly 10 storeys). If in the future -[rabotnik-obm](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/rabotnik-obm/-/tree/master) +[rabotnik-obm](https://git.gfz-potsdam.de/ehre/obm-used-by-ehre/rabotnik-obm) were to gather parts of a building relation, the method `get_OBM_buildings_in_data_unit_by_occupancy_types` of -[DatabaseQueries](../gdecore/database_queries.py#L30) would directly return what is now being +[DatabaseQueries](../ehrecore/database_queries.py#L30) would directly return what is now being returned by the combination of running `get_OBM_buildings_in_data_unit_by_occupancy_types` -followed by `post_process_obm_relations` of [GDEProcessor](../gdecore/processor.py#L34), and the +followed by `post_process_obm_relations` of [GDEProcessor](../ehrecore/processor.py#L34), and the latter (i.e. step 2) would thus no longer be needed. -Some limitations associated with the fact that `gde-core` is attempting to gather parts of the +Some limitations associated with the fact that `ehre-core` is attempting to gather parts of the same building relation (instead of having this done by `rabotnik-obm`) are the following: -- `gde-core` queries the -[OBM buildings database](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/database-obmbuildings) +- `ehre-core` queries the +[OBM buildings database](https://git.gfz-potsdam.de/ehre/databases/database-obmbuildings) by data unit, which means that if some parts of a relation fall in one data unit and some others fall in another, this building will be counted twice, once in each data unit. Results from the -GDE model for Greece indicate this happens just once for the whole country. -- `gde-core` queries the -[OBM buildings database](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/database-obmbuildings) +EHRE model for Greece indicate this happens just once for the whole country. +- `ehre-core` queries the +[OBM buildings database](https://git.gfz-potsdam.de/ehre/databases/database-obmbuildings) by occupancy case, which means that if some parts of a relation have occupancy types that belong to a different occupancy case (or to no occupancy case at all), they will not be drawn together. - `rabotnik-obm` only retrieves the first relation ID it finds for each building part, but a building might be represented in OpenStreetMap by a nested series of relations, as explained -[here](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/rabotnik-obm/-/blob/master/docs/general_overview.md). +[here](https://git.gfz-potsdam.de/ehre/obm-used-by-ehre/rabotnik-obm/-/blob/master/docs/general_overview.md). -Trying to solve these limitations within `gde-core` would incur in dupplicating efforts with +Trying to solve these limitations within `ehre-core` would incur in dupplicating efforts with respect to `rabotnik-obm`, which is not desirable. For example, bringing together different occupancy types coming from parts of a relation should follow the same processing logic that is already being used by `rabotnik-obm` to define a final occupancy type based on different types stemming from different tags of the same building (details -[here](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/rabotnik-obm/-/blob/master/docs/occupancy.md)). -The minimum has been done within `gde-core` to minimise the impact of buildings being +[here](https://git.gfz-potsdam.de/ehre/obm-used-by-ehre/rabotnik-obm/-/blob/master/docs/occupancy.md)). +The minimum has been done within `ehre-core` to minimise the impact of buildings being represented by relation IDs (i.e. step 2 above is this minimum). Out of over 30 million buildings processed so far (at the time of writing, end of June 2022) by `rabotnik-obm`, only 0.07% belong to a relation (these buildings belong to Greece, Italy and part of Germany). @@ -223,22 +223,22 @@ buildings processed so far (at the time of writing, end of June 2022) by `rabotn As shown in Fig. 4.5, the following tasks are carried out in order to calculate the number of remainder buildings in each data-unit tile: -<img src="images/gde_core_algorithm_05_remainder_buildings.png" width=75%> +<img src="images/ehre_core_algorithm_05_remainder_buildings.png" width=75%> Fig. 4.5 Algorithm used to process data-unit tiles. 1. **Retrieve data-unit tiles**: The data-unit tiles associated with the data unit being processed are retrieved from the `data_unit_tiles` table of the -[GDE Tiles database](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/database-gdetiles), +[EHRE Tiles database](https://git.gfz-potsdam.de/ehre/databases/database-ehretiles), in terms of their quadkeys and the number of aggregated buildings associated with them (i.e. the -number of buildings assigned to the tile by `gde-importer` when distributing the aggregated +number of buildings assigned to the tile by `ehre-importer` when distributing the aggregated exposure model onto the tiles). 2. **Retrieve completeness of data-unit tiles**: The completeness status of the data-unit tiles is retrieved from the `obm_built_area_assessments` table of the -[OBM tiles database](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/database-obmtiles) +[OBM tiles database](https://git.gfz-potsdam.de/ehre/databases/database-obmtiles) (entries associated with `source_id=1` or `source_id=3`, depending on the country, whih correspond to the Global Human Settlement Layer GHSL, Corbane et al., 2018). As -[obmgapanalysis](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/obmgapanalysis) +[obmgapanalysis](https://git.gfz-potsdam.de/ehre/obm-used-by-ehre/obmgapanalysis) does not create entries for tiles for which the built-up area is zero, if a quadkey does not have an entry for `source_id=1`/`source_id=3`, it is assumed to be complete. If the entry exists, the completeness status is retrieved from the database. If the entry exists but the @@ -264,10 +264,10 @@ assigned to the tile. If the number of OBM buildings is larger than that of aggr buildings, zero remainder buildings are assigned to the tile. This algorithm is shown in Fig. 4.6. -<img src="images/gde_core_algorithm_06_remainder_buildings_calculation.png" width=75%> +<img src="images/ehre_core_algorithm_06_remainder_buildings_calculation.png" width=75%> Fig. 4.6 Algorithm used to calculate remainder buildings. 4. **Store number of OBM and remainder buildings of the data-unit tiles**: The number of OBM and remainder buildings in the data-unit tile are stored in the `data_unit_tiles` table of the -[GDE Tiles database](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/database-gdetiles). +[EHRE Tiles database](https://git.gfz-potsdam.de/ehre/databases/database-ehretiles). diff --git a/docs/05_Querying_Results.md b/docs/05_Querying_Results.md index 30bef7e..681e848 100644 --- a/docs/05_Querying_Results.md +++ b/docs/05_Querying_Results.md @@ -1,18 +1,17 @@ # Querying Results -Example queries that allow to access data from the GDE model are shown herein. Unless stated +Example queries that allow to access data from the EHRE model are shown herein. Unless stated otherwise, all queries refer to the -[GDE Tiles database](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/database-gdetiles). +[EHRE tiles](https://git.gfz-potsdam.de/ehre/databases/database-ehretiles). ## Available aggregated sources To list all available aggregated sources, that is, aggregated exposure models that have been imported by the -[gde-importer](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/gde-importer), -do: +[ehre-importer](https://git.gfz-potsdam.de/ehre/ehre-software/ehre-importer), do: ``` -gde_tiles=> SELECT aggregated_source_id, name, format FROM aggregated_sources; +ehre_tiles=> SELECT aggregated_source_id, name, format FROM aggregated_sources; ``` ## Total number of buildings of an exposure entity, grouped by occupancy case @@ -22,7 +21,7 @@ remainder buildings in Greece (`GRC`), as per `aggregated_source_id=1` (i.e., ES by each occupancy case: ``` -gde_tiles=> SELECT occupancy_case, SUM(aggregated_buildings), SUM(obm_buildings), SUM(remainder_buildings) FROM data_unit_tiles WHERE exposure_entity='GRC' AND aggregated_source_id=1 GROUP BY occupancy_case; +ehre_tiles=> SELECT occupancy_case, SUM(aggregated_buildings), SUM(obm_buildings), SUM(remainder_buildings) FROM data_unit_tiles WHERE exposure_entity='GRC' AND aggregated_source_id=1 GROUP BY occupancy_case; occupancy_case | sum | sum | sum ----------------+--------------------+--------+-------------------- @@ -35,8 +34,8 @@ gde_tiles=> SELECT occupancy_case, SUM(aggregated_buildings), SUM(obm_buildings) The summation of all OBM buildings and all remainder buildings gives the total number of buildings in Greece for the three occupancy cases. This does not mean that these are all the OBM buildings that exist in Greece in the -[OBM buildings database](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/database-obmbuildings), -as many of those correspond to occupancy cases not covered by ESRM20 (and, thus, by GDE). +[OBM buildings database](https://git.gfz-potsdam.de/ehre/databases/database-obmbuildings), +as many of those correspond to occupancy cases not covered by ESRM20 (and, thus, by EHRE). ## Number of tiles associated with an exposure entity @@ -44,7 +43,7 @@ The following example query returns the number of zoom-level 18 tiles associated (`LUX`), as per `aggregated_source_id=1` (i.e., ESRM20): ``` -gde_tiles=> SELECT COUNT(DISTINCT(quadkey)) FROM data_unit_tiles WHERE exposure_entity='LUX' AND aggregated_source_id=1; +ehre_tiles=> SELECT COUNT(DISTINCT(quadkey)) FROM data_unit_tiles WHERE exposure_entity='LUX' AND aggregated_source_id=1; count -------- @@ -65,7 +64,7 @@ entry, the data unit ID, occupancy case and numbers of aggregated, OBM and remai are requested. ``` -gde_tiles=> SELECT data_unit_id, occupancy_case, aggregated_buildings, obm_buildings, remainder_buildings FROM data_unit_tiles WHERE quadkey='122100203132021122' AND aggregated_source_id=1; +ehre_tiles=> SELECT data_unit_id, occupancy_case, aggregated_buildings, obm_buildings, remainder_buildings FROM data_unit_tiles WHERE quadkey='122100203132021122' AND aggregated_source_id=1; data_unit_id | occupancy_case | aggregated_buildings | obm_buildings | remainder_buildings ---------------------------+----------------+----------------------+---------------+--------------------- @@ -79,9 +78,9 @@ gde_tiles=> SELECT data_unit_id, occupancy_case, aggregated_buildings, obm_build This result says that both for residential and commercial exposure this tile is intersected by the boundary between data units `GRC_3514508` and `GRC_3514608` of Greece (`GRC`), but is fully -contained in the filler data-unit created for industrial exposure by the `gde-importer` to +contained in the filler data-unit created for industrial exposure by the `ehre-importer` to ensure full geographic coverage (see more details -[here](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/gde-importer/-/blob/master/docs/06_Ensuring_Full_Geographic_Coverage.md)). +[here](https://git.gfz-potsdam.de/ehre/ehre-software/ehre-importer/-/blob/master/docs/06_Ensuring_Full_Geographic_Coverage.md)). ## Building classes associated with a data unit ID @@ -90,7 +89,7 @@ classes associated with `GRC_3514508`, as per `aggregated_source_id=1` (i.e., ES following query could be run: ``` -gde_tiles=> SELECT building_class_name, settlement_type, occupancy_subtype, proportions FROM data_units_buildings WHERE data_unit_id='GRC_3514508' AND occupancy_case='residential' AND aggregated_source_id=1; +ehre_tiles=> SELECT building_class_name, settlement_type, occupancy_subtype, proportions FROM data_units_buildings WHERE data_unit_id='GRC_3514508' AND occupancy_case='residential' AND aggregated_source_id=1; building_class_name | settlement_type | occupancy_subtype | proportions ------------------------------------+-----------------+-------------------+------------------------ @@ -105,7 +104,7 @@ gde_tiles=> SELECT building_class_name, settlement_type, occupancy_subtype, prop It is possible to check that all these proportions add up to unity by doing: ``` -gde_tiles=> SELECT SUM(proportions) FROM data_units_buildings WHERE data_unit_id='GRC_3514508' AND occupancy_case='residential' AND aggregated_source_id=1; +ehre_tiles=> SELECT SUM(proportions) FROM data_units_buildings WHERE data_unit_id='GRC_3514508' AND occupancy_case='residential' AND aggregated_source_id=1; sum -------------------- @@ -119,7 +118,7 @@ Taking one of the building classes from the output to the query above, it is pos retrieve the census people and total replacement cost per building by doing: ``` -gde_tiles=> SELECT census_people_per_building, total_cost_per_building FROM data_units_buildings WHERE building_class_name='CR/LDUAL+CDH+LFC:15.0/HBET:3-5' AND settlement_type='urban' AND occupancy_subtype='ALL' AND occupancy_case='residential' AND data_unit_id='GRC_3514508' AND aggregated_source_id=1; +ehre_tiles=> SELECT census_people_per_building, total_cost_per_building FROM data_units_buildings WHERE building_class_name='CR/LDUAL+CDH+LFC:15.0/HBET:3-5' AND settlement_type='urban' AND occupancy_subtype='ALL' AND occupancy_case='residential' AND data_unit_id='GRC_3514508' AND aggregated_source_id=1; census_people_per_building | total_cost_per_building ----------------------------+------------------------- 7.905868678536067 | 303187.50000000006 @@ -130,7 +129,7 @@ Note that not specifying the data unit ID would lead to more than one entry bein in this other example: ``` -gde_tiles=> SELECT data_unit_id, census_people_per_building, total_cost_per_building FROM data_units_buildings WHERE building_class_name='CR/LDUAL+CDH+LFC:15.0/HBET:3-5' AND settlement_type='urban' AND occupancy_subtype='ALL' AND occupancy_case='residential' AND aggregated_source_id=1; +ehre_tiles=> SELECT data_unit_id, census_people_per_building, total_cost_per_building FROM data_units_buildings WHERE building_class_name='CR/LDUAL+CDH+LFC:15.0/HBET:3-5' AND settlement_type='urban' AND occupancy_subtype='ALL' AND occupancy_case='residential' AND aggregated_source_id=1; data_unit_id | census_people_per_building | total_cost_per_building --------------+----------------------------+------------------------- @@ -151,7 +150,7 @@ corresponding exposure entity (`GRC`), occupancy case (`residential`) and aggreg (`1`): ``` -gde_tiles=> SELECT day, night, transit FROM exposure_entities_population_time_distribution WHERE exposure_entity='GRC' AND occupancy_case='residential' AND aggregated_source_id=1; +ehre_tiles=> SELECT day, night, transit FROM exposure_entities_population_time_distribution WHERE exposure_entity='GRC' AND occupancy_case='residential' AND aggregated_source_id=1; day | night | transit -----------+----------+---------- @@ -172,7 +171,7 @@ first retrieve the respective coefficients for the corresponding exposure entity occupancy case (`residential`) and aggregated source ID (`1`): ``` -gde_tiles=> SELECT structural, non_structural, contents, currency FROM exposure_entities_costs_assumptions WHERE exposure_entity='GRC' AND occupancy_case='residential' AND aggregated_source_id=1; +ehre_tiles=> SELECT structural, non_structural, contents, currency FROM exposure_entities_costs_assumptions WHERE exposure_entity='GRC' AND occupancy_case='residential' AND aggregated_source_id=1; structural | non_structural | contents | currency ------------+----------------+----------+---------- 0.3 | 0.5 | 0.2 | EUR 2020 @@ -189,14 +188,15 @@ As indicated in the output from the query, these values correspond to euros of t ## Building classes associated with an OBM building (defined by its OSM ID) In order to retrieve the building classes associated with an OBM building one can query the -`gde_buildings` table by `osm_id` and `aggregated_source_id`, as shown in the example below. The +`ehre_buildings` table by `osm_id` and `aggregated_source_id`, as shown in the example below. The +`ehre_buildings` table by `osm_id` and `aggregated_source_id`, as shown in the example below. The query is split into three to facilitate the visualisation of the results. The actual OSM ID that this query corresponds to is anonymised as XXXXXXXXX. Querying first just for building class names and their probabilities: ``` -gde_tiles=> SELECT building_class_names, probabilities FROM gde_buildings WHERE osm_id=XXXXXXXXX AND aggregated_source_id=1; +ehre_tiles=> SELECT building_class_names, probabilities FROM ehre_buildings WHERE osm_id=XXXXXXXXX AND aggregated_source_id=1; building_class_names | probabilities ---------------------------------------------------------------------------------------------------------------------------+----------------------- @@ -208,7 +208,7 @@ Querying then for their settlement types and occupancy sub-types it is noted tha above are associated with "all" settlement types and the "Hotels" occupancy subtype: ``` -gde_tiles=> SELECT settlement_types, occupancy_subtypes FROM gde_buildings WHERE osm_id=XXXXXXXXX AND aggregated_source_id=1; +ehre_tiles=> SELECT settlement_types, occupancy_subtypes FROM ehre_buildings WHERE osm_id=XXXXXXXXX AND aggregated_source_id=1; settlement_types | occupancy_subtypes -------------------+------------------------------- @@ -220,15 +220,14 @@ Additionaly, one can know the ID of the data unit and the occupancy case this bu to: ``` -gde_tiles=> SELECT data_unit_id, occupancy_case FROM gde_buildings WHERE osm_id=XXXXXXXXX AND aggregated_source_id=1; +ehre_tiles=> SELECT data_unit_id, occupancy_case FROM ehre_buildings WHERE osm_id=XXXXXXXXX AND aggregated_source_id=1; data_unit_id | occupancy_case --------------+---------------- GRC_4626901 | commercial (1 row) ``` -Querying the -[OBM buildings](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/database-obmbuildings) +Querying the [OBM buildings](https://git.gfz-potsdam.de/ehre/databases/database-obmbuildings) database: ``` @@ -242,4 +241,4 @@ obm_buildings=> SELECT storeys, occupancy FROM obm_buildings WHERE osm_id=XXXXXX As can be observed, only building classes associated with the occupancy type `RES3` (i.e. hotels) and compatible with 7 storeys (i.e. `HBET:6-`) have been assigned to this building by -`gde-core`. +`ehre-core`. diff --git a/docs/README.md b/docs/README.md index 9599cb5..970d9bf 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,18 +1,18 @@ -# Documentation for `gde-core` +# Documentation for `ehre-core` -The `gde-core` is a software that combines the output of -[gde-importer](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/gde-importer/-/tree/master/), -[rabotnik-obm](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/rabotnik-obm) and -[obmgapanalysis](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/obmgapanalysis) to -generate the Global Dynamic Exposure (GDE) model. This output is retrieved from the -[GDE tiles](https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/database-gdetiles), -[OBM buildings](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/database-obmbuildings) +The `ehre-core` is a software that combines the output of +[ehre-importer](https://git.gfz-potsdam.de/ehre/ehre-software/ehre-importer), +[rabotnik-obm](https://git.gfz-potsdam.de/ehre/obm-used-by-ehre/rabotnik-obm) and +[obmgapanalysis](https://git.gfz-potsdam.de/ehre/obm-used-by-ehre/obmgapanalysis) to +generate the European High-Resolution Exposure (EHRE) model. This output is retrieved from the +[EHRE tiles](https://git.gfz-potsdam.de/ehre/databases/database-ehretiles), +[OBM buildings](https://git.gfz-potsdam.de/ehre/databases/database-obmbuildings) and -[OBM tiles](https://git.gfz-potsdam.de/dynamicexposure/openbuildingmap/database-obmtiles) +[OBM tiles](https://git.gfz-potsdam.de/ehre/databases/database-obmtiles) databases to which each of the aforementioned software store their output. In broad terms, each provides the following: -- `gde-importer`: aggregated exposure models distributed onto zoom level 18 tiles in terms of +- `ehre-importer`: aggregated exposure models distributed onto zoom level 18 tiles in terms of numbers of buildings, their building classes as well as attributes of the building classes such as the number of people and replacement cost per building; - `rabotnik-obm`: individual buildings, retrieved from @@ -25,7 +25,7 @@ representation on the map) and in which it is incomplete. ## Contents 1. [General concepts](01_General_Concepts.md) -2. [The Global Dynamic Exposure model](02_GDE_Model.md) +2. [The European High-Resolution Exposure model](02_EHRE_Model.md) 3. [Configuration](03_Configuration.md) 4. [Processing logic](04_Processing_Logic.md) 5. [Querying results](05_Querying_Results.md) @@ -33,7 +33,7 @@ representation on the map) and in which it is incomplete. # Installation and running -For instructions on how to install and run the `gdecore`, please refer to the main +For instructions on how to install and run the `ehrecore`, please refer to the main [README](../README.md). # Copyright and copyleft diff --git a/docs/images/gde_core_algorithm_01.png b/docs/images/ehre_core_algorithm_01.png similarity index 100% rename from docs/images/gde_core_algorithm_01.png rename to docs/images/ehre_core_algorithm_01.png diff --git a/docs/images/gde_core_algorithm_02_process_data_unit_overview.png b/docs/images/ehre_core_algorithm_02_process_data_unit_overview.png similarity index 100% rename from docs/images/gde_core_algorithm_02_process_data_unit_overview.png rename to docs/images/ehre_core_algorithm_02_process_data_unit_overview.png diff --git a/docs/images/gde_core_algorithm_03_OBM_buildings.png b/docs/images/ehre_core_algorithm_03_OBM_buildings.png similarity index 100% rename from docs/images/gde_core_algorithm_03_OBM_buildings.png rename to docs/images/ehre_core_algorithm_03_OBM_buildings.png diff --git a/docs/images/gde_core_algorithm_04_OBM_buildings_assign_classes.png b/docs/images/ehre_core_algorithm_04_OBM_buildings_assign_classes.png similarity index 100% rename from docs/images/gde_core_algorithm_04_OBM_buildings_assign_classes.png rename to docs/images/ehre_core_algorithm_04_OBM_buildings_assign_classes.png diff --git a/docs/images/gde_core_algorithm_05_remainder_buildings.png b/docs/images/ehre_core_algorithm_05_remainder_buildings.png similarity index 100% rename from docs/images/gde_core_algorithm_05_remainder_buildings.png rename to docs/images/ehre_core_algorithm_05_remainder_buildings.png diff --git a/docs/images/gde_core_algorithm_06_remainder_buildings_calculation.png b/docs/images/ehre_core_algorithm_06_remainder_buildings_calculation.png similarity index 100% rename from docs/images/gde_core_algorithm_06_remainder_buildings_calculation.png rename to docs/images/ehre_core_algorithm_06_remainder_buildings_calculation.png diff --git a/docs/images/gde_model_overview.png b/docs/images/ehre_model_overview.png similarity index 100% rename from docs/images/gde_model_overview.png rename to docs/images/ehre_model_overview.png diff --git a/gdecore/__init__.py b/ehrecore/__init__.py similarity index 100% rename from gdecore/__init__.py rename to ehrecore/__init__.py diff --git a/gdecore/configuration.py b/ehrecore/configuration.py similarity index 98% rename from gdecore/configuration.py rename to ehrecore/configuration.py index 01352da..4a00669 100644 --- a/gdecore/configuration.py +++ b/ehrecore/configuration.py @@ -19,9 +19,9 @@ import sys import logging from copy import deepcopy -from gdeimporter.tools.configuration_methods import ConfigurationMethods -from gdeimporter.exposureentity import ExposureEntity -from gdecore.database_queries import DatabaseQueries +from ehreimporter.tools.configuration_methods import ConfigurationMethods +from ehreimporter.exposureentity import ExposureEntity +from ehrecore.database_queries import DatabaseQueries logger = logging.getLogger() diff --git a/gdecore/database_queries.py b/ehrecore/database_queries.py similarity index 99% rename from gdecore/database_queries.py rename to ehrecore/database_queries.py index bf4ebce..6448ef8 100644 --- a/gdecore/database_queries.py +++ b/ehrecore/database_queries.py @@ -22,7 +22,7 @@ import shapely import pandas import geopandas import warnings -from gdeimporter.tools.database import Database +from ehreimporter.tools.database import Database logger = logging.getLogger() diff --git a/gdecore/database_storage.py b/ehrecore/database_storage.py similarity index 99% rename from gdecore/database_storage.py rename to ehrecore/database_storage.py index fcc1212..f5d54dd 100644 --- a/gdecore/database_storage.py +++ b/ehrecore/database_storage.py @@ -17,7 +17,7 @@ # along with this program. If not, see http://www.gnu.org/licenses/. import logging -from gdeimporter.tools.database import Database +from ehreimporter.tools.database import Database logger = logging.getLogger() diff --git a/gdecore/gdecore.py b/ehrecore/ehrecore.py similarity index 96% rename from gdecore/gdecore.py rename to ehrecore/ehrecore.py index cb8e807..121f624 100644 --- a/gdecore/gdecore.py +++ b/ehrecore/ehrecore.py @@ -20,11 +20,11 @@ import logging import sys from multiprocessing import Pool from functools import partial -from gdecore.configuration import Configuration -from gdecore.database_queries import DatabaseQueries -from gdecore.database_storage import DatabaseStorage -from gdecore.processor import GDEProcessor -from gdecore.occupancy_cases import OccupancyCasesESRM20 +from ehrecore.configuration import Configuration +from ehrecore.database_queries import DatabaseQueries +from ehrecore.database_storage import DatabaseStorage +from ehrecore.processor import GDEProcessor +from ehrecore.occupancy_cases import OccupancyCasesESRM20 # Add a logger printing error, warning, info and debug messages to the screen logger = logging.getLogger() diff --git a/gdecore/occupancy_cases.py b/ehrecore/occupancy_cases.py similarity index 100% rename from gdecore/occupancy_cases.py rename to ehrecore/occupancy_cases.py diff --git a/gdecore/processor.py b/ehrecore/processor.py similarity index 99% rename from gdecore/processor.py rename to ehrecore/processor.py index 37ed3c9..391b8d4 100644 --- a/gdecore/processor.py +++ b/ehrecore/processor.py @@ -22,8 +22,8 @@ import numpy import mercantile import pandas import pyproj -from gdecore.database_queries import DatabaseQueries -from gdecore.database_storage import DatabaseStorage +from ehrecore.database_queries import DatabaseQueries +from ehrecore.database_storage import DatabaseStorage pandas.options.mode.chained_assignment = None diff --git a/setup.py b/setup.py index 6acc0dd..f40daf4 100644 --- a/setup.py +++ b/setup.py @@ -22,10 +22,10 @@ tests_require = ["pytest"] linters_require = ["black>=22.1.0", "pylint", "flake8"] setup( - name="gdecore", + name="ehrecore", version="0.0.1", - description="Core operations of GDE", - keywords="Global Dynamic Exposure, GDE, buildings, exposure model", + description="Core operations of EHRE", + keywords="European High-Resolution Exposure, EHRE, buildings, exposure model", author="Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ", license="AGPLv3+", install_requires=[ @@ -33,13 +33,13 @@ setup( "shapely", "geopandas", # pylint: disable=line-too-long - "gdeimporter@git+https://git.gfz-potsdam.de/dynamicexposure/globaldynamicexposure/gde-importer.git", # noqa: E501 + "ehreimporter@git+https://git.gfz-potsdam.de/ehre/ehre-software/ehre-importer.git", # noqa: E501 ], extras_require={ "tests": tests_require, "linters": linters_require, }, packages=find_packages(), - entry_points={"console_scripts": ["gdecore = gdecore.gdecore:main"]}, + entry_points={"console_scripts": ["ehrecore = ehrecore.ehrecore:main"]}, python_requires=">=3.7, <3.10", ) diff --git a/tests/conftest.py b/tests/conftest.py index 51a0b07..57f15eb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,7 +20,7 @@ import os from pathlib import Path import pytest from dotenv import load_dotenv -from gdeimporter.tools.database import Database +from ehreimporter.tools.database import Database load_dotenv(Path(".env").resolve()) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 4054f85..479f944 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -18,7 +18,7 @@ import os import pytest -from gdecore.configuration import Configuration +from ehrecore.configuration import Configuration def test_Configuration(): diff --git a/tests/test_database_queries.py b/tests/test_database_queries.py index 604b833..0d76922 100644 --- a/tests/test_database_queries.py +++ b/tests/test_database_queries.py @@ -20,8 +20,8 @@ import os import numpy import pytest import shapely -from gdecore.configuration import Configuration -from gdecore.database_queries import DatabaseQueries +from ehrecore.configuration import Configuration +from ehrecore.database_queries import DatabaseQueries def test_retrieve_aggregated_source_id_and_format(test_db): diff --git a/tests/test_database_storage.py b/tests/test_database_storage.py index ea86334..14caf94 100644 --- a/tests/test_database_storage.py +++ b/tests/test_database_storage.py @@ -20,9 +20,9 @@ import os import numpy import pandas import shapely -from gdeimporter.tools.database import Database -from gdecore.configuration import Configuration -from gdecore.database_storage import DatabaseStorage +from ehreimporter.tools.database import Database +from ehrecore.configuration import Configuration +from ehrecore.database_storage import DatabaseStorage def test_store_number_OBM_and_remainder_buildings(test_db): diff --git a/tests/test_gdecore.py b/tests/test_ehrecore.py similarity index 100% rename from tests/test_gdecore.py rename to tests/test_ehrecore.py diff --git a/tests/test_processor.py b/tests/test_processor.py index 832e7cd..b107965 100644 --- a/tests/test_processor.py +++ b/tests/test_processor.py @@ -20,9 +20,9 @@ import os import logging import numpy import pandas -from gdecore.processor import GDEProcessor -from gdecore.configuration import Configuration -from gdecore.database_queries import DatabaseQueries +from ehrecore.processor import GDEProcessor +from ehrecore.configuration import Configuration +from ehrecore.database_queries import DatabaseQueries logger = logging.getLogger() -- GitLab