From 0dc2ab9f9fce49d17482155cfc9f9e20d98e0767 Mon Sep 17 00:00:00 2001 From: Cecilia Nievas Date: Thu, 28 Apr 2022 11:21:59 +0200 Subject: [PATCH] Added and changed logging messages --- gdeimporter/aggregatedexposuremodel.py | 4 +- gdeimporter/gdeimporter.py | 149 ++++++++++++++++--------- 2 files changed, 100 insertions(+), 53 deletions(-) diff --git a/gdeimporter/aggregatedexposuremodel.py b/gdeimporter/aggregatedexposuremodel.py index 019bf75..bc0e235 100644 --- a/gdeimporter/aggregatedexposuremodel.py +++ b/gdeimporter/aggregatedexposuremodel.py @@ -540,8 +540,8 @@ class ExposureModelESRM20(AggregatedExposureModel): exposure_entities = {} logger.info( - "Retrieving exposure_entities from exposure with format %s " - "with retrieve_exposure_entities" % configuration.exposure_format + "Retrieving exposure_entities from aggregated exposure model with format '%s' " + % configuration.exposure_format ) # Read metadata on the distribution of population during day, night and transit times diff --git a/gdeimporter/gdeimporter.py b/gdeimporter/gdeimporter.py index ee0d0f4..3287753 100644 --- a/gdeimporter/gdeimporter.py +++ b/gdeimporter/gdeimporter.py @@ -49,22 +49,64 @@ def main(): if len(aem.exposure_entities) < 1: raise ValueError("no exposure entities found in %s" % (aem.model_name)) + logger.info("Name of the model: %s" % (aem.model_name)) + logger.info("Format: %s" % (aem.exposure_format)) + if aem.occupancy_cases is not None: + logger.info("Available occupancy cases: %s" % (", ".join(aem.occupancy_cases))) + # Store information on the source of the aggregated exposure model db_gde_tiles = Database(**config.database_gde_tiles) db_gde_tiles.create_connection_and_cursor() aem_source_id, aem_writing_mode = aem.write_source_to_database( db_gde_tiles.cursor, "aggregated_sources" ) - logger.info("Source ID %s: %s" % (aem_source_id, aem_writing_mode)) + logger.info("ID %s for source '%s': %s" % (aem_source_id, aem.model_name, aem_writing_mode)) db_gde_tiles.close_connection() # Interpret and update config.exposure_entities_to_run config.interpret_exposure_entities_to_run(aem) + logger.info( + "%s exposure entity(ies) will be run: %s" + % ( + str(len(config.exposure_entities_to_run)), + ", ".join(config.exposure_entities_to_run), + ) + ) + + logger.info( + "%s occupancy case(s) will be run: %s" + % ( + str(len(config.occupancies_to_run)), + ", ".join(config.occupancies_to_run), + ) + ) + # Retrieve data units per exposure entity and occupancy case for exposure_entity_name in config.exposure_entities_to_run: for occupancy_case in config.occupancies_to_run: + aux_log_string = "Exposure entity '%s' (%s), occupancy case '%s'" % ( + exposure_entity_name, + aem.exposure_entities[exposure_entity_name].code, + occupancy_case, + ) + logger.info("%s: process to retrieve/create data units started." % (aux_log_string)) + + # Retrieve data units aem.get_data_units(config, exposure_entity_name, occupancy_case) + logger.info( + "%s: %s data units retrieved." + % ( + aux_log_string, + str( + len( + aem.exposure_entities[exposure_entity_name] + .occupancy_cases[occupancy_case]["data_units"] + .keys() + ) + ), + ) + ) aem.store_data_units( config.database_gde_tiles, "data_units", @@ -73,71 +115,40 @@ def main(): occupancy_case, aem_source_id, ) - aem.exposure_entities[exposure_entity_name].write_costs_assumptions_to_database( - config.database_gde_tiles, - "exposure_entities_costs_assumptions", - aem_source_id, - occupancy_case, - ) - aem.exposure_entities[ - exposure_entity_name - ].write_people_distribution_in_time_to_database( - config.database_gde_tiles, - "exposure_entities_population_time_distribution", - aem_source_id, - occupancy_case, - ) - aem.exposure_entities[exposure_entity_name].create_data_unit_tiles( - occupancy_case, - config.number_cores, - config.database_built_up, - "obm_built_area_assessments", - config.database_gde_tiles, - "data_unit_tiles", - aem_source_id, - ) + logger.info("%s: all data units stored." % (aux_log_string)) - logger.info("Name of the model: %s" % (aem.model_name)) - logger.info("Format: %s" % (aem.exposure_format)) - if aem.occupancy_cases is not None: - logger.info("Available occupancy cases: %s" % (", ".join(aem.occupancy_cases))) - - logger.info("Data retrieved:") - for exposure_entity in config.exposure_entities_to_run: - logger.info( - " %s (%s):" % (exposure_entity, aem.exposure_entities[exposure_entity].code) - ) - for case in aem.exposure_entities[exposure_entity].occupancy_cases.keys(): - logger.info(" %s:" % case) - for attr in [ - "data_units_type", - "data_units_level", - "data_units_definition", - ]: + # Log summary on the definition of the data units + logger.info("%s, definition of the data units:" % (aux_log_string)) + for attr in ["data_units_type", "data_units_level", "data_units_definition"]: logger.info( " %s: %s" % ( attr, - aem.exposure_entities[exposure_entity].occupancy_cases[case][attr], + aem.exposure_entities[exposure_entity_name].occupancy_cases[ + occupancy_case + ][attr], ) ) - if "data_units" in aem.exposure_entities[exposure_entity].occupancy_cases[case]: + if ( + "data_units" + in aem.exposure_entities[exposure_entity_name].occupancy_cases[occupancy_case] + ): if ( len( - aem.exposure_entities[exposure_entity].occupancy_cases[case][ - "data_units" - ] + aem.exposure_entities[exposure_entity_name].occupancy_cases[ + occupancy_case + ]["data_units"] ) > 0 ): logger.info(" data_units names: retrieved") geometries_retrieved = True - for data_unit_id in aem.exposure_entities[exposure_entity].occupancy_cases[ - case - ]["data_units"]: + for data_unit_id in aem.exposure_entities[ + exposure_entity_name + ].occupancy_cases[occupancy_case]["data_units"]: if ( - aem.exposure_entities[exposure_entity] - .occupancy_cases[case]["data_units"][data_unit_id] + aem.exposure_entities[exposure_entity_name] + .occupancy_cases[occupancy_case]["data_units"][data_unit_id] .geometry is None ): @@ -154,6 +165,42 @@ def main(): logger.info(" data_units names: not retrieved") logger.info(" data_units geometries: not retrieved") + # Assumptions on costs + aem.exposure_entities[exposure_entity_name].write_costs_assumptions_to_database( + config.database_gde_tiles, + "exposure_entities_costs_assumptions", + aem_source_id, + occupancy_case, + ) + logger.info("%s: cost assumptions stored." % (aux_log_string)) + + # Distribution of the population in time + aem.exposure_entities[ + exposure_entity_name + ].write_people_distribution_in_time_to_database( + config.database_gde_tiles, + "exposure_entities_population_time_distribution", + aem_source_id, + occupancy_case, + ) + logger.info("%s: distribution of the population in time stored." % (aux_log_string)) + + # Create and store data-unit tiles + logger.info( + "%s: creation of data-unit tiles started, using (a max. of) %s cores." + % (aux_log_string, str(config.number_cores)) + ) + aem.exposure_entities[exposure_entity_name].create_data_unit_tiles( + occupancy_case, + config.number_cores, + config.database_built_up, + "obm_built_area_assessments", + config.database_gde_tiles, + "data_unit_tiles", + aem_source_id, + ) + logger.info("%s: data-unit tiles created and stored." % (aux_log_string)) + # Leave the program logger.info("gde-importer has finished") sys.exit() -- GitLab