Merge branch 'bug_fix_20200910_b' into 'master'

Bug fixes associated with having adapted the code to run with the config file

See merge request dynamicexposure/globaldynamicexposure/gde_calculations_prototype!3

GDE_check_consistency.py

@@ -392,7 +392,7 @@ def run_this_file(config_dict):
             print('ERROR WHEN CHECKING check_obm_leftover_visual_vs_OQ.csv!!!')
     out_csv.close()  
     print('\n')
-
+    print('Done!')
 
                     
 if __name__=='__main__':

GDE_gather_SERA_and_OBM.py

@@ -95,13 +95,15 @@ def run_this_file(config_dict):
     # Define part of the output name based on the list of cells processed:
     if grid_cells_def_method=='country_in_DB':
         #TO DO: WRITE FUNCTION THAT RETRIEVES CELLS IDs FOR AN ADMIN BOUNDARY GIVEN AS A SHAPEFILE, SO AS TO NOT NEED TO LINK TO THE DATABASE
-        out_name= sera_disaggregation_to_consider + '_' + config_dict['GDE_gather_SERA_and_OBM']['country_name']
+        out_name= sera_disaggregation_to_consider + '_' + config_dict['Cells to Process']['country_name']
+    elif grid_cells_def_method=='admin_ID_in_country':    
+        out_name= 'exposure_grid_'+sera_disaggregation_to_consider+'_'+config_dict['Cells to Process']['country_name']+'_'+str(config_dict['Cells to Process']['admin_id_in_country'])
     elif grid_cells_def_method=='bbox':
         out_name= sera_disaggregation_to_consider + '_bbox_' +str(grid_cells_list[0])+'_'+str(grid_cells_list[-1])
     elif grid_cells_def_method=='arbitrary':
         out_name= sera_disaggregation_to_consider + '_arbitrary_' +str(grid_cells_list[0])+'_'+str(grid_cells_list[-1])
     elif grid_cells_def_method== 'random_from_country':
-        out_name= sera_disaggregation_to_consider + '_' + config_dict['GDE_gather_SERA_and_OBM']['country_name'] + '_random'
+        out_name= sera_disaggregation_to_consider + '_' + config_dict['Cells to Process']['country_name'] + '_random'
     else:
         print('ERROR!!! METHOD FOR DEFINING LIST OF CELL IDs UNKNOWN. ERROR!!!')
     print('Retrieving completeness information for grid cells...')

GDE_plot_maps.py

@@ -226,7 +226,8 @@ def run_this_file(config_dict):
                     plt.savefig(filename, format="png", dpi=300)
                     plt.close()  
                     plt.clf()   
-
+    print('\n')
+    print('Done!')
                     
                     
 if __name__=='__main__':

OBM_assign_cell_ids_and_adm_ids_to_footprints.py

@@ -84,7 +84,9 @@ def run_this_file(config_dict):
     if assign_admin_IDs:
         gdet_psql.assign_admin_IDs_to_OBM_footprints(DB_name_bdgs, DB_username_bdgs, DB_schema_name_bdgs, DB_table_name_bdgs, DB_schema_name_admin, DB_table_name_admin, occupancy_case, adm_level, method=method_to_use, obm_ids_to_process= obm_ids_to_process_list)
         print('\n')   
-
+    print('\n')
+    print('Done!')
+	
 
 if __name__=='__main__':
     # This code needs to be run from the command line as python3 namefile.py configfile.ini

OBM_buildings_per_cell.py

@@ -97,7 +97,8 @@ def run_this_file(config_dict):
                         out_str= gdet_gral.write_bdgs_to_cell_hdf(bdgs_df_i_case, grid_cell_id, case, os.path.join(out_path, 'OBM_bdgs_in_cells'), sera_disaggregation_to_consider, out_path)
                         if out_str!='':
                             log.append('Cell ID '+str(grid_cell_id)+', '+case+': '+out_str)
-    print('\n')    
+    print('\n')
+    print('Done!')   
     # Dump the log list to a file:
     now= datetime.datetime.now()
     now_str= str(now.year)+'_'+str(now.month)+'_'+str(now.day)+'_'+str(now.hour)+'_'+str(now.minute)+'_'+str(now.second)

SERA_create_HDF5_metadata.py

@@ -60,6 +60,7 @@ def run_this_file(config_dict):
         print('Working on '+country_name+', country '+str(k+1)+' of '+str(len(countries))+'\n')
         admin_levels, level_groups, level_groups_levels, _= gdet_sera.get_admin_level_definition(country_name, sera_models_path, output_dfs=False, full_files=True, complete_missing_cols=False)
         gdet_gral.write_SERA_metadata_HDF5(country_name, admin_levels, level_groups, level_groups_levels, os.path.join(out_path, 'Europe_SERA_metadata_v_'+version_of_SERA_metadata+'.hdf5'))
+    print('\n')
     print('Done!')            
             
             

SERA_create_OQ_input_files.py

@@ -179,7 +179,8 @@ def run_this_file(config_dict):
     now= datetime.datetime.now()
     now_str= str(now.year)+'_'+str(now.month)+'_'+str(now.day)+'_'+str(now.hour)+'_'+str(now.minute)+'_'+str(now.second)
     gdet_gral.write_log_file(log, os.path.join(out_path, 'input_files_for_OQ', 'log_file_'+now_str+'.txt'))
-
+    print('\n')
+    print('Done!')
 
 
 if __name__=='__main__':

SERA_create_visual_output_of_grid_model_full_files.py

@@ -165,12 +165,12 @@ def run_this_file(config_dict):
     print('Summary:')    
     out_csv= open(os.path.join(out_path, 'export_grids_QGIS', out_name+'_summary.txt'), "w")
     if grid_cells_def_method=='country_in_DB':
-        out_csv.write('Country: '+country_name+'\n')
+        out_csv.write('Country: '+config_dict['Cells to Process']['country_name']+'\n')
     elif grid_cells_def_method=='admin_ID_in_country':    
-        out_csv.write('Country: '+country_name+'\n')
-        out_csv.write('Admin ID in country: '+str(admin_id_in_country)+'\n')
+        out_csv.write('Country: '+config_dict['Cells to Process']['country_name']+'\n')
+        out_csv.write('Admin ID in country: '+config_dict['Cells to Process']['admin_id_in_country']+'\n')
     elif grid_cells_def_method=='random_from_country':
-        out_csv.write(config_dict['Cells to Process']['num_random_cells']+' random cells selected from country: '+country_name+'\n')
+        out_csv.write(config_dict['Cells to Process']['num_random_cells']+' random cells selected from country: '+config_dict['Cells to Process']['country_name']+'\n')
     elif grid_cells_def_method=='bbox':
         out_csv.write('Bounding box used:\n')
         out_csv.write('South-West Longitude: '+'{:.5f}'.format(bb_sw_lon)+'\n')
@@ -180,7 +180,7 @@ def run_this_file(config_dict):
     elif grid_cells_def_method=='arbitrary':
         out_csv.write('Arbitrary list of cells used:\n')
         out_str= []
-        for cell_id in arbitr_list:
+        for cell_id in grid_cells_list:
             out_str.append(str(cell_id))
         out_csv.write(','.join(out_str))
     out_csv.write('\n')
@@ -205,7 +205,8 @@ def run_this_file(config_dict):
         print(string)
         out_csv.write(string+'\n')
     out_csv.close()
-
+    print('\n')
+    print('Done!')
 
           
 if __name__=='__main__':

SERA_distributing_exposure_to_cells.py

@@ -109,7 +109,9 @@ def run_this_file(config_dict):
     now= datetime.datetime.now()
     now_str= str(now.year)+'_'+str(now.month)+'_'+str(now.day)+'_'+str(now.hour)+'_'+str(now.minute)+'_'+str(now.second)
     gdet_gral.write_log_file(log, os.path.join(out_path, 'log_file_'+now_str+'.txt'))
-
+    print('\n')
+    print('Done!')
+    
 
 if __name__=='__main__':
     # This code needs to be run from the command line as python3 namefile.py configfile.ini

SERA_exploration_check_IDs_unequivocally_defined.py

@@ -63,7 +63,9 @@ def run_this_file(config_dict):
     now= datetime.datetime.now()
     now_str= str(now.year)+'_'+str(now.month)+'_'+str(now.day)+'_'+str(now.hour)+'_'+str(now.minute)+'_'+str(now.second)
     gdet_gral.write_log_file(log, os.path.join(out_path, 'SERA_check_IDs_unequivocally_defined_results_'+now_str+'.txt'))
-
+    print('\n')
+    print('Done!')
+    
 
 if __name__=='__main__':
     # This code needs to be run from the command line as python3 namefile.py configfile.ini

SERA_exploration_check_one_point_per_ID.py

@@ -61,7 +61,8 @@ def run_this_file(config_dict):
             print('      '+case+': Only one point per admin unit (at which the model is defined): '+str(have_one_point_per_ID[k]))
     # Dump the log list to a file:
     gdet_gral.write_log_file(log, os.path.join(out_path, 'SERA_check_one_point_per_ID_results.txt'))
-
+    print('\n')
+    print('Done!')
 
 
 if __name__=='__main__':

SERA_exploration_countries_read.py

@@ -81,7 +81,8 @@ def run_this_file(config_dict):
     outcsv= open(os.path.join(out_path, 'list_SERA_countries.csv'), 'w')
     outcsv.write(','.join(out_res))
     outcsv.close()
-
+    print('\n')
+    print('Done!')
 
 
 if __name__=='__main__':

SERA_exploration_investigate_full_CSV_files.py

@@ -134,7 +134,8 @@ def run_this_file(config_dict):
             check_05= do_check_tolerance(ratio_05, diff_05, tolerance_ratio, tolerance_diff)
             f_out.write(country_name+','+case+','+check_01+','+check_02+','+check_03+','+check_04+','+check_05+','+'{:.2f}'.format(diff_01.max())+','+'{:.2f}'.format(diff_02.max())+','+'{:.2f}'.format(diff_03.max())+','+'{:.2f}'.format(diff_04.max())+','+'{:.2f}'.format(diff_05.max())+','+'{:.6f}'.format(ratio_01.max())+','+'{:.6f}'.format(ratio_02.max())+','+'{:.6f}'.format(ratio_03.max())+','+'{:.6f}'.format(ratio_04.max())+','+'{:.6f}'.format(ratio_05.max())+'\n')
     f_out.close()
-
+    print('\n')
+    print('Done!')
 
 
 if __name__=='__main__':

SERA_exploration_longest_strings.py

@@ -70,7 +70,7 @@ def run_this_file(config_dict):
                 max_tot= len_sum
     print('\n')
     print('Largest combination of all has '+str(max_tot)+' characters.') # answer on 10 March 2020: 45 characters
-
+    print('Done!')
 
 if __name__=='__main__':
     # This code needs to be run from the command line as python3 namefile.py configfile.ini

SERA_exploration_summarise_columns_in_full_CSV_files.py

@@ -94,7 +94,8 @@ def run_this_file(config_dict):
                     which_dont_str= ''
                 f.write(col+','+str(countries_have_it)+','+str(len(countries)-countries_have_it)+','+which_do_str+','+which_dont_str+'\n')
         f.close()
-    
+    print('\n')
+    print('Done!') 
     
 
 if __name__=='__main__':

SERA_mapping_admin_units_to_cells.py

@@ -108,7 +108,8 @@ def run_this_file(config_dict):
     now= datetime.datetime.now()
     now_str= str(now.year)+'_'+str(now.month)+'_'+str(now.day)+'_'+str(now.hour)+'_'+str(now.minute)+'_'+str(now.second)
     gdet_gral.write_log_file(log, os.path.join(out_path, 'log_file_'+now_str+'.txt'))
-
+    print('\n')
+    print('Done!')
 
 
 if __name__=='__main__':

SERA_mapping_admin_units_to_cells_add_GHS.py

@@ -89,8 +89,12 @@ def run_this_file(config_dict):
     log= [] # list that I fill in for logging purposes and then dump to a txt file
     if grid_cells_def_method=='country_in_DB':
         print('Defining grid cells list using method '+grid_cells_def_method+'... Country: '+config_dict['Cells to Process']['country_name'])
+    elif grid_cells_def_method=='admin_ID_in_country': 
+        print('Defining grid cells list using method '+grid_cells_def_method+'... Country: '+config_dict['Cells to Process']['country_name']+'... Admin ID: '+config_dict['Cells to Process']['admin_id_in_country'])
+    elif grid_cells_def_method== 'random_from_country':
+        print('Defining grid cells list using method '+grid_cells_def_method+'... Country: '+config_dict['Cells to Process']['country_name']+'... Number of random cells: '+config_dict['Cells to Process']['num_random_cells'])
     else:
-        print('Defining grid cells list using method '+grid_cells_def_method+'...')
+        print('Defining grid cells list using method '+grid_cells_def_method+'...')  
     # Load GHS grid:
     # The GHS value is a percentage between 0 and 100%.
     fle= h5py.File(os.path.join(ghs_path, ghs_filename))
@@ -119,6 +123,7 @@ def run_this_file(config_dict):
     now_str= str(now.year)+'_'+str(now.month)+'_'+str(now.day)+'_'+str(now.hour)+'_'+str(now.minute)+'_'+str(now.second)
     gdet_gral.write_log_file(log, os.path.join(out_path, 'log_file_adding_GHS_'+now_str+'.txt')) 
     print('\n')
+    print('Done!')
 
 
 

SERA_mapping_admin_units_to_cells_add_GPW.py

@@ -81,8 +81,12 @@ def run_this_file(config_dict):
     log= [] # list that I fill in for logging purposes and then dump to a txt file
     if grid_cells_def_method=='country_in_DB':
         print('Defining grid cells list using method '+grid_cells_def_method+'... Country: '+config_dict['Cells to Process']['country_name'])
+    elif grid_cells_def_method=='admin_ID_in_country': 
+        print('Defining grid cells list using method '+grid_cells_def_method+'... Country: '+config_dict['Cells to Process']['country_name']+'... Admin ID: '+config_dict['Cells to Process']['admin_id_in_country'])
+    elif grid_cells_def_method== 'random_from_country':
+        print('Defining grid cells list using method '+grid_cells_def_method+'... Country: '+config_dict['Cells to Process']['country_name']+'... Number of random cells: '+config_dict['Cells to Process']['num_random_cells'])
     else:
-        print('Defining grid cells list using method '+grid_cells_def_method+'...')
+        print('Defining grid cells list using method '+grid_cells_def_method+'...')  
     # Load GPW grid:
     print('Loading GPW grid...')
     population_values, _, gpw_pop_x_coord_edges, gpw_pop_y_coord_edges, _, _, _, _, _, _= gdet_gpw.get_gpw_grids(gpw_path)
@@ -105,6 +109,7 @@ def run_this_file(config_dict):
     now_str= str(now.year)+'_'+str(now.month)+'_'+str(now.day)+'_'+str(now.hour)+'_'+str(now.minute)+'_'+str(now.second)
     gdet_gral.write_log_file(log, os.path.join(out_path, 'log_file_adding_GPW_'+now_str+'.txt')) 
     print('\n')
+    print('Done!')
 
 
 

SERA_mapping_admin_units_to_cells_add_Sat.py

@@ -76,11 +76,15 @@ def run_this_file(config_dict):
     ####################################################
     # START
     ####################################################
-    log= [] # list that I fill in for logging purposes and then dump to a txt file
+    log= [] # list that I fill in for logging purposes and then dump to a txt file 
     if grid_cells_def_method=='country_in_DB':
         print('Defining grid cells list using method '+grid_cells_def_method+'... Country: '+config_dict['Cells to Process']['country_name'])
+    elif grid_cells_def_method=='admin_ID_in_country': 
+        print('Defining grid cells list using method '+grid_cells_def_method+'... Country: '+config_dict['Cells to Process']['country_name']+'... Admin ID: '+config_dict['Cells to Process']['admin_id_in_country'])
+    elif grid_cells_def_method== 'random_from_country':
+        print('Defining grid cells list using method '+grid_cells_def_method+'... Country: '+config_dict['Cells to Process']['country_name']+'... Number of random cells: '+config_dict['Cells to Process']['num_random_cells'])
     else:
-        print('Defining grid cells list using method '+grid_cells_def_method+'...')
+        print('Defining grid cells list using method '+grid_cells_def_method+'...')   
     # The built-up area from Nicolas' satellite imagery is in m2:
     sat_df= pd.read_csv(os.path.join(sat_path, 'SAR_F_Grid_with_cell_IDs.csv'), sep=';')
     for i, grid_cell_id in enumerate(grid_cells_list):
@@ -104,6 +108,7 @@ def run_this_file(config_dict):
     now_str= str(now.year)+'_'+str(now.month)+'_'+str(now.day)+'_'+str(now.hour)+'_'+str(now.minute)+'_'+str(now.second)
     gdet_gral.write_log_file(log, os.path.join(out_path, 'log_file_adding_Sat_'+now_str+'.txt')) 
     print('\n')
+    print('Done!')
 
 
 if __name__=='__main__':

SERA_testing_compare_visual_output_vs_OQ_input_files.py

@@ -94,11 +94,10 @@ def run_this_file(config_dict):
                 print('       ERROR!! CELL ID '+str(cell_id)+' FOUND MORE THAN ONCE IN VISUAL OUTPUT CSV!!!')            
             row_read[which]= True
     print('\n')
-    print('Done!')
-    print('\n')
     print('Total cost OQ: '+'{:.2f}'.format(full_cost_oq))
     print('Total cost visual: '+'{:.2f}'.format(full_cost_visual))
-
+    print('\n')
+    print('Done!')
 
 if __name__=='__main__':
     # This code needs to be run from the command line as python3 namefile.py configfile.ini