Draft: Resolve "Add option to do the entire process in batches"

Closes #18 (closed)

\approve @ds @gislars

buildinginitializer/buildinginitializer.py

@@ -61,6 +61,10 @@ class BuildingInitializer:
         building_geometry_column,
     ):
         self.rule_handler = RuleHandler(rules)
+        outputs = []
+        for rule in self.rule_handler:
+            outputs.extend(rule.outputs)
+        self.outputs = set(outputs)
         self.source_db = source_db
         self.building_db = building_db
         self.source_building_db = source_building_db
@@ -151,11 +155,13 @@ class BuildingInitializer:
             queue.put(_building_ids)
 
         # Wait until queue is empty and meanwhile log information
-        logger.info(f"{n + 1} batches are in the queue")
+        logger.info(f"{(n + 1) * batch_size} buildings are in the queue")
         while queue.qsize() > 0:
             time_spent = (datetime.datetime.now() - start_process).total_seconds()
             logger.info(
-                f"Length queue: {queue.qsize()}\tTotal time spent: {time_spent} seconds"
+                f"Buildings in queue: {(queue.qsize() * batch_size)}\t"
+                f"Total time spent: {time_spent} seconds\t"
+                f"Number of active workers: {pool._processes}"
             )
             time.sleep(10)
 
@@ -255,7 +261,8 @@ class BuildingInitializer:
         columns = set()
         for _, args in result.items():
             for key in args.keys():
-                columns.add(key)
+                if key in self.outputs:
+                    columns.add(key)
         columns = list(columns)
 
         # Format values so that they can be uploaded as a batch on a PostGIS database.