database.py 84.1 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
#!/usr/bin/env python3

# Copyright (C) 2021:
#   Helmholtz-Zentrum Potsdam Deutsches GeoForschungsZentrum GFZ
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
# General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see http://www.gnu.org/licenses/.

import logging
20
from exposurelib.database import SpatialiteDatabase
21
import pandas
22
23
import numpy
from exposurejapan import constants
24
25
26
import shapely.wkt
import pyproj
from shapely.ops import transform
27
28
29
30
31

# Initialize log
logger = logging.getLogger(__name__)


32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
def add_element_and_get_index(element, element_list):
    """
    Checks if an element is in a list and adds it to the list if not.
    Returns the index of the element in the list.

    Args:
        element (str):
            Element to be added to the list
        element_list (list):
            List to add the element to

    Returns:
         Index of inserted element in the list
    """

    if element not in element_list:
        element_list.append(element)
    return element_list.index(element)


52
class JapanDatabase(SpatialiteDatabase):
53
54
    """
    The JapanDatabase class represents a Spatialite database for the Japan
55
56
57
58
59
60
61
62
63
64
65
66
67
68
    exposure model. It is derived from the generic Database class.

    Args:
        database_filepath (str):
            File path for the Spatialite database file.
        spatialite_filepath (str):
            File path of the Spatialite extension.

    Attributes:
        database_filepath (str): Spatialite database file path.
        spatialite_filepath (str): File path to the Spatialite extension.
    """

    def __init__(self, database_filepath, spatialite_filepath="mod_spatialite"):
69
        SpatialiteDatabase.__init__(self, database_filepath, spatialite_filepath)
70
71
72
73

    def create_tables(self):
        """
        Creates all necessary tables in the database. These are:
74
75
76
77
78
            District            : Stores the districts with their IDs, names and geometries
            DwellingNumber      : Stores the number of dwellings depending on building types,
                                  construction material and number of stories for each district
            BuildingNumber      : Stores the number of buildings depending on building types,
                                  construction material and number of stories for each district
79
80
            HouseholdData       : Stores different parameters describing the household numbers,
                                  household members, and household spaces for each district
81
82
83
            DwellingFloorspace  : Stores the floorspace per dwelling depending on the building,
                                  dwelling, tenure types and construction material for each
                                  district
84
85
86
            DwellingDistribution: Stores the number of dwellings of different sizes depending
                                  on the building, dwelling, tenure types and construction
                                  material for each district
87
            BuildingType        : Stores the different types of buildings
88
89
            DwellingType        : Stores the different types of dwellings
            TenureType          : Stores the different types of tenures
90
            ConstructionMaterial: Stores the construction-material types
91
92
93
            StoryNumber         : Stores the number (e.g. 1, 2, 3, 4, ...) and classifications
                                  (1-2, 3-5, ...) of numbers of stories.
            DwellingSizeType    : Stores the different types of dwelling sizes
94
95
96
97
98
99
100
101
102
103
104
105
106
        """

        # Create table District
        sql_statement = "CREATE TABLE District ("
        sql_statement += "id          INTEGER PRIMARY KEY AUTOINCREMENT, "
        sql_statement += "admin_id    TEXT, "
        sql_statement += "admin_name  TEXT)"
        self.connection.execute(sql_statement)
        sql_statement = "SELECT AddGeometryColumn('District', 'geom', 4326, "
        sql_statement += "'MULTIPOLYGON', 'XY')"
        self.connection.execute(sql_statement)
        logger.debug("Table District created")

107
108
109
110
111
112
        # Create table DwellingNumber
        sql_statement = "CREATE TABLE DwellingNumber ("
        sql_statement += "id                        INTEGER PRIMARY KEY AUTOINCREMENT, "
        sql_statement += "district_id               INTEGER, "
        sql_statement += "building_type_id          INTEGER, "
        sql_statement += "construction_material_id  INTEGER, "
113
        sql_statement += "story_number_id           INTEGER, "
114
115
116
117
        sql_statement += "number_dwelling           REAL)"
        self.connection.execute(sql_statement)
        logger.debug("Table DwellingNumber created")

118
119
120
121
122
123
124
125
126
127
128
        # Create table BuildingNumber
        sql_statement = "CREATE TABLE BuildingNumber ("
        sql_statement += "id                        INTEGER PRIMARY KEY AUTOINCREMENT, "
        sql_statement += "district_id               INTEGER, "
        sql_statement += "building_type_id          INTEGER, "
        sql_statement += "construction_material_id  INTEGER, "
        sql_statement += "story_number_id           INTEGER, "
        sql_statement += "number_building           REAL)"
        self.connection.execute(sql_statement)
        logger.debug("Table BuildingNumber created")

129
130
131
132
133
134
135
        # Create table HouseholdData
        sql_statement = "CREATE TABLE HouseholdData ("
        sql_statement += "id                        INTEGER PRIMARY KEY AUTOINCREMENT, "
        sql_statement += "district_id               INTEGER, "
        sql_statement += "building_type_id          INTEGER, "
        sql_statement += "dwelling_type_id          INTEGER, "
        sql_statement += "tenure_type_id            INTEGER, "
136
        sql_statement += "construction_material_id  INTEGER, "
137
138
139
140
141
142
143
144
145
146
147
        sql_statement += "number_dwelling           REAL, "
        sql_statement += "number_household          REAL, "
        sql_statement += "number_household_member   REAL, "
        sql_statement += "rooms_per_dwelling        REAL, "
        sql_statement += "tatami_per_dwelling       REAL, "
        sql_statement += "floorspace_per_dwelling   REAL, "
        sql_statement += "tatami_per_person         REAL, "
        sql_statement += "person_per_room           REAL)"
        self.connection.execute(sql_statement)
        logger.debug("Table HouseholdData created")

148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
        # Create table DwellingFloorspace
        sql_statement = "CREATE TABLE DwellingFloorspace ("
        sql_statement += "id                        INTEGER PRIMARY KEY, "
        sql_statement += "district_id               INTEGER, "
        sql_statement += "building_type_id          INTEGER, "
        sql_statement += "dwelling_type_id          INTEGER, "
        sql_statement += "tenure_type_id            INTEGER, "
        sql_statement += "construction_material_id  INTEGER, "
        sql_statement += "floorspace                REAL)"
        self.connection.execute(sql_statement)
        logger.debug("Table DwellingFloorspace created")

        # Create table DwellingDistribution
        sql_statement = "CREATE TABLE DwellingDistribution ("
        sql_statement += "id                        INTEGER PRIMARY KEY, "
        sql_statement += "district_id               INTEGER, "
        sql_statement += "building_type_id          INTEGER, "
        sql_statement += "dwelling_type_id          INTEGER, "
        sql_statement += "tenure_type_id            INTEGER, "
        sql_statement += "construction_material_id  INTEGER, "
        sql_statement += "dwelling_size_type_id     INTEGER, "
        sql_statement += "number_dwelling           REAL)"
        self.connection.execute(sql_statement)
        logger.debug("Table DwellingDistribution created")

shinde's avatar
shinde committed
173
174
175
176
177
178
179
180
181
182
183
        # Create table PopulationDistribution
        sql_statement = "CREATE TABLE PopulationDistribution ("
        sql_statement += "id                INTEGER PRIMARY KEY, "
        sql_statement += "district_id       INTEGER, "
        sql_statement += "total             INTEGER, "
        sql_statement += "male              INTEGER, "
        sql_statement += "female            INTEGER, "
        sql_statement += "number_household  INTEGER)"
        self.connection.execute(sql_statement)
        logger.debug("Table PopulationDistribution created")

184
185
186
187
188
        # Create table BuildingType
        sql_statement = "CREATE TABLE BuildingType ("
        sql_statement += "id           INTEGER PRIMARY KEY, "
        sql_statement += "description  TEXT)"
        self.connection.execute(sql_statement)
189
        logger.debug("Table BuildingType created")
190

191
192
193
194
195
        # Create table DwellingType
        sql_statement = "CREATE TABLE DwellingType ("
        sql_statement += "id           INTEGER PRIMARY KEY, "
        sql_statement += "description  TEXT)"
        self.connection.execute(sql_statement)
196
        logger.debug("Table DwellingType created")
197
198
199
200
201
202

        # Create table TenureType
        sql_statement = "CREATE TABLE TenureType ("
        sql_statement += "id           INTEGER PRIMARY KEY, "
        sql_statement += "description  TEXT)"
        self.connection.execute(sql_statement)
203
        logger.debug("Table TenureType created")
204

205
206
207
208
209
        # Create table ConstructionMaterial
        sql_statement = "CREATE TABLE ConstructionMaterial ("
        sql_statement += "id           INTEGER PRIMARY KEY, "
        sql_statement += "description  TEXT)"
        self.connection.execute(sql_statement)
210
        logger.debug("Table ConstructionMaterial created")
211

212
        # Create table StoryNumber
213
        sql_statement = "CREATE TABLE StoryNumber ("
214
215
216
        sql_statement += "id           INTEGER PRIMARY KEY, "
        sql_statement += "description  TEXT)"
        self.connection.execute(sql_statement)
217
218
219
220
221
222
223
224
        logger.debug("Table StoryNumber created")

        # Create table DwellingSizeType
        sql_statement = "CREATE TABLE DwellingSizeType ("
        sql_statement += "id           INTEGER PRIMARY KEY, "
        sql_statement += "description  TEXT)"
        self.connection.execute(sql_statement)
        logger.debug("Table DwellingSizeType created")
225

226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
    def insert_district(self, admin_id, admin_name, geom):
        """
        Inserts the ID, name, and geometry of a district to the District table.

        Args:
            admin_id (str):
                ID of the administrative district
            admin_name (str):
                Name of the administrative district
            geom (multipolygon):
                Geometry (Spatialite) of the district boundary
        """

        sql_statement = "INSERT INTO District "
        sql_statement += "(admin_id, admin_name, geom) "
        sql_statement += "VALUES ('%s', '%s', %s)" % (admin_id, admin_name, geom)
        self.cursor.execute(sql_statement)

244
245
246
247
248
    def insert_dwelling_number(
        self,
        district_id,
        building_type_id,
        construction_material_id,
249
        story_number_id,
250
251
252
        number_dwelling,
    ):
        """
253
        Inserts a full dataset to the DwellingNumber table.
254
255
256
257
258
259
260
261

        Args:
            district_id (int):
                ID of the district. Corresponds to District.id
            building_type_id (int):
                ID of the building type. Corresponds to BuildingType.id
            construction_material_id (int):
                ID of the construction material. Corresponds to ConstructionMaterial.id
262
            story_number_id  (int):
263
264
                ID of the number of stories type. Corresponds to NumberStories.id
            number_dwelling (float):
265
                Number of dwellings for the combination of the other parameters
266
267
268
269
        """

        sql_statement = "INSERT INTO DwellingNumber "
        sql_statement += "(district_id, building_type_id, construction_material_id, "
270
        sql_statement += "story_number_id, number_dwelling) "
271
272
273
274
        sql_statement += "VALUES (%d, %d, %d, %d, %f)" % (
            district_id,
            building_type_id,
            construction_material_id,
275
            story_number_id,
276
277
278
279
            number_dwelling,
        )
        self.cursor.execute(sql_statement)

280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
    def insert_building_number(
        self,
        district_id,
        building_type_id,
        construction_material_id,
        story_number_id,
        number_building,
    ):
        """
        Inserts a full dataset to the BuildingNumber table.

        Args:
            district_id (int):
                ID of the district. Corresponds to District.id
            building_type_id (int):
                ID of the building type. Corresponds to BuildingType.id
            construction_material_id (int):
                ID of the construction material. Corresponds to ConstructionMaterial.id
            story_number_id (int):
                ID of the number of stories type. Corresponds to NumberStories.id
            number_building (float):
                Number of buildings for the combination of the other parameters
        """

        sql_statement = "INSERT INTO BuildingNumber "
        sql_statement += "(district_id, building_type_id, construction_material_id, "
        sql_statement += "story_number_id, number_building) "
        sql_statement += "VALUES (%d, %d, %d, %d, %f)" % (
            district_id,
            building_type_id,
            construction_material_id,
            story_number_id,
            number_building,
        )
        self.cursor.execute(sql_statement)

316
317
318
319
320
321
    def insert_household_data(
        self,
        district_id,
        building_type_id,
        dwelling_type_id,
        tenure_type_id,
322
        construction_material_id,
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
        number_dwelling,
        number_household,
        number_household_member,
        rooms_per_dwelling,
        tatami_per_dwelling,
        floorspace_per_dwelling,
        tatami_per_person,
        person_per_room,
    ):
        """
        Inserts a full dataset to the HouseholdData table.

        Args:
            district_id (int):
                ID of the district. Corresponds to District.id
            building_type_id (int):
                ID of the building type. Corresponds to BuildingType.id
            dwelling_type_id (int):
                ID of the dwelling type. Corresponds to DwellingType.id
            tenure_type_id (int):
                ID of the tenure type. Corresponds to TenureType.id
344
345
            construction_material_id (int):
                ID of the construction material. Corresponds to ConstructionMaterial.id
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
            number_dwelling (float):
                Number of dwellings for the combination of the other parameters
            number_household (float):
                Number of households for the combination of the other parameters
            number_household_member (float):
                Number of household members for the combination of the other parameters
            rooms_per_dwelling (float):
                Number of rooms per dwelling for the combination of the other parameters
            tatami_per_dwelling (float):
                Number of tatami mats per dwelling for the combination of the other parameters
            floorspace_per_dwelling (float):
                Size of floorspace (in square meters) per dwelling for the combination of the
                other parameters
            tatami_per_person (float):
                Number of tatami mats per person for the combination of the other parameters
            person_per_room (float):
                Number of persons per room for the combination of the other parameters
        """

        sql_statement = "INSERT INTO HouseholdData "
        sql_statement += "(district_id, building_type_id, dwelling_type_id, tenure_type_id, "
367
368
        sql_statement += "construction_material_id, number_dwelling, "
        sql_statement += "number_household, number_household_member, "
369
370
        sql_statement += "rooms_per_dwelling, tatami_per_dwelling, floorspace_per_dwelling, "
        sql_statement += "tatami_per_person, person_per_room) "
371

372
        if construction_material_id == constants.TOTAL:
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
            sql_statement += "VALUES (%d, %d, %d, %d, %d, %f, %f, %f, %f, %f, %f, %f, %f)" % (
                district_id,
                building_type_id,
                dwelling_type_id,
                tenure_type_id,
                construction_material_id,
                number_dwelling,
                number_household,
                number_household_member,
                rooms_per_dwelling,
                tatami_per_dwelling,
                floorspace_per_dwelling,
                tatami_per_person,
                person_per_room,
            )
388
389
390
391
392
393
394
395
396
397
398
399
400
401
        elif construction_material_id != constants.TOTAL and (
            tenure_type_id != constants.TOTAL or dwelling_type_id != constants.TOTAL
        ):
            sql_statement += (
                "VALUES (%d, %d, %d, %d, %d, NULL, NULL, NULL, NULL, "
                "NULL, NULL, NULL, NULL)"
                % (
                    district_id,
                    building_type_id,
                    dwelling_type_id,
                    tenure_type_id,
                    construction_material_id,
                )
            )
402
403
404
405
406
407
408
409
410
411
412
413
414
        else:
            sql_statement += (
                "VALUES (%d, %d, %d, %d, %d, NULL, %f, %f, NULL, NULL, NULL, NULL, NULL)"
                % (
                    district_id,
                    building_type_id,
                    dwelling_type_id,
                    tenure_type_id,
                    construction_material_id,
                    number_household,
                    number_household_member,
                )
            )
415
416
        self.cursor.execute(sql_statement)

417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
    def insert_dwelling_floorspace(
        self,
        district_id,
        building_type_id,
        dwelling_type_id,
        tenure_type_id,
        construction_material_id,
        floorspace,
    ):
        """
        Inserts a full dataset to the DwellingFloorspace table.

        Args:
            district_id (int):
                ID of the district. Corresponds to District.id
            building_type_id (int):
                ID of the building type. Corresponds to BuildingType.id
            dwelling_type_id (int):
                ID of the dwelling type. Corresponds to DwellingType.id
            tenure_type_id (int):
                ID of the tenure type. Corresponds to TenureType.id
            construction_material_id (int):
                ID of the construction material. Corresponds to ConstructionMaterial.id
            floorspace (float):
                Size of the floorspace in square meters
        """

        sql_statement = "INSERT INTO DwellingFloorspace "
        sql_statement += "(district_id, building_type_id, dwelling_type_id, tenure_type_id, "
        sql_statement += "construction_material_id, floorspace) "
        sql_statement += "VALUES (%d, %d, %d, %d, %d, %f)" % (
            district_id,
            building_type_id,
            dwelling_type_id,
            tenure_type_id,
            construction_material_id,
            floorspace,
        )
        self.cursor.execute(sql_statement)

    def insert_dwelling_distribution(
        self,
        district_id,
        building_type_id,
        dwelling_type_id,
        tenure_type_id,
        construction_material_id,
        dwelling_size_type_id,
        number_dwelling,
    ):
        """
        Inserts a full dataset to the DwellingDistribution table.

        Args:
            district_id (int):
                ID of the district. Corresponds to District.id
            building_type_id (int):
                ID of the building type. Corresponds to BuildingType.id
            dwelling_type_id (int):
                ID of the dwelling type. Corresponds to DwellingType.id
            tenure_type_id (int):
                ID of the tenure type. Corresponds to TenureType.id
            construction_material_id (int):
                ID of the construction material. Corresponds to ConstructionMaterial.id
            dwelling_size_type_id (int):
                ID of the dwelling-size type. Corresponds to DwellingSizeType.id
            number_dwelling (float):
                Number of dwellings for the combination of the other parameters
        """

        sql_statement = "INSERT INTO DwellingDistribution "
        sql_statement += "(district_id, building_type_id, dwelling_type_id, tenure_type_id, "
        sql_statement += "construction_material_id, dwelling_size_type_id, number_dwelling) "
        sql_statement += "VALUES (%d, %d, %d, %d, %d, %d, %f)" % (
            district_id,
            building_type_id,
            dwelling_type_id,
            tenure_type_id,
            construction_material_id,
            dwelling_size_type_id,
            number_dwelling,
        )
        self.cursor.execute(sql_statement)

shinde's avatar
shinde committed
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
    def insert_population_distribution(
        self, district_id, total, male, female, number_household
    ):
        """
        Inserts a full dataset to the PopulationDistribution table.

        Args:
            district_id (int):
                ID of the district. Corresponds to District.id
            total (int):
                Total population of the district
            male (int):
                Male population of the district
            female (int):
                Female population of the district
            number_household (int):
                Total number of households of the district
        """

        sql_statement = "INSERT INTO PopulationDistribution "
        sql_statement += "(district_id, total, male, female, "
        sql_statement += "number_household) "
        sql_statement += "VALUES (%d, %d, %d, %d, %d)" % (
            district_id,
            total,
            male,
            female,
            number_household,
        )
        self.cursor.execute(sql_statement)

532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
    def insert_building_type(self, building_type_id, description):
        """
        Inserts a building-type description to the BuildingType table.

        Args:
            building_type_id (int):
                ID of the building type
            description (str):
                Description of the building type
        """

        sql_statement = "INSERT INTO BuildingType "
        sql_statement += "(id, description) "
        sql_statement += "VALUES (%d, '%s')" % (building_type_id, description)
        self.cursor.execute(sql_statement)

548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
    def insert_dwelling_type(self, dwelling_type_id, description):
        """
        Inserts a dwelling-type description to the DwellingType table.

        Args:
            dwelling_type_id (int):
                ID of the dwelling type
            description (str):
                Description of the dwelling type
        """

        sql_statement = "INSERT INTO DwellingType "
        sql_statement += "(id, description) "
        sql_statement += "VALUES (%d, '%s')" % (dwelling_type_id, description)
        self.cursor.execute(sql_statement)

    def insert_tenure_type(self, tenure_type_id, description):
        """
        Inserts a tenure-type description to the TenureType table.

        Args:
            tenure_type_id (int):
                ID of the tenure type
            description (str):
                Description of the tenure type
        """

        sql_statement = "INSERT INTO TenureType "
        sql_statement += "(id, description) "
        sql_statement += "VALUES (%d, '%s')" % (tenure_type_id, description)
        self.cursor.execute(sql_statement)

580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
    def insert_construction_material(self, construction_material_id, description):
        """
        Inserts a construction-material description to the ConstructionMaterial table.

        Args:
            construction_material_id (int):
                ID of the construction material
            description (str):
                Description of the construction material
        """

        sql_statement = "INSERT INTO ConstructionMaterial "
        sql_statement += "(id, description) "
        sql_statement += "VALUES (%d, '%s')" % (construction_material_id, description)
        self.cursor.execute(sql_statement)

596
    def insert_story_number(self, story_number_id, description):
597
        """
598
        Inserts a story-number description to the StoryNumber table.
599
600

        Args:
601
            story_number_id (int):
602
603
604
605
606
                ID of the number of stories entry
            description (str):
                Description of the number of stories entry
        """

607
        sql_statement = "INSERT INTO StoryNumber "
608
        sql_statement += "(id, description) "
609
        sql_statement += "VALUES (%d, '%s')" % (story_number_id, description)
610
611
        self.cursor.execute(sql_statement)

612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
    def insert_dwelling_size_type(self, dwelling_size_type_id, description):
        """
        Inserts a Dwelling-size description to the DwellingSizeType table.

        Args:
            dwelling_size_type_id (int):
                ID of the dwelling-size type entry
            description (str):
                Description of the dwelling-size type entry
        """

        sql_statement = "INSERT INTO DwellingSizeType "
        sql_statement += "(id, description) "
        sql_statement += "VALUES (%d, '%s')" % (dwelling_size_type_id, description)
        self.cursor.execute(sql_statement)

628
629
    def read_districts_and_boundaries(self, district_boundary_filepath):
        """
630
        Imports all districts and boundaries from a prepared geopackage file.
631
632
633
634
635
636
637
638
        ('estat_bound_municipal`). The function opens this file as Spatialite database
        and copies all necessary district and boundary information into the `District` table.

        Args:
            district_boundary_filepath (str):
                File path to the boundary file
        """

639
640
        boundary_db = SpatialiteDatabase(district_boundary_filepath, self.spatialite_filepath)
        boundary_db.connect(init_spatial_metadata=False)
641
642
643
644

        sql_statement = "SELECT key_code_ward, "
        sql_statement += "CITY_NAME, "
        sql_statement += "AsWKT(CastAutomagic(geom)) "
645
        sql_statement += "FROM Boundary"
646
647
648
649
650
        boundary_db.cursor.execute(sql_statement)
        for row in boundary_db.cursor:
            self.insert_district(
                row[0],
                row[1],
651
652
                "ST_Transform(CastToMultiPolygon(GeomFromText('%s', %d)), %d)"
                % (row[2], constants.JGD2000, constants.WGS84),
653
654
655
656
            )
            logger.debug("District %d with boundary added" % row[0])
        self.connection.commit()
        logger.info("Districts and boundaries added")
657

shinde's avatar
shinde committed
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
    def get_district_id(self, admin_id):
        """
        Gets the ID of the district in the Districts table corresponding
        to the ID of the administrative district specified.

        Args:
            admin_id (str):
                ID of the administrative district
        """

        # Identify district_id based on admin_id from the District table
        sql_statement = "SELECT id FROM District "
        sql_statement += "WHERE admin_id = %d" % admin_id
        self.cursor.execute(sql_statement)
        result = self.cursor.fetchone()
        return result

675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
    def create_list_with_dwelling_data(self):
        """
        Create an empty list and store the required building attributes from
        the DwellingNumbers table in the list.
        """

        # Create an empty list to add dwelling attributes
        dwelling_attributes = []

        # Select the building_type_id and construction_material_id from DwellingNumber table
        sql_statement = "SELECT district_id, building_type_id, "
        sql_statement += "construction_material_id, "
        sql_statement += "number_dwelling "
        sql_statement += (
            "FROM DwellingNumber WHERE (construction_material_id != %d "
            "AND construction_material_id != %d "
            "AND construction_material_id != %d"
            ")"
            % (
                constants.TOTAL,
                constants.WOODEN_EXCLUDING_WOODEN_AND_FIRE_PROOFED,
                constants.WOODEN_AND_FIRE_PROOFED,
            )
        )
        self.cursor.execute(sql_statement)

        # Store the DwellingNumber attributes to the dwelling_attributes list
        for row in self.cursor:
            dwelling_attributes.append(row)
        return dwelling_attributes

    @staticmethod
    def calculate_dwelling_numbers_proportion(dwelling_numbers):
        """
        Calculates the proportion of dwelling_numbers for
        wooden and non-wooden construction material.

        Args:
            dwelling_numbers (list):
                List of building_type_id, construction_material_id and its
                corresponding number_dwelling for each district_id.
        """

        # Create empty lists for wooden, non-wooden and a list to store building attributes
        # only.
        wooden = []
        non_wooden = []
        proportion = []
        # Store dwelling-number values into wooden, non-wooden and numbers
        for row in dwelling_numbers:
            if row[constants.CONSTRUCTION_MATERIAL_ID] == constants.WOODEN:
726
                wooden.append(row[constants.NUMBER_DWELLING_HOUSEHOLD_PROPORTIONS])
727
            else:  # Construction_material non-wooden
728
                non_wooden.append(row[constants.NUMBER_DWELLING_HOUSEHOLD_PROPORTIONS])
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755

            # Create a tuple with district_id, building_type_id and construction_material_id
            attributes = (
                row[constants.DISTRICT_ID],
                row[constants.BUILDING_TYPE_ID],
                row[constants.CONSTRUCTION_MATERIAL_ID],
            )
            add_element_and_get_index(attributes, proportion)
        # Sum the wooden and non-wooden dwelling numbers
        sum_wooden = sum(filter(None, wooden))
        sum_non_wooden = sum(filter(None, non_wooden))
        sum_wooden_non_wooden = sum_wooden + sum_non_wooden
        # Calculate proportion of dwellings numbers
        # and append numbers to store the proportions
        for row in proportion:
            if row[constants.CONSTRUCTION_MATERIAL_ID] == constants.WOODEN:
                prop = numpy.true_divide(sum_wooden, sum_wooden_non_wooden)
                index = proportion.index(row)
                proportion[index] = row + (prop,)
                numpy.seterr(divide="ignore", invalid="ignore")
            else:
                prop = numpy.true_divide(sum_non_wooden, sum_wooden_non_wooden)
                index = proportion.index(row)
                proportion[index] = row + (prop,)
                numpy.seterr(divide="ignore", invalid="ignore")
        return proportion

756
    @staticmethod
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
    def frequency_distribution_numbers(number_total, attributes_row, sum_numbers):
        """
        Calculates the frequency distribution number, reduces the tuple and returns a tuple
        with reduced row and frequency distribution number.

        Args:
            number_total (int):
                Dwelling and building number for building_type_id,
                construction_material_id and story_number_id value 0 (i.e. total).
            attributes_row (tuple):
                Tuple with district_id, building_type_id, construction_material_id and
                number_dwelling or number_building
                story_number_id.
            sum_numbers ():
                Sum of number_dwelling or number_building for each district_id.

        Returns:
            Tuple with with reduced row and frequency distribution number
        """

        frequency_distribution_numbers = numpy.true_divide(
            (number_total * attributes_row[constants.NUMBER_DWELLING_OR_NUMBER_BUILDING]),
            sum_numbers,
        )
        numpy.seterr(divide="ignore")

        # Remove building_number from row
        row_reduced = attributes_row[
            constants.DISTRICT_ID : constants.NUMBER_DWELLING_OR_NUMBER_BUILDING  # noqa: E203,E501
        ]

        return row_reduced + (frequency_distribution_numbers,)

    def calculate_frequency_distributions(self, attributes):
791
792
793
794
795
796
797
798
799
        """
        For combinations of building_type_id, construction_material_id and
        story_number_id values other than 0 (i.e. total), the dwelling_number or
        building_number proportions are calculated to the dwelling_number or building_number of
        building_type_id, construction_material_id and story_number_id value of 0 (i.e.
        total) to fix the rounding errors.

        Args:
            attributes (list):
800
                List of building_type_id, construction_material_id, story_number_id and its
801
802
803
804
805
                corresponding number_dwelling or number_building for each district_id.

        Returns:
            Building attributes with number_dwelling or number_building adjusted for their
            rounding up errors.
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
        """

        # Extract dwelling and building numbers for building_type_id,
        # construction_material_id and story_number_id value 0 (i.e. total)
        for row in attributes:
            if (
                row[constants.BUILDING_TYPE_ID] == constants.TOTAL
                and row[constants.CONSTRUCTION_MATERIAL_ID] == constants.TOTAL
                and row[constants.STORY_NUMBER_ID] == constants.TOTAL
            ):
                number_total = row[constants.NUMBER_DWELLING_OR_NUMBER_BUILDING]

        # Create empty list to store the building or dwelling numbers
        attributes_numbers_sum = []

        # Append attributes_numbers_sum to add building or dwelling
        # numbers and calculate their sum
        for row in attributes:
            if (
                row[constants.BUILDING_TYPE_ID] != constants.TOTAL
                and row[constants.BUILDING_TYPE_ID] != constants.OTHER
                and row[constants.CONSTRUCTION_MATERIAL_ID] != constants.TOTAL
                and row[constants.CONSTRUCTION_MATERIAL_ID] != constants.WOODEN_AND_FIRE_PROOFED
                and row[constants.CONSTRUCTION_MATERIAL_ID]
                != constants.WOODEN_EXCLUDING_WOODEN_AND_FIRE_PROOFED
                and row[constants.STORY_NUMBER_ID] != constants.TOTAL
            ):
                attributes_numbers_sum.append(row[constants.NUMBER_DWELLING_OR_NUMBER_BUILDING])

            if (
                row[constants.BUILDING_TYPE_ID] == constants.OTHER
                and row[constants.CONSTRUCTION_MATERIAL_ID] != constants.TOTAL
                and row[constants.CONSTRUCTION_MATERIAL_ID] != constants.WOODEN_AND_FIRE_PROOFED
                and row[constants.CONSTRUCTION_MATERIAL_ID]
                != constants.WOODEN_EXCLUDING_WOODEN_AND_FIRE_PROOFED
            ):
                attributes_numbers_sum.append(row[constants.NUMBER_DWELLING_OR_NUMBER_BUILDING])
        sum_num = sum(attributes_numbers_sum)

        # Calculate the proportion of the building or dwelling numbers to their sum
846
847
        # and add the building or dwelling number calculated with the proportion to the
        # attributes list
848
849
850
851
852
853
854
855
856
857
858
        for row in attributes:
            if (
                row[constants.BUILDING_TYPE_ID] != constants.TOTAL
                and row[constants.BUILDING_TYPE_ID] != constants.OTHER
                and row[constants.CONSTRUCTION_MATERIAL_ID] != constants.TOTAL
                and row[constants.CONSTRUCTION_MATERIAL_ID] != constants.WOODEN_AND_FIRE_PROOFED
                and row[constants.CONSTRUCTION_MATERIAL_ID]
                != constants.WOODEN_EXCLUDING_WOODEN_AND_FIRE_PROOFED
                and row[constants.STORY_NUMBER_ID] != constants.TOTAL
            ):
                index = attributes.index(row)
859

860
                # Add the building or dwelling number in the tuple
861
862
863
                attributes[index] = self.frequency_distribution_numbers(
                    number_total, row, sum_num
                )
864
865
866
867
868
869
870
871
872

            if (
                row[constants.BUILDING_TYPE_ID] == constants.OTHER
                and row[constants.CONSTRUCTION_MATERIAL_ID] != constants.TOTAL
                and row[constants.CONSTRUCTION_MATERIAL_ID] != constants.WOODEN_AND_FIRE_PROOFED
                and row[constants.CONSTRUCTION_MATERIAL_ID]
                != constants.WOODEN_EXCLUDING_WOODEN_AND_FIRE_PROOFED
            ):
                index = attributes.index(row)
873

874
                # Add the building or dwelling number in the tuple
875
876
877
                attributes[index] = self.frequency_distribution_numbers(
                    number_total, row, sum_num
                )
878
879
        return attributes

880
881
882
883
884
885
886
887
888
889
890
891
    @staticmethod
    def calculate_household_frequency_distributions(attributes):
        """
        For combinations of building_type_id and construction_material_id values other than 0
        (i.e. total), the household_number and household_member proportions are calculated to
        the household_number and household_member of building_type_id and
        construction_material_id value of 0 (i.e. total) to fix the rounding errors.

        Args:
            attributes (list):
                List of building_type_id, construction_material_id and its
                corresponding household_number and household_member for each district_id.
892
893
894
895

        Returns:
            Building attributes with household_number and household_member adjusted for their
            rounding up errors.
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
        """

        # Extract household and household_member numbers for building_type_id and
        # construction_material_id value 0 (i.e. total)
        for row in attributes:
            if (
                row[constants.BUILDING_TYPE_ID] == constants.TOTAL
                and row[constants.CONSTRUCTION_MATERIAL_ID] == constants.TOTAL
                and row[constants.DWELLING_TYPE_ID] == constants.TOTAL
                and row[constants.TENURE_TYPE_ID] == constants.TOTAL
            ):
                household_number_total = row[constants.NUMBER_HOUSEHOLD]
                household_member_total = row[constants.NUMBER_HOUSEHOLD_MEMBER]

        # Create empty list to store the household and household_member numbers
        number_household_sum = []
        number_household_members_sum = []

        # Append number_household_sum and number_household_members_sum to add household
        # and household member numbers and calculate the sum
        for row in attributes:
            if (
                row[constants.BUILDING_TYPE_ID] != constants.TOTAL
                and row[constants.CONSTRUCTION_MATERIAL_ID] != constants.TOTAL
920
921
                and row[constants.DWELLING_TYPE_ID] == constants.TOTAL
                and row[constants.TENURE_TYPE_ID] == constants.TOTAL
922
923
            ):
                # Append list with household numbers
924
                number_household_sum.append(row[constants.NUMBER_HOUSEHOLD])
925
                # Append list with household members
926
                number_household_members_sum.append(row[constants.NUMBER_HOUSEHOLD_MEMBER])
927
928
929
930
931
932
933
934
935
936
        sum_household_number = sum(number_household_sum)
        sum_household_member = sum(number_household_members_sum)

        # Calculate the proportion of the household and household member numbers to their sum
        # and add the household and household member numbers calculated with the proportion
        # to the attributes list
        for row in attributes:
            if (
                row[constants.BUILDING_TYPE_ID] != constants.TOTAL
                and row[constants.CONSTRUCTION_MATERIAL_ID] != constants.TOTAL
937
938
                and row[constants.DWELLING_TYPE_ID] == constants.TOTAL
                and row[constants.TENURE_TYPE_ID] == constants.TOTAL
939
940
941
942
943
944
945
946
947
948
            ):
                frequency_distribution_household_number = numpy.true_divide(
                    (household_number_total * row[constants.NUMBER_HOUSEHOLD]),
                    sum_household_number,
                )
                frequency_distribution_household_member = numpy.true_divide(
                    (household_member_total * row[constants.NUMBER_HOUSEHOLD_MEMBER]),
                    sum_household_member,
                )
                numpy.seterr(divide="ignore")
949
950
951
952
953
954
955
956

                # Add the household numbers in the tuple
                index = attributes.index(row)
                attributes[index] = row + (
                    frequency_distribution_household_number,
                    frequency_distribution_household_member,
                )

957
958
        return attributes

959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
    @staticmethod
    def get_max_neighbours(population_list):
        """
        Returns the maximum number of neighbors as 10. If the length of the population_list
        is less than 10, then the function returns the max_neighbors as the length of the
        population_list.

        Args:
            population_list list():
                Population distribution list

        Returns:
            Maximum number of neighbors.
        """

        max_neighbors = 10

        if len(population_list) < max_neighbors:
            return len(population_list)
        else:
            return max_neighbors

    @staticmethod
    def reproject_polygon(input_polygon, input_crs, target_crs):
        """
        Returns a (multi)polygon object transformed from input_crs to target_crs. if
        "aea" is specified as target crs, returns the Albers equal area transformed
        polygon.

        More information on the Albers equal area can be found at:
            https://pro.arcgis.com/en/pro-app/latest/help/mapping/properties/albers.htm

        Args:
            input_polygon (shapely.geometry.polygon.Polygon):
                Polygon object to reproject.

            input_crs (str):
                Initial crs of the input_polygon (e.g. "epsg:4326").

            target_crs (str):
                Target crs for the final polygon object. If "aea" is
                specified, process the complete Albers equal area transformation.
For faster browsing, not all history is shown. View entire blame