Merge pull request #589 from colouring-london/features/migrations_sustainability

Features/migrations sustainability

etl/join_building_data/load_shapefile_to_staging.py

@@ -0,0 +1,140 @@
+"""Join shapefile data to buildings
+
+This is effectively an example script using the HTTP API, tailored to particular collected
+datasets for Camden (age data) and Fitzrovia (number of storeys).
+
+- read through shapes
+- locate building by toid
+- else locate building by representative point
+- update building with data
+"""
+import json
+import os
+import sys
+from functools import partial
+
+import fiona
+import pyproj
+import requests
+from shapely.geometry import shape
+from shapely.ops import transform
+
+
+osgb_to_ll = partial(
+    pyproj.transform,
+    pyproj.Proj(init='epsg:27700'),
+    pyproj.Proj(init='epsg:4326')
+)
+
+
+def main(base_url, api_key, process, source_file):
+    """Read from file, update buildings
+    """
+    with fiona.open(source_file, 'r') as source:
+        for feature in source:
+            props = feature['properties']
+
+            if process == "camden":
+                toid, data = process_camden(props)
+            else:
+                toid, data = process_fitzrovia(props)
+
+            if data is None:
+                continue
+
+            building_id = find_building(toid, feature['geometry'], base_url)
+            if not building_id:
+                print("no_match", toid, "-")
+                continue
+
+            save_data(building_id, data, api_key, base_url)
+
+
+def process_camden(props):
+    toid = osgb_toid(props['TOID'])
+    data = {
+        'date_year': props['Year_C'],
+        'date_source_detail': props['Date_sou_1']
+    }
+    return toid, data
+
+
+def process_fitzrovia(props):
+    toid = osgb_toid(props['TOID'])
+    storeys = props['Storeys']
+
+    if storeys is None:
+        return toid, None
+
+    if props['Basement'] == 'Yes':
+        data = {
+            'size_storeys_core': int(storeys) - 1,
+            'size_storeys_basement': 1
+        }
+    else:
+        data = {
+            'size_storeys_core': int(storeys),
+            'size_storeys_basement': 0
+        }
+    return toid, data
+
+
+def osgb_toid(toid):
+    if toid is None:
+        toid = ""
+    return "osgb" + toid.lstrip("0")
+
+
+def save_data(building_id, data, api_key, base_url):
+    """Save data to a building
+    """
+    r = requests.post(
+        "{}/buildings/{}.json?api_key={}".format(base_url, building_id, api_key),
+        json=data
+    )
+
+
+def find_building(toid, geom, base_url):
+    """Find building_id by TOID or location
+    """
+    r = requests.get(base_url + "/buildings/reference", params={
+        'key': 'toid',
+        'id': toid
+    })
+    buildings = r.json()
+    if buildings and len(buildings) == 1:
+        bid = buildings[0]['building_id']
+        print("match_by_toid", toid, bid)
+        return bid
+
+    # try location
+    poly = shape(geom)
+    point_osgb = poly.centroid
+    if not poly.contains(point_osgb):
+        point_osgb = poly.representative_point()
+
+    point_ll = transform(osgb_to_ll, point_osgb)
+    r = requests.get(base_url + "/buildings/locate", params={
+        'lng': point_ll.x,
+        'lat': point_ll.y
+    })
+    buildings = r.json()
+    if buildings and len(buildings) == 1:
+        bid = buildings[0]['building_id']
+        print("match_by_location", toid, bid)
+        return bid
+
+    return None
+
+
+if __name__ == '__main__':
+    try:
+        url, api_key, process, filename = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
+    except IndexError:
+        print(
+            "Usage: {} <URL> <api_key> <camden|fitzrovia> ./path/to/camden.shp".format(
+            os.path.basename(__file__)
+        ))
+        exit()
+
+    main(url, api_key, process, filename)

migrations/011.sustainability.down1-extra.sql

@@ -0,0 +1,35 @@
+-- Remove sustainability fields, update in paralell with adding new fields
+-- Last significant retrofit date YYYY
+-- Need to add a constraint to sust_retrofit_date
+-- Renewal technologies
+-- Constraint - Front end multi select back end ENUM
+-- Values:
+ALTER TABLE buildings DROP COLUMN IF EXISTS sust_renewables_tech;
+
+--Has a building had a major renovation without extenstion (captured in form)
+--Boolean yes/no - links to the the DATE
+ALTER TABLE buildings DROP COLUMN IF EXISTS sust_retrofitted;
+
+-- Generating capacity of those renewables, on selection of one of the above generate correspondening front end input for this. Pair values
+-- Constraint more than 0 less than?, integer only
+ALTER TABLE buildings DROP COLUMN IF EXISTS sust_renewables_capax;
+
+-- Biodiversity
+-- Green roof, green wall, both
+-- Constrain drop down and enum
+ALTER TABLE buildings DROP COLUMN IF EXISTS sust_biodiversity;
+
+-- Insulation, tool tip for glazing in construction to cross link
+-- Which components are insulated
+-- Cosntraint multi-entry and ENUM stored in josnb object
+-- Values; Wall, Roof, FLOOR
+ALTER TABLE buildings DROP COLUMN IF EXISTS constrctn_insulation;
+
+-- Water recycling
+-- yes / no
+ALTER TABLE buildings DROP COLUMN IF EXISTS sust_h2o_recyling;
+
+-- Rain water harvesting
+-- Does building store it's rainwater, helps combat flood risk
+-- yes / no
+ALTER TABLE buildings DROP COLUMN IF EXISTS sust_rainwater_harvest;

migrations/011.sustainability.up1-extra.sql

@@ -0,0 +1,66 @@
+-- Remove sustainability fields, update in paralell with adding new fields
+-- Last significant retrofit date YYYY
+-- Need to add a constraint to sust_retrofit_date
+ALTER TABLE buildings
+    ADD CONSTRAINT sust_retrofit_date_end CHECK (sust_retrofit_date <= DATE_PART('year', CURRENT_DATE));
+
+--Has a building had a major renovation without extenstion (captured in form)
+--Boolean yes/no - links to the the DATE
+ALTER TABLE buildings
+    ADD COLUMN IF NOT EXISTS sust_retrofitted boolean DEFAULT 'n';
+
+-- Renewal technologies
+-- Constraint - Front end multi select back end ENUM
+-- Values: Solar PV, Solar thermal, Wind, Ground sourced heat pump, Air sourced heat pump,
+CREATE TYPE sust_renewables_tech
+AS ENUM ('Solar photovoltaic',
+    'Solar thermal',
+    'Wind',
+    'Ground source heat pump',
+    'Air-source heat pump',
+    'Water source heat pump',
+    'Anaerobic digester');
+
+ALTER TABLE buildings
+    ADD COLUMN IF NOT EXISTS sust_renewables_tech sust_renewables_tech DEFAULT NULL;
+
+-- Generating capacity of those renewables, on selection of one of the above generate correspondening front end input for this. Pair values
+-- Constraint more than 0 less than 9999
+ALTER TABLE buildings
+    ADD COLUMN IF NOT EXISTS sust_renewables_capax int CONSTRAINT high_renewables_capx CHECK (sust_renewables_capax >= 0);
+
+-- Biodiversity
+-- Green roof, green wall, both
+-- Constrain drop down and enum
+CREATE TYPE sust_biodiversity
+AS ENUM ('Green roof',
+    'Green wall',
+    'Green wall & roof',
+    'Anaerobic digester');
+
+ALTER TABLE buildings
+    ADD COLUMN IF NOT EXISTS sust_biodiversity sust_biodiversity DEFAULT NULL;
+
+-- Insulation, tool tip for glazing in construction to cross link
+-- Which components are insulated
+-- Cosntraint multi-entry and ENUM stored in josnb object
+-- Values; Wall, Roof, FLOOR
+CREATE TYPE constrctn_insulation
+AS ENUM ('Cavity wall',
+    'External wall',
+    'Roof',
+    'Floor');
+
+ALTER TABLE buildings
+    ADD COLUMN IF NOT EXISTS constrctn_insulation constrctn_insulation DEFAULT NULL;
+
+-- Water recycling
+-- yes / no
+ALTER TABLE buildings
+    ADD COLUMN IF NOT EXISTS sust_h2o_recyling boolean DEFAULT 'n';
+
+-- Rain water harvesting
+-- Does building store it's rainwater, helps combat flood risk
+-- yes / no
+ALTER TABLE buildings
+    ADD COLUMN IF NOT EXISTS sust_rainwater_harvest boolean DEFAULT 'n';

migrations/README.md

@@ -54,6 +54,16 @@ Set or update passwords:
 psql -c "ALTER USER appusername WITH PASSWORD 'longsecurerandompassword';"
 ```
 
+## File naming syntax
+
+Initial up and down migrations as `###.name.up.sql` file number should be sequential 
+and incremental to last migrations file number is same for up/down. 
+
+If adjusting a prior migration syntax is:
+
+    ###.name.up.sql
+
 Syntax for adding to existing migration:
 
-`0##.filename-extension-#.up.sql`
+    0##.filename-extension-#.up.sql
+