From de927faf7c3e14752a1765f5c768b2fa15fe482b Mon Sep 17 00:00:00 2001 From: Dominic H Date: Tue, 1 Oct 2019 16:23:58 +0100 Subject: [PATCH 1/4] Delete 011.sustainability-extra.up.sql Duplicate file house keeping remove sustainability-extra --- migrations/011.sustainability-extra.up.sql | 67 ---------------------- 1 file changed, 67 deletions(-) delete mode 100644 migrations/011.sustainability-extra.up.sql diff --git a/migrations/011.sustainability-extra.up.sql b/migrations/011.sustainability-extra.up.sql deleted file mode 100644 index 854d61f2..00000000 --- a/migrations/011.sustainability-extra.up.sql +++ /dev/null @@ -1,67 +0,0 @@ --- BREEAM ratings, one of: --- - Outstanding --- - Excellent --- - Very good --- - Good --- - Pass --- - Unclassified -CREATE TYPE sust_breeam_rating -AS ENUM ('Outstanding', - 'Excellent', - 'Very good', - 'Good', - 'Pass', - 'Unclassified'); - -ALTER TABLE buildings - ADD COLUMN IF NOT EXISTS sust_breeam_rating sust_breeam_rating DEFAULT 'Unclassified'; - --- Date of BREEAM -ALTER TABLE buildings - ADD COLUMN IF NOT EXISTS sust_breeam_date smallint; - --- DEC (display energy certifcate, only applies to non domestic buildings) --- A - G -CREATE TYPE sust_dec -AS ENUM ('A', - 'B', - 'C', - 'D', - 'E', - 'F', - 'G'); - --- Date of DEC YYYY -ALTER TABLE buildings - ADD COLUMN IF NOT EXISTS sust_dec_date smallint; - - -ALTER TABLE buildings - ADD COLUMN IF NOT EXISTS sust_dec sust_dec; - --- Aggregate EPC rating (Estimated) for a building, derived from inidividual certificates --- A+ - G -CREATE TYPE sust_aggregate_estimate_epc -AS ENUM ('A', - 'B', - 'C', - 'D', - 'E', - 'F', - 'G'); - -ALTER TABLE buildings - ADD COLUMN IF NOT EXISTS sust_aggregate_estimate_epc sust_aggregate_estimate_epc; - --- Last significant retrofit date YYYY -ALTER TABLE buildings - ADD COLUMN IF NOT EXISTS sust_retrofit_date smallint; - ---How much embodied carbon? One for ML, tons CO2 int -ALTER TABLE buildings ADD COLUMN IF NOT EXISTS sust_embodied_carbon numeric(7,2); - ---Life expectancy of the building, via further analysis -ALTER TABLE buildings ADD COLUMN IF NOT EXISTS sust_life_expectancy smallint; - ---Average lifespan of typology based on statistical analysis of similar stock -ALTER TABLE buildings ADD COLUMN IF NOT EXISTS sust_lifespan_average smallint; From 53e8d893b04434cf7b80d65f80464835519445a6 Mon Sep 17 00:00:00 2001 From: Dominic H Date: Fri, 8 Nov 2019 17:48:00 +0000 Subject: [PATCH 2/4] Stashing --- migrations/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/migrations/README.md b/migrations/README.md index d770f1c4..4474a3d9 100644 --- a/migrations/README.md +++ b/migrations/README.md @@ -53,3 +53,6 @@ Set or update passwords: ```bash psql -c "ALTER USER appusername WITH PASSWORD 'longsecurerandompassword';" ``` + +# File naming syntax +Initial up and down migrations as `###.name.up.sql` file number should be sequential and incremental to last migrations file number is same for up/down. If adjusting a prior migration syntax is `###.name.up.sql` From 5ca6a6f7fe92c5b7ed692cde66b358fda8840240 Mon Sep 17 00:00:00 2001 From: dom_ucl_mb Date: Thu, 13 Feb 2020 17:22:16 +0000 Subject: [PATCH 3/4] WIP for #405 --- .../load_shapefile_to_staging.py | 140 ++++++++++++++++++ migrations/011.sustainability-extra.down.sql | 39 +++++ 2 files changed, 179 insertions(+) create mode 100644 etl/join_building_data/load_shapefile_to_staging.py create mode 100644 migrations/011.sustainability-extra.down.sql diff --git a/etl/join_building_data/load_shapefile_to_staging.py b/etl/join_building_data/load_shapefile_to_staging.py new file mode 100644 index 00000000..4b805844 --- /dev/null +++ b/etl/join_building_data/load_shapefile_to_staging.py @@ -0,0 +1,140 @@ +"""Join shapefile data to buildings + +This is effectively an example script using the HTTP API, tailored to particular collected +datasets for Camden (age data) and Fitzrovia (number of storeys). + +- read through shapes +- locate building by toid +- else locate building by representative point +- update building with data +""" +import json +import os +import sys +from functools import partial + +import fiona +import pyproj +import requests +from shapely.geometry import shape +from shapely.ops import transform + + +osgb_to_ll = partial( + pyproj.transform, + pyproj.Proj(init='epsg:27700'), + pyproj.Proj(init='epsg:4326') +) + + +def main(base_url, api_key, process, source_file): + """Read from file, update buildings + """ + with fiona.open(source_file, 'r') as source: + for feature in source: + props = feature['properties'] + + if process == "camden": + toid, data = process_camden(props) + else: + toid, data = process_fitzrovia(props) + + if data is None: + continue + + building_id = find_building(toid, feature['geometry'], base_url) + if not building_id: + print("no_match", toid, "-") + continue + + save_data(building_id, data, api_key, base_url) + + +def process_camden(props): + toid = osgb_toid(props['TOID']) + data = { + 'date_year': props['Year_C'], + 'date_source_detail': props['Date_sou_1'] + } + return toid, data + + +def process_fitzrovia(props): + toid = osgb_toid(props['TOID']) + storeys = props['Storeys'] + + if storeys is None: + return toid, None + + if props['Basement'] == 'Yes': + data = { + 'size_storeys_core': int(storeys) - 1, + 'size_storeys_basement': 1 + } + else: + data = { + 'size_storeys_core': int(storeys), + 'size_storeys_basement': 0 + } + return toid, data + + +def osgb_toid(toid): + if toid is None: + toid = "" + return "osgb" + toid.lstrip("0") + + +def save_data(building_id, data, api_key, base_url): + """Save data to a building + """ + r = requests.post( + "{}/buildings/{}.json?api_key={}".format(base_url, building_id, api_key), + json=data + ) + + +def find_building(toid, geom, base_url): + """Find building_id by TOID or location + """ + r = requests.get(base_url + "/buildings/reference", params={ + 'key': 'toid', + 'id': toid + }) + buildings = r.json() + if buildings and len(buildings) == 1: + bid = buildings[0]['building_id'] + print("match_by_toid", toid, bid) + return bid + + # try location + poly = shape(geom) + point_osgb = poly.centroid + if not poly.contains(point_osgb): + point_osgb = poly.representative_point() + + point_ll = transform(osgb_to_ll, point_osgb) + r = requests.get(base_url + "/buildings/locate", params={ + 'lng': point_ll.x, + 'lat': point_ll.y + }) + buildings = r.json() + if buildings and len(buildings) == 1: + bid = buildings[0]['building_id'] + print("match_by_location", toid, bid) + return bid + + return None + + +if __name__ == '__main__': + try: + url, api_key, process, filename = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] + except IndexError: + print( + "Usage: {} ./path/to/camden.shp".format( + os.path.basename(__file__) + )) + exit() + + main(url, api_key, process, filename) diff --git a/migrations/011.sustainability-extra.down.sql b/migrations/011.sustainability-extra.down.sql new file mode 100644 index 00000000..5accb10f --- /dev/null +++ b/migrations/011.sustainability-extra.down.sql @@ -0,0 +1,39 @@ +-- Remove sustainability fields, update in paralell with adding new fields + +-- Last significant retrofit date YYYY +-- Need to add a constraint to sust_retrofit_date + +-- Renewal technologies +-- Constraint - Front end multi select back end ENUM +-- Values: +ALTER TABLE buildings DROP COLUMN IF EXISTS sust_renewables_tech; + + +-- Generating capacity of those renewables, on selection of one of the above generate correspondening front end input for this. Pair values +-- Constraint more than 0 less than?, integer only +ALTER TABLE buildings DROP COLUMN IF EXISTS sust_renewables_capax; + + +-- Biodiversity +-- Green roof, green wall, both +-- Constrain drop down and enum +ALTER TABLE buildings DROP COLUMN IF EXISTS sust_biodiversity; + + +-- Insulation, tool tip for glazing in construction to cross link +-- Which components are insulated +-- Cosntraint multi-entry and ENUM stored in josnb object +-- Values; Wall, Roof, Floor +ALTER TABLE buildings DROP COLUMN IF EXISTS constrctn_insulation; + + +-- Water recycling +-- yes / no +ALTER TABLE buildings DROP COLUMN IF EXISTS sust_h2o_recyling; + + +-- Rain water harvesting +-- Does builing store it's rainwater +-- yes / no + +ALTER TABLE buildings DROP COLUMN IF EXISTS sust_h2o_harvest; From 6dfc82a3973e3bd6042762480cca6a76186a7593 Mon Sep 17 00:00:00 2001 From: dominijk Date: Thu, 27 Feb 2020 16:19:52 +0000 Subject: [PATCH 4/4] Ready for staging, tested on local for #405 --- ...sql => 011.sustainability.down1-extra.sql} | 16 ++--- migrations/011.sustainability.up1-extra.sql | 66 +++++++++++++++++++ 2 files changed, 72 insertions(+), 10 deletions(-) rename migrations/{011.sustainability-extra.down.sql => 011.sustainability.down1-extra.sql} (76%) create mode 100644 migrations/011.sustainability.up1-extra.sql diff --git a/migrations/011.sustainability-extra.down.sql b/migrations/011.sustainability.down1-extra.sql similarity index 76% rename from migrations/011.sustainability-extra.down.sql rename to migrations/011.sustainability.down1-extra.sql index 5accb10f..0dc39b72 100644 --- a/migrations/011.sustainability-extra.down.sql +++ b/migrations/011.sustainability.down1-extra.sql @@ -1,39 +1,35 @@ -- Remove sustainability fields, update in paralell with adding new fields - -- Last significant retrofit date YYYY -- Need to add a constraint to sust_retrofit_date - -- Renewal technologies -- Constraint - Front end multi select back end ENUM -- Values: ALTER TABLE buildings DROP COLUMN IF EXISTS sust_renewables_tech; +--Has a building had a major renovation without extenstion (captured in form) +--Boolean yes/no - links to the the DATE +ALTER TABLE buildings DROP COLUMN IF EXISTS sust_retrofitted; -- Generating capacity of those renewables, on selection of one of the above generate correspondening front end input for this. Pair values -- Constraint more than 0 less than?, integer only ALTER TABLE buildings DROP COLUMN IF EXISTS sust_renewables_capax; - -- Biodiversity -- Green roof, green wall, both -- Constrain drop down and enum ALTER TABLE buildings DROP COLUMN IF EXISTS sust_biodiversity; - -- Insulation, tool tip for glazing in construction to cross link -- Which components are insulated -- Cosntraint multi-entry and ENUM stored in josnb object --- Values; Wall, Roof, Floor +-- Values; Wall, Roof, FLOOR ALTER TABLE buildings DROP COLUMN IF EXISTS constrctn_insulation; - -- Water recycling -- yes / no ALTER TABLE buildings DROP COLUMN IF EXISTS sust_h2o_recyling; - -- Rain water harvesting --- Does builing store it's rainwater +-- Does building store it's rainwater, helps combat flood risk -- yes / no - -ALTER TABLE buildings DROP COLUMN IF EXISTS sust_h2o_harvest; +ALTER TABLE buildings DROP COLUMN IF EXISTS sust_rainwater_harvest; diff --git a/migrations/011.sustainability.up1-extra.sql b/migrations/011.sustainability.up1-extra.sql new file mode 100644 index 00000000..11cc62d5 --- /dev/null +++ b/migrations/011.sustainability.up1-extra.sql @@ -0,0 +1,66 @@ +-- Remove sustainability fields, update in paralell with adding new fields +-- Last significant retrofit date YYYY +-- Need to add a constraint to sust_retrofit_date +ALTER TABLE buildings + ADD CONSTRAINT sust_retrofit_date_end CHECK (sust_retrofit_date <= DATE_PART('year', CURRENT_DATE)); + +--Has a building had a major renovation without extenstion (captured in form) +--Boolean yes/no - links to the the DATE +ALTER TABLE buildings + ADD COLUMN IF NOT EXISTS sust_retrofitted boolean DEFAULT 'n'; + +-- Renewal technologies +-- Constraint - Front end multi select back end ENUM +-- Values: Solar PV, Solar thermal, Wind, Ground sourced heat pump, Air sourced heat pump, +CREATE TYPE sust_renewables_tech +AS ENUM ('Solar photovoltaic', + 'Solar thermal', + 'Wind', + 'Ground source heat pump', + 'Air-source heat pump', + 'Water source heat pump', + 'Anaerobic digester'); + +ALTER TABLE buildings + ADD COLUMN IF NOT EXISTS sust_renewables_tech sust_renewables_tech DEFAULT NULL; + +-- Generating capacity of those renewables, on selection of one of the above generate correspondening front end input for this. Pair values +-- Constraint more than 0 less than 9999 +ALTER TABLE buildings + ADD COLUMN IF NOT EXISTS sust_renewables_capax int CONSTRAINT high_renewables_capx CHECK (sust_renewables_capax >= 0); + +-- Biodiversity +-- Green roof, green wall, both +-- Constrain drop down and enum +CREATE TYPE sust_biodiversity +AS ENUM ('Green roof', + 'Green wall', + 'Green wall & roof', + 'Anaerobic digester'); + +ALTER TABLE buildings + ADD COLUMN IF NOT EXISTS sust_biodiversity sust_biodiversity DEFAULT NULL; + +-- Insulation, tool tip for glazing in construction to cross link +-- Which components are insulated +-- Cosntraint multi-entry and ENUM stored in josnb object +-- Values; Wall, Roof, FLOOR +CREATE TYPE constrctn_insulation +AS ENUM ('Cavity wall', + 'External wall', + 'Roof', + 'Floor'); + +ALTER TABLE buildings + ADD COLUMN IF NOT EXISTS constrctn_insulation constrctn_insulation DEFAULT NULL; + +-- Water recycling +-- yes / no +ALTER TABLE buildings + ADD COLUMN IF NOT EXISTS sust_h2o_recyling boolean DEFAULT 'n'; + +-- Rain water harvesting +-- Does building store it's rainwater, helps combat flood risk +-- yes / no +ALTER TABLE buildings + ADD COLUMN IF NOT EXISTS sust_rainwater_harvest boolean DEFAULT 'n';