From 4a7eb350f2a64b3e82ed831c67f8bc8b09519351 Mon Sep 17 00:00:00 2001 From: dominic Date: Fri, 30 Aug 2019 13:37:56 +0100 Subject: [PATCH 1/8] WIP for #363 - Needs landuse table from #358 --- migrations/014.type.down.sql | 9 +++++++++ migrations/014.type.up.sql | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 migrations/014.type.down.sql create mode 100644 migrations/014.type.up.sql diff --git a/migrations/014.type.down.sql b/migrations/014.type.down.sql new file mode 100644 index 00000000..776a86d3 --- /dev/null +++ b/migrations/014.type.down.sql @@ -0,0 +1,9 @@ +--Original building use, this is same NLUD fields as current building use +ALTER TABLE buildings DROP COLUMN IF EXISTS original_building_use; + +-- Building attachment, ENUM: Detached, Semi-detached, End-Terrace, Mid-Terrace +ALTER TABLE buildings DROP COLUMN IF EXISTS building_attachment_form; + +-- [Disabled for launch] Date of change of use +-- This needs to pair with demolition +ALTER TABLE buildings DROP COLUMN IF EXISTS date_change_building_use; diff --git a/migrations/014.type.up.sql b/migrations/014.type.up.sql new file mode 100644 index 00000000..6326efdb --- /dev/null +++ b/migrations/014.type.up.sql @@ -0,0 +1,19 @@ +--Original building use, this is same NLUD fields as current building use +ALTER TABLE buildings + ADD COLUMN IF NOT EXISTS original_building_use; + +-- Building attachment, ENUM: Detached, Semi-detached, End-Terrace, Mid-Terrace +CREATE TYPE building_attachment_form +AS ENUM ('Deatched', + 'Semi-Detached', + 'End-Terrace', + 'Mid-Terrace', + 'Unknown'); + +ALTER TABLE buildings + ADD COLUMN IF NOT EXISTS building_attachment_form building_attachment_form DEFAULT 'Unknown'; + +-- [Disabled for launch] Date of change of use +-- This needs to pair with demolition +ALTER TABLE buildings + ADD COLUMN IF NOT EXISTS date_change_building_use smallint; From 13fbea77d992c331c4e26bf3f8877f91f8a40305 Mon Sep 17 00:00:00 2001 From: Dominic H Date: Wed, 18 Sep 2019 11:08:02 +0100 Subject: [PATCH 2/8] Typo correction closes #413 --- migrations/011.sustainability-extra.up.sql | 67 ++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 migrations/011.sustainability-extra.up.sql diff --git a/migrations/011.sustainability-extra.up.sql b/migrations/011.sustainability-extra.up.sql new file mode 100644 index 00000000..854d61f2 --- /dev/null +++ b/migrations/011.sustainability-extra.up.sql @@ -0,0 +1,67 @@ +-- BREEAM ratings, one of: +-- - Outstanding +-- - Excellent +-- - Very good +-- - Good +-- - Pass +-- - Unclassified +CREATE TYPE sust_breeam_rating +AS ENUM ('Outstanding', + 'Excellent', + 'Very good', + 'Good', + 'Pass', + 'Unclassified'); + +ALTER TABLE buildings + ADD COLUMN IF NOT EXISTS sust_breeam_rating sust_breeam_rating DEFAULT 'Unclassified'; + +-- Date of BREEAM +ALTER TABLE buildings + ADD COLUMN IF NOT EXISTS sust_breeam_date smallint; + +-- DEC (display energy certifcate, only applies to non domestic buildings) +-- A - G +CREATE TYPE sust_dec +AS ENUM ('A', + 'B', + 'C', + 'D', + 'E', + 'F', + 'G'); + +-- Date of DEC YYYY +ALTER TABLE buildings + ADD COLUMN IF NOT EXISTS sust_dec_date smallint; + + +ALTER TABLE buildings + ADD COLUMN IF NOT EXISTS sust_dec sust_dec; + +-- Aggregate EPC rating (Estimated) for a building, derived from inidividual certificates +-- A+ - G +CREATE TYPE sust_aggregate_estimate_epc +AS ENUM ('A', + 'B', + 'C', + 'D', + 'E', + 'F', + 'G'); + +ALTER TABLE buildings + ADD COLUMN IF NOT EXISTS sust_aggregate_estimate_epc sust_aggregate_estimate_epc; + +-- Last significant retrofit date YYYY +ALTER TABLE buildings + ADD COLUMN IF NOT EXISTS sust_retrofit_date smallint; + +--How much embodied carbon? One for ML, tons CO2 int +ALTER TABLE buildings ADD COLUMN IF NOT EXISTS sust_embodied_carbon numeric(7,2); + +--Life expectancy of the building, via further analysis +ALTER TABLE buildings ADD COLUMN IF NOT EXISTS sust_life_expectancy smallint; + +--Average lifespan of typology based on statistical analysis of similar stock +ALTER TABLE buildings ADD COLUMN IF NOT EXISTS sust_lifespan_average smallint; From 97d47d54acefe5acf2c22e33acaad7f7f15121b8 Mon Sep 17 00:00:00 2001 From: Dominic H Date: Fri, 27 Sep 2019 14:47:16 +0100 Subject: [PATCH 3/8] Readies #363 for launch --- migrations/014.type-extension-1.up.sql | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 migrations/014.type-extension-1.up.sql diff --git a/migrations/014.type-extension-1.up.sql b/migrations/014.type-extension-1.up.sql new file mode 100644 index 00000000..6326efdb --- /dev/null +++ b/migrations/014.type-extension-1.up.sql @@ -0,0 +1,19 @@ +--Original building use, this is same NLUD fields as current building use +ALTER TABLE buildings + ADD COLUMN IF NOT EXISTS original_building_use; + +-- Building attachment, ENUM: Detached, Semi-detached, End-Terrace, Mid-Terrace +CREATE TYPE building_attachment_form +AS ENUM ('Deatched', + 'Semi-Detached', + 'End-Terrace', + 'Mid-Terrace', + 'Unknown'); + +ALTER TABLE buildings + ADD COLUMN IF NOT EXISTS building_attachment_form building_attachment_form DEFAULT 'Unknown'; + +-- [Disabled for launch] Date of change of use +-- This needs to pair with demolition +ALTER TABLE buildings + ADD COLUMN IF NOT EXISTS date_change_building_use smallint; From 8e6e94e23b9f3355248bb3e9c236b8d3783da5c5 Mon Sep 17 00:00:00 2001 From: Dominic H Date: Fri, 27 Sep 2019 14:51:41 +0100 Subject: [PATCH 4/8] Readies #363 for launch --- migrations/014.type-extension-1.up.sql | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 migrations/014.type-extension-1.up.sql diff --git a/migrations/014.type-extension-1.up.sql b/migrations/014.type-extension-1.up.sql deleted file mode 100644 index 6326efdb..00000000 --- a/migrations/014.type-extension-1.up.sql +++ /dev/null @@ -1,19 +0,0 @@ ---Original building use, this is same NLUD fields as current building use -ALTER TABLE buildings - ADD COLUMN IF NOT EXISTS original_building_use; - --- Building attachment, ENUM: Detached, Semi-detached, End-Terrace, Mid-Terrace -CREATE TYPE building_attachment_form -AS ENUM ('Deatched', - 'Semi-Detached', - 'End-Terrace', - 'Mid-Terrace', - 'Unknown'); - -ALTER TABLE buildings - ADD COLUMN IF NOT EXISTS building_attachment_form building_attachment_form DEFAULT 'Unknown'; - --- [Disabled for launch] Date of change of use --- This needs to pair with demolition -ALTER TABLE buildings - ADD COLUMN IF NOT EXISTS date_change_building_use smallint; From b4cbe77ea2a48756ebd5562991bcd4b37dec0497 Mon Sep 17 00:00:00 2001 From: dominic Date: Fri, 27 Sep 2019 15:03:20 +0100 Subject: [PATCH 5/8] Update 011.sustainability.up.sql --- migrations/011.sustainability.up.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/011.sustainability.up.sql b/migrations/011.sustainability.up.sql index c76d0133..854d61f2 100644 --- a/migrations/011.sustainability.up.sql +++ b/migrations/011.sustainability.up.sql @@ -64,4 +64,4 @@ ALTER TABLE buildings ADD COLUMN IF NOT EXISTS sust_embodied_carbon numeric(7,2) ALTER TABLE buildings ADD COLUMN IF NOT EXISTS sust_life_expectancy smallint; --Average lifespan of typology based on statistical analysis of similar stock -ALTER TABLE buildings ADD COLUMN IF NOTE EXISTS sust_lifespan_average smallint; +ALTER TABLE buildings ADD COLUMN IF NOT EXISTS sust_lifespan_average smallint; From f25942904130e29af20534535854f88a3417ca2f Mon Sep 17 00:00:00 2001 From: Dominic H Date: Fri, 27 Sep 2019 17:33:37 +0100 Subject: [PATCH 6/8] Ensure this file is on PR --- migrations/014.type.up.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/014.type.up.sql b/migrations/014.type.up.sql index 6326efdb..496ae71d 100644 --- a/migrations/014.type.up.sql +++ b/migrations/014.type.up.sql @@ -4,7 +4,7 @@ ALTER TABLE buildings -- Building attachment, ENUM: Detached, Semi-detached, End-Terrace, Mid-Terrace CREATE TYPE building_attachment_form -AS ENUM ('Deatched', +AS ENUM ('Detached', 'Semi-Detached', 'End-Terrace', 'Mid-Terrace', From c055bea38ba0d07a153e4b4bc2b7744f0d1a57a5 Mon Sep 17 00:00:00 2001 From: Tom Russell Date: Mon, 30 Sep 2019 10:39:43 +0100 Subject: [PATCH 7/8] Add generic CSV upload script (rename load_data to load_shapefile) --- etl/join_building_data/load_csv.py | 96 +++++++++++++++++++ .../{load_data.py => load_shapefile.py} | 0 2 files changed, 96 insertions(+) create mode 100644 etl/join_building_data/load_csv.py rename etl/join_building_data/{load_data.py => load_shapefile.py} (100%) diff --git a/etl/join_building_data/load_csv.py b/etl/join_building_data/load_csv.py new file mode 100644 index 00000000..3d3e7388 --- /dev/null +++ b/etl/join_building_data/load_csv.py @@ -0,0 +1,96 @@ +"""Join csv data to buildings + +This script uses the HTTP API, and can process CSV files which identify buildings by id, TOID, +UPRN. + +The process: + - assume first line of the CSV is a header, where column names are either + - building identifiers - one of: + - building_id + - toid + - uprn + - building data field names + - read through lines of CSV: + - use building id if provided + - else lookup by toid + - else lookup by uprn + - else locate building by representative point + - update building + +TODO extend to allow latitude,longitude or easting,northing columns and lookup by location. + +""" +import csv +import json +import os +import sys + +import requests + + +def main(base_url, api_key, source_file): + """Read from file, update buildings + """ + with open(source_file, 'r') as source: + reader = csv.reader(source) + for line in reader: + building_id = find_building(line, base_url) + + if building_id is None: + continue + + update_building(building_id, line, api_key, base_url) + + +def update_building(building_id, data, api_key, base_url): + """Save data to a building + """ + r = requests.post( + "{}/buildings/{}.json?api_key={}".format(base_url, building_id, api_key), + json=data + ) + + +def find_building(data, base_url): + if 'toid' in data: + building_id = find_by_reference(base_url, 'toid', data['toid']) + print("match_by_toid", data['toid'], building_id) + return building_id + + if 'uprn' in data: + building_id = find_by_reference(base_url, 'uprn', data['uprn']) + print("match_by_uprn", data['uprn'], building_id) + return building_id + + print("no_match", data) + return None + + +def find_by_reference(base_url, ref_key, ref_id): + """Find building_id by TOID or UPRN + """ + r = requests.get(base_url + "/buildings/reference", params={ + 'key': ref_key, + 'id': ref_id + }) + buildings = r.json() + + if buildings and len(buildings) == 1: + building_id = buildings[0]['building_id'] + else: + building_id = None + + return building_id + + +if __name__ == '__main__': + try: + url, api_key, filename = sys.argv[1], sys.argv[2], sys.argv[3] + except IndexError: + print( + "Usage: {} ./path/to/data.csv".format( + os.path.basename(__file__) + )) + exit() + + main(url, api_key, filename) diff --git a/etl/join_building_data/load_data.py b/etl/join_building_data/load_shapefile.py similarity index 100% rename from etl/join_building_data/load_data.py rename to etl/join_building_data/load_shapefile.py From ea150250a3108d0a04207cfa4e5bc0e9ef594a8a Mon Sep 17 00:00:00 2001 From: Dominic H Date: Tue, 1 Oct 2019 16:30:15 +0100 Subject: [PATCH 8/8] File naming consistency Sequential numbering for migrations - Also removed sustainability-up-extras as original sustainbility file had not been run --- migrations/011.sustainability-extra.up.sql | 67 ------------------- .../{014.type.down.sql => 012.type.down.sql} | 0 .../{014.type.up.sql => 012.type.up.sql} | 0 3 files changed, 67 deletions(-) delete mode 100644 migrations/011.sustainability-extra.up.sql rename migrations/{014.type.down.sql => 012.type.down.sql} (100%) rename migrations/{014.type.up.sql => 012.type.up.sql} (100%) diff --git a/migrations/011.sustainability-extra.up.sql b/migrations/011.sustainability-extra.up.sql deleted file mode 100644 index 854d61f2..00000000 --- a/migrations/011.sustainability-extra.up.sql +++ /dev/null @@ -1,67 +0,0 @@ --- BREEAM ratings, one of: --- - Outstanding --- - Excellent --- - Very good --- - Good --- - Pass --- - Unclassified -CREATE TYPE sust_breeam_rating -AS ENUM ('Outstanding', - 'Excellent', - 'Very good', - 'Good', - 'Pass', - 'Unclassified'); - -ALTER TABLE buildings - ADD COLUMN IF NOT EXISTS sust_breeam_rating sust_breeam_rating DEFAULT 'Unclassified'; - --- Date of BREEAM -ALTER TABLE buildings - ADD COLUMN IF NOT EXISTS sust_breeam_date smallint; - --- DEC (display energy certifcate, only applies to non domestic buildings) --- A - G -CREATE TYPE sust_dec -AS ENUM ('A', - 'B', - 'C', - 'D', - 'E', - 'F', - 'G'); - --- Date of DEC YYYY -ALTER TABLE buildings - ADD COLUMN IF NOT EXISTS sust_dec_date smallint; - - -ALTER TABLE buildings - ADD COLUMN IF NOT EXISTS sust_dec sust_dec; - --- Aggregate EPC rating (Estimated) for a building, derived from inidividual certificates --- A+ - G -CREATE TYPE sust_aggregate_estimate_epc -AS ENUM ('A', - 'B', - 'C', - 'D', - 'E', - 'F', - 'G'); - -ALTER TABLE buildings - ADD COLUMN IF NOT EXISTS sust_aggregate_estimate_epc sust_aggregate_estimate_epc; - --- Last significant retrofit date YYYY -ALTER TABLE buildings - ADD COLUMN IF NOT EXISTS sust_retrofit_date smallint; - ---How much embodied carbon? One for ML, tons CO2 int -ALTER TABLE buildings ADD COLUMN IF NOT EXISTS sust_embodied_carbon numeric(7,2); - ---Life expectancy of the building, via further analysis -ALTER TABLE buildings ADD COLUMN IF NOT EXISTS sust_life_expectancy smallint; - ---Average lifespan of typology based on statistical analysis of similar stock -ALTER TABLE buildings ADD COLUMN IF NOT EXISTS sust_lifespan_average smallint; diff --git a/migrations/014.type.down.sql b/migrations/012.type.down.sql similarity index 100% rename from migrations/014.type.down.sql rename to migrations/012.type.down.sql diff --git a/migrations/014.type.up.sql b/migrations/012.type.up.sql similarity index 100% rename from migrations/014.type.up.sql rename to migrations/012.type.up.sql