From 9943bdd9a54b439946320ff609b184243fae909f Mon Sep 17 00:00:00 2001 From: Dominic H Date: Wed, 2 Oct 2019 16:35:56 +0100 Subject: [PATCH 01/10] Test load to staging - Copy of main load_csv.py with edit to get round ssh error --- etl/join_building_data/load_csv_to_staging.py | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 etl/join_building_data/load_csv_to_staging.py diff --git a/etl/join_building_data/load_csv_to_staging.py b/etl/join_building_data/load_csv_to_staging.py new file mode 100644 index 00000000..d1d9bbe1 --- /dev/null +++ b/etl/join_building_data/load_csv_to_staging.py @@ -0,0 +1,108 @@ +"""Join csv data to buildings +Example usage (replace URL with test/staging/localhost as necessary, API key with real key for +the appropriate site): + python load_csv.py \ + https://colouring.london \ + a0a00000-0a00-0aaa-a0a0-0000aaaa0000 \ + data.csv +This script uses the HTTP API, and can process CSV files which identify buildings by id, TOID, +UPRN. +The process: + - assume first line of the CSV is a header, where column names are either + - building identifiers - one of: + - building_id + - toid + - uprn + - building data field names + - read through lines of CSV: + - use building id if provided + - else lookup by toid + - else lookup by uprn + - else locate building by representative point + - update building +TODO extend to allow latitude,longitude or easting,northing columns and lookup by location. +""" +import csv +import json +import os +import sys + +import requests +session = requests.Session() +session.verify = False + +def main(base_url, api_key, source_file): + """Read from file, update buildings + """ + with open(source_file, 'r') as source: + reader = csv.DictReader(source) + for line in reader: + building_id = find_building(line, base_url) + + if building_id is None: + continue + + response_code, response_data = update_building(building_id, line, api_key, base_url) + if response_code != 200: + print('ERROR', building_id, response_code, response_data) + + +def update_building(building_id, data, api_key, base_url): + """Save data to a building + """ + r = requests.post( + "{}/api/buildings/{}.json".format(base_url, building_id), + params={'api_key': api_key}, + json=data, + verify=False + ) + return r.status_code, r.json() + + +def find_building(data, base_url): + if 'toid' in data: + building_id = find_by_reference(base_url, 'toid', data['toid']) + if building_id is not None: + print("match_by_toid", data['toid'], building_id) + return building_id + + if 'uprn' in data: + building_id = find_by_reference(base_url, 'uprn', data['uprn']) + if building_id is not None: + print("match_by_uprn", data['uprn'], building_id) + return building_id + + print("no_match", data) + return None + + +def find_by_reference(base_url, ref_key, ref_id): + """Find building_id by TOID or UPRN + """ + r = requests.get("{}/api/buildings/reference".format(base_url), params={ + 'key': ref_key, + 'id': ref_id, + }, + verify=False + ) + buildings = r.json() + + if buildings and 'error' not in buildings and len(buildings) == 1: + building_id = buildings[0]['building_id'] + else: + building_id = None + + return building_id + + +if __name__ == '__main__': + try: + url, api_key, filename = sys.argv[1], sys.argv[2], sys.argv[3] + except IndexError: + print( + "Usage: {} ./path/to/data.csv".format( + os.path.basename(__file__) + )) + exit() + + main(url, api_key, filename) From f100e7b7517042286d9956c86f03701caa32fa3e Mon Sep 17 00:00:00 2001 From: Tom Russell Date: Thu, 3 Oct 2019 07:32:54 +0100 Subject: [PATCH 02/10] Add attribution note to welcome box --- app/src/frontend/pages/welcome.tsx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/src/frontend/pages/welcome.tsx b/app/src/frontend/pages/welcome.tsx index 7d4bb04d..f2c99c4a 100644 --- a/app/src/frontend/pages/welcome.tsx +++ b/app/src/frontend/pages/welcome.tsx @@ -17,11 +17,18 @@ const Welcome = () => ( volunteers of all ages and abilities to test and provide feedback on the site as we build it.

+

+ All of the data we collect is made openly available – + please read our data ethics policy and + credit Colouring London if you use or share our maps or data. +

Start Colouring Here! +

Colouring London collaborating organisations: The Bartlett UCL, Ordnance Survey, Historic England, Greater London Authority +

); From 8e3d3c56223e57f695d012dc56655e89fa5ecd24 Mon Sep 17 00:00:00 2001 From: Tom Russell Date: Thu, 3 Oct 2019 07:36:17 +0100 Subject: [PATCH 03/10] Add reference links to agreements on account page --- app/src/frontend/user/my-account.tsx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/src/frontend/user/my-account.tsx b/app/src/frontend/user/my-account.tsx index 0659d217..b6368604 100644 --- a/app/src/frontend/user/my-account.tsx +++ b/app/src/frontend/user/my-account.tsx @@ -107,11 +107,17 @@ class MyAccountPage extends Component { // TODO: add proper types

Welcome, {this.props.user.username}!

- Colouring London is under active development, Please discuss + Colouring London is under active development. Please discuss suggestions for improvements and report issues or problems. +

+

+ For reference, here are the privacy policy, contributor agreement and data accuracy agreement.

From bcfd89196c47f052143ea520cb5bda23dedc1448 Mon Sep 17 00:00:00 2001 From: Tom Russell Date: Thu, 3 Oct 2019 07:39:13 +0100 Subject: [PATCH 04/10] Add licensing/attribution note to downloads page --- app/src/frontend/pages/data-extracts.tsx | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/app/src/frontend/pages/data-extracts.tsx b/app/src/frontend/pages/data-extracts.tsx index 0b8220cb..37de3f76 100644 --- a/app/src/frontend/pages/data-extracts.tsx +++ b/app/src/frontend/pages/data-extracts.tsx @@ -1,7 +1,7 @@ import React, { FunctionComponent } from 'react'; import { dateReviver } from '../../helpers'; -import { NavLink } from 'react-router-dom'; +import { Link } from 'react-router-dom'; interface ExtractViewModel { extract_id: number; @@ -33,7 +33,7 @@ export default class DataExtracts extends React.Component<{}, DataExtractsState> .sort((a, b) => a.extracted_on.valueOf() - b.extracted_on.valueOf()) .reverse(); - + this.setState({ extracts: extracts, latestExtract: extracts[0], previousExtracts: extracts.slice(1) }); } @@ -44,8 +44,18 @@ export default class DataExtracts extends React.Component<{}, DataExtractsState>

Open data extracts

-

Choose one of the links below to download an archive containing the open data collected on the Colouring London platform

-

By downloading data extracts from this site, you agree to the data accuracy agreement

+

+ Colouring London contributions are open data, licensed under the Open Data Commons Open Database License (ODbL) by Colouring London contributors. +

+

+ You are free to copy, distribute, transmit and adapt our data, as long as you credit Colouring London and our contributors. If you alter or build upon our data, you may distribute the result only under the same licence. +

+

+ Choose one of the links below to download an archive containing the open data collected on the Colouring London platform. +

+

+ By downloading data extracts from this site, you agree to the data accuracy agreement . +

{ this.state.extracts == undefined ?

Loading extracts...

: @@ -69,14 +79,14 @@ export default class DataExtracts extends React.Component<{}, DataExtractsState>

Older extracts

    { - this.state.previousExtracts.map(e => + this.state.previousExtracts.map(e =>
    • ) }
- ) : + ) : null } From 386fb22c10dd7d0ed8a0dadac27f0187c00e2284 Mon Sep 17 00:00:00 2001 From: Tom Russell Date: Thu, 3 Oct 2019 07:49:17 +0100 Subject: [PATCH 05/10] Link contact email as mailto --- app/src/frontend/pages/contact.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/src/frontend/pages/contact.tsx b/app/src/frontend/pages/contact.tsx index 5106437a..62c50b83 100644 --- a/app/src/frontend/pages/contact.tsx +++ b/app/src/frontend/pages/contact.tsx @@ -9,13 +9,13 @@ const ContactPage = () => (

Colouring London has been designed as a sustainable, low-cost model for knowledge exchange/open data platforms able to be reproduced by other towns and cities using our open platform code.

-

It is being developed by a small, dedicated team at UCL. We are unable to answer individual queries but welcome constructive comments on how to improve the site, and on other types of data and new features you might like to see.​

+

It is being developed by a small, dedicated team at UCL. We are unable to answer individual queries but welcome constructive comments on how to improve the site, and on other types of data and new features you might like to see.

You can send us comments or ask questions on our discussion threads at https://discuss.colouring.london/.

- +

To view our technical site and platform code please visit Github at: https://github.com/tomalrussell/colouring-london.

-

For press enquiries please contact the Bartlett Press and Communications team at architecture@ucl.ac.uk.

+

For press enquiries please contact the Bartlett Press and Communications team at architecture@ucl.ac.uk

From 67114ebd88472743fc9997ae14acbd9bde37bfb2 Mon Sep 17 00:00:00 2001 From: Tom Russell Date: Thu, 3 Oct 2019 07:50:04 +0100 Subject: [PATCH 06/10] Add attribution note to contact page --- app/src/frontend/pages/contact.tsx | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/app/src/frontend/pages/contact.tsx b/app/src/frontend/pages/contact.tsx index 62c50b83..eb6fb99e 100644 --- a/app/src/frontend/pages/contact.tsx +++ b/app/src/frontend/pages/contact.tsx @@ -17,6 +17,17 @@ const ContactPage = () => (

For press enquiries please contact the Bartlett Press and Communications team at architecture@ucl.ac.uk

+

+ If you capture images from the maps on Colouring London, please credit our + contributors (who collected the data) and Ordnance Survey + (who provided the basemaps and building geometries) as follows: +

+

+ 


+            Colouring London https://colouring.london Building attribute data is © Colouring London contributors. Maps contain OS data © Crown copyright: OS Maps baselayers and building outlines.
+
+

+

From 0240e94adc7fe6819d34b21a7164ceb0d9b5d417 Mon Sep 17 00:00:00 2001 From: Maciej Ziarkowski Date: Tue, 8 Oct 2019 17:40:30 +0100 Subject: [PATCH 07/10] Allow zoom level 19 with OS basemap interpolation --- app/src/frontend/map/map.tsx | 10 +++++++--- app/src/tiles/rendererDefinition.ts | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/app/src/frontend/map/map.tsx b/app/src/frontend/map/map.tsx index 84697c36..5a8a8789 100644 --- a/app/src/frontend/map/map.tsx +++ b/app/src/frontend/map/map.tsx @@ -112,10 +112,12 @@ class ColouringMap extends Component { // const baseLayer = ; const buildingsBaseUrl = `/tiles/base_${this.state.theme}/{z}/{x}/{y}{r}.png`; - const buildingBaseLayer = ; + const buildingBaseLayer = ; // colour-data tiles const cat = this.props.category; @@ -136,6 +138,7 @@ class ColouringMap extends Component { // key={tileset} url={`/tiles/${tileset}/{z}/{x}/{y}{r}.png?rev=${rev}`} minZoom={9} + maxZoom={19} /> : null; @@ -144,7 +147,8 @@ class ColouringMap extends Component { // : null; @@ -157,7 +161,7 @@ class ColouringMap extends Component { // center={position} zoom={this.state.zoom} minZoom={9} - maxZoom={18} + maxZoom={19} doubleClickZoom={false} zoomControl={false} attributionControl={false} diff --git a/app/src/tiles/rendererDefinition.ts b/app/src/tiles/rendererDefinition.ts index 16b6865e..db3aef84 100644 --- a/app/src/tiles/rendererDefinition.ts +++ b/app/src/tiles/rendererDefinition.ts @@ -34,7 +34,7 @@ const tileCache = new TileCache( { tilesets: ['date_year', 'size_storeys', 'location', 'likes', 'conservation_area', 'sust_dec', 'building_attachment_form'], minZoom: 9, - maxZoom: 18, + maxZoom: 19, scales: [1, 2] }, ({ tileset, z }: TileParams) => (tileset === 'date_year' && z <= 16) || From e00d9126863d0d2560099262295f8490098b245b Mon Sep 17 00:00:00 2001 From: Dominic H Date: Fri, 8 Nov 2019 12:31:40 +0000 Subject: [PATCH 08/10] Enable python upload to staging - Based on load_csv.py with edit for staging --- etl/join_building_data/load_csv_to_staging.py | 1 + 1 file changed, 1 insertion(+) diff --git a/etl/join_building_data/load_csv_to_staging.py b/etl/join_building_data/load_csv_to_staging.py index d1d9bbe1..84bb6435 100644 --- a/etl/join_building_data/load_csv_to_staging.py +++ b/etl/join_building_data/load_csv_to_staging.py @@ -56,6 +56,7 @@ def update_building(building_id, data, api_key, base_url): json=data, verify=False ) + print(r) return r.status_code, r.json() From 1d372823edecffbb8927f80e9dd4bbef2a267398 Mon Sep 17 00:00:00 2001 From: Dominic H Date: Fri, 8 Nov 2019 12:36:54 +0000 Subject: [PATCH 09/10] Adjusts guide notes no code change --- etl/join_building_data/load_csv_to_staging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etl/join_building_data/load_csv_to_staging.py b/etl/join_building_data/load_csv_to_staging.py index 84bb6435..413cd533 100644 --- a/etl/join_building_data/load_csv_to_staging.py +++ b/etl/join_building_data/load_csv_to_staging.py @@ -1,8 +1,8 @@ """Join csv data to buildings Example usage (replace URL with test/staging/localhost as necessary, API key with real key for the appropriate site): - python load_csv.py \ - https://colouring.london \ + python load_csv_to_staging.py \ + https://clstaging.casa.ucl.ac.uk \ a0a00000-0a00-0aaa-a0a0-0000aaaa0000 \ data.csv This script uses the HTTP API, and can process CSV files which identify buildings by id, TOID, From 215f6bdee000bbaa29a5e193442bd302aee0ff78 Mon Sep 17 00:00:00 2001 From: Tom Russell Date: Tue, 12 Nov 2019 10:49:15 +0000 Subject: [PATCH 10/10] Fix closing Link tag --- app/src/frontend/pages/data-extracts.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/frontend/pages/data-extracts.tsx b/app/src/frontend/pages/data-extracts.tsx index 43baf2a7..16d11748 100644 --- a/app/src/frontend/pages/data-extracts.tsx +++ b/app/src/frontend/pages/data-extracts.tsx @@ -57,7 +57,7 @@ export default class DataExtracts extends React.Component<{}, DataExtractsState> Choose one of the links below to download an archive containing the open data collected on the Colouring London platform.

- By downloading data extracts from this site, you agree to the data accuracy agreement and the Ordnance Survey terms of UPRN usage. + By downloading data extracts from this site, you agree to the data accuracy agreement and the Ordnance Survey terms of UPRN usage.

{