Merge branch 'master' of github.com:tomalrussell/colouring-london

.gitignore

@@ -7,6 +7,7 @@ config.json
 ecosystem.config.js
 app/start.sh
 html/norton*.html
+.vscode
 
 # Data
 etl/cache/*

app/map_styles/polygon.xml

@@ -345,13 +345,17 @@
         </Rule>
     </Style>
     <Style name="landuse">
+        <Rule>
+            <Filter>[current_landuse_order] = "Agriculture And Fisheries"</Filter>
+            <PolygonSymbolizer fill="#52403c" />
+        </Rule>
         <Rule>
             <Filter>[current_landuse_order] = "Recreation And Leisure"</Filter>
             <PolygonSymbolizer fill="#ffbfbf" />
         </Rule>
         <Rule>
             <Filter>[current_landuse_order] = "Transport"</Filter>
-            <PolygonSymbolizer fill="#bee8ff" />
+            <PolygonSymbolizer fill="#b3de69" />
         </Rule>
         <Rule>
             <Filter>[current_landuse_order] = "Utilities And Infrastructure"</Filter>
@@ -373,6 +377,10 @@
             <Filter>[current_landuse_order] = "Industry And Business"</Filter>
             <PolygonSymbolizer fill="#f5f58f" />
         </Rule>
+        <Rule>
+            <Filter>[current_landuse_order] = "Vacant And Derelict"</Filter>
+            <PolygonSymbolizer fill="#ffffff" />
+        </Rule>
         <Rule>
             <Filter>[current_landuse_order] = "Defence"</Filter>
             <PolygonSymbolizer fill="#898944" />

app/package.json

@@ -17,6 +17,7 @@
     "@fortawesome/free-solid-svg-icons": "^5.13.0",
     "@fortawesome/react-fontawesome": "^0.1.9",
     "@mapbox/sphericalmercator": "^1.1.0",
+    "babel-runtime": "^6.26.0",
     "body-parser": "^1.19.0",
     "bootstrap": "^4.4.1",
     "connect-pg-simple": "^6.1.0",
@@ -59,7 +60,7 @@
     "eslint": "^5.16.0",
     "eslint-plugin-jest": "^22.21.0",
     "eslint-plugin-react": "^7.19.0",
-    "razzle": "^3.0.0",
+    "razzle": "^3.1.3",
     "razzle-plugin-typescript": "^3.0.0",
     "ts-jest": "^24.3.0",
     "typescript": "^3.8.3"

app/src/frontend/building/data-containers/use.tsx

@@ -29,7 +29,7 @@ const UseView: React.FunctionComponent<CategoryViewProps> = (props) => (
         />
         {
             props.mode != 'view' &&
-            <InfoBox msg="Land use order is automatically derived from the land use groups"></InfoBox>
+            <InfoBox msg="Land use order, shown below, is automatically derived from the land use groups"></InfoBox>
         }
         <DataEntry
             title={dataFields.current_landuse_order.title}

app/src/frontend/map/legend.tsx

@@ -22,13 +22,15 @@ const LEGEND_CONFIG = {
         elements: [
             { color: '#4a54a6', text: 'Residential' },
             { color: '#e5050d', text: 'Mixed Use' },
-            { color: '#ff8c00', text: 'Retail' },
+            { color: '#ff8c00', text: 'Retail (pure)' },
             { color: '#f5f58f', text: 'Industry & Business' },
             { color: '#73ccd1', text: 'Community Services' },
             { color: '#ffbfbf', text: 'Recreation & Leisure' },
             { color: '#b3de69', text: 'Transport' },
             { color: '#cccccc', text: 'Utilities & Infrastructure' },
             { color: '#898944', text: 'Defence' },
+            { color: '#52403c', text: 'Agriculture & Fisheries'},
+            { color: '#ffffff', text: 'Vacant & Derelict' }
         ]
     },
     type: {

etl/README.md

@@ -8,9 +8,41 @@ for Colouring London:
 
 ## Prerequisites
 
+Install PostgreSQL and create a database for colouringlondon, with a database
+user that can connect to it. The [PostgreSQL
+documentation](https://www.postgresql.org/docs/12/tutorial-start.html) covers
+installation and getting started.
+
+Install the [PostGIS extension](https://postgis.net/).
+
+Connect to the colouringlondon database and add the PostGIS, pgcrypto and
+pg_trgm extensions:
+
+```sql
+create extension postgis;
+create extension pgcrypto;
+create extension pg_trgm;
+```
+
+Create the core database tables:
+
+```bash
+psql < ../migrations/001.core.up.sql
+```
+
+There is some performance benefit to creating indexes after bulk loading data.
+Otherwise, it's fine to run all the migrations at this point and skip the index
+creation steps below.
+
+Install GNU parallel, this is used to speed up loading bulk data.
+
+
+## Process and load Ordance Survey data
+
 Before running any of these scripts, you will need the OS data for your area of
 interest. AddressBase and MasterMap are available directly from [Ordnance
-Survey](https://www.ordnancesurvey.co.uk/).
+Survey](https://www.ordnancesurvey.co.uk/). The alternative setup below uses
+OpenStreetMap.
 
 The scripts should be run in the following order:
 
@@ -32,6 +64,10 @@ load_uprns.py ./addressbase_dir
 psql < ../migrations/003.index-buildings.sql
 ```
 
+## Alternative, using OpenStreetMap
+
+This uses the [osmnx](https://github.com/gboeing/osmnx) python package to get OpenStreetMap data. You will need python and osmnx to run `get_test_polygons.py`.
+
 To help test the Colouring London application, `get_test_polygons.py` will attempt to save a
 small (1.5km²) extract from OpenStreetMap to a format suitable for loading to the database.
 
@@ -39,13 +75,17 @@ In this case, run:
 
 ```bash
 # download test data
-get_test_polygons.py
+python get_test_polygons.py
 # load all building outlines
-load_geometries.sh ./test_data_dir
+./load_geometries.sh ./
 # index geometries (should be faster after loading)
-psql < ../migrations/002.index-geometries.sql
+psql < ../migrations/002.index-geometries.up.sql
 # create a building record per outline
-create_building_records.sh
+./create_building_records.sh
 # index building records
-psql < ../migrations/002.index-buildings.sql
+psql < ../migrations/003.index-buildings.up.sql
 ```
+
+## Finally
+
+Run the remaining migrations in `../migrations` to create the rest of the database structure.

etl/get_test_polygons.py

@@ -21,11 +21,13 @@ size = 256
 # load buildings from about 1.5km² around UCL
 point = (51.524498, -0.133874)
 dist = 612
-gdf = osmnx.buildings_from_point(point=point, distance=dist)
+gdf = osmnx.footprints_from_point(point=point, dist=dist)
 
 # preview image
-gdf_proj = osmnx.project_gdf(gdf, to_crs={'init': 'epsg:3857'})
-fig, ax = osmnx.plot_buildings(gdf_proj, bgcolor='#333333', color='w', figsize=(4,4),
+gdf_proj = osmnx.projection.project_gdf(gdf, to_crs={'init': 'epsg:3857'})
+gdf_proj = gdf_proj[gdf_proj.geometry.apply(lambda g: g.geom_type != 'MultiPolygon')]
+
+fig, ax = osmnx.plot_footprints(gdf_proj, bgcolor='#333333', color='w', figsize=(4,4),
                                save=True, show=False, close=True,
                                filename='test_buildings_preview', dpi=600)
 

migrations/README.md

@@ -8,6 +8,7 @@ $ psql "host={hostname} user={username} port={port} sslmode=require dbname=postg
 > \c colouringlondon
 > create extension postgis;
 > create extension pgcrypto;
+> create extension pg_trgm;
 > \q
 ```
 

provision/vm_provision.sh

@@ -40,7 +40,7 @@ apt-get install -y \
 #
 
 # node version and platform
-NODE_VERSION=v12.14.1
+NODE_VERSION=v12.18.1
 DISTRO=linux-x64
 
 # download