Merge branch 'colouring-cities:master' into master

README.md

@@ -26,6 +26,9 @@ for download under a liberal open data license
 
 ## Setup and run
 
+#### Forking the Core Repository
+If you are working with us as part of this GitHub organisation, then a repository and team may already have been set up for you. If you are already usign the repository and want to add a new project, perhaps because you are expanding to additional cities, then please following these instructions to create a Fork of the core repository: [docs/working-with-colouring-core](docs/working-with-colouring-core.md).
+
 #### Customise the Application:
 
 You can customise the Colouring Cities application by changing the values in the following file:

app/map_styles/polygon.xml

@@ -623,6 +623,34 @@
             <LineSymbolizer stroke="#888" stroke-width="3.0"/>
         </Rule>
     </Style>
+    <Style name="survival_status">
+        <Rule>
+            <Filter>[survival_status] = "Same as Historical Map (Unchanged)"</Filter>
+            <PolygonSymbolizer fill="#6ded45" />
+        </Rule>
+        <Rule>
+            <Filter>[survival_status] = "Similar to Historical Map (Some Changes)"</Filter>
+            <PolygonSymbolizer fill="#f7c725" />
+        </Rule>
+        <Rule>
+            <Filter>[survival_status] = "Historical Building Demolished"</Filter>
+            <PolygonSymbolizer fill="#ff2121" />
+        </Rule>
+        <Rule>
+            <Filter>[survival_status] = "Current Building on Previous Green Space"</Filter>
+            <PolygonSymbolizer fill="#CF26DF" />
+        </Rule>
+        <Rule>
+            <MaxScaleDenominator>17061</MaxScaleDenominator>
+            <MinScaleDenominator>4264</MinScaleDenominator>
+            <LineSymbolizer stroke="#888" stroke-width="1.0"/>
+        </Rule>
+        <Rule>
+            <MaxScaleDenominator>4264</MaxScaleDenominator>
+            <MinScaleDenominator>0</MinScaleDenominator>
+            <LineSymbolizer stroke="#888" stroke-width="3.0"/>
+        </Rule>
+    </Style>
     <Style name="likes">
         <Rule>
             <Filter>[likes] &gt;= 100</Filter>

app/package.json

@@ -20,11 +20,11 @@
     "@mapbox/sphericalmercator": "^1.1.0",
     "ajv": "^7.1.1",
     "ajv-formats": "^1.5.1",
-    "body-parser": "^1.19.0",
+    "body-parser": "^1.20.1",
     "bootstrap": "^4.5.0",
     "canvas-confetti": "^1.2.0",
     "connect-pg-simple": "^6.1.0",
-    "express": "^4.17.1",
+    "express": "^4.18.2",
     "express-session": "^1.17.1",
     "geojson": "^0.5.0",
     "leaflet": "^1.7.1",
@@ -41,7 +41,7 @@
     "react-leaflet": "^3.1.0",
     "react-router-dom": "^5.2.0",
     "serialize-javascript": "^5.0.1",
-    "sharp": "^0.28.3",
+    "sharp": "^0.30.5",
     "use-throttle": "0.0.3"
   },
   "devDependencies": {

app/src/api/config/dataFields.ts

@@ -296,6 +296,14 @@ export const buildingAttributesConfig = valueType<DataFieldConfig>()({ /* eslint
         edit: true,
         verify: true
     },
+    survival_status: {
+        edit: true,
+        verify: true
+    },
+    survival_source: {
+        edit: true,
+        verify: true
+    },
     likes_total: {
         edit: false,
         derivedEdit: true,

app/src/frontend/building/data-containers/age.tsx

@@ -20,10 +20,11 @@ import { CategoryViewProps } from './category-view-props';
 import { LogicalDataEntry } from '../data-components/logical-data-entry/logical-data-entry';
 import { useDisplayPreferences } from '../../displayPreferences-context';
 
-const HistoricalStatusOptions = [
+const SurvivalStatusOptions = [
-    'The current footprint matches/almost exactly matches the historical map beneath, and/or is known to have been built before the map was made',
+    'Same as Historical Map (Unchanged)',
-    'The building core is the same as the historical map but has had multiple additions/changes',
+    'Similar to Historical Map (Some Changes)',
-    'The building no longer exists',
+    'Historical Building Demolished',
+    'Current Building on Previous Green Space'
 ];
 
 /**
@@ -227,26 +228,32 @@ const AgeView: React.FunctionComponent<CategoryViewProps> = (props) => {
             </DataEntryGroup>
             <DataEntryGroup name="Survival and Loss tracked using Historical Maps" collapsed={true} >
                 <InfoBox>
-                    This section is under development.
+                    Can you help us create a map that shows how many buildings in London have survived since the 1890s? 
+                    Choose a colour to indicate whether the building has survived.
                 </InfoBox>
                 <button className={`map-switcher-inline ${historicData}-state btn btn-outline btn-outline-dark ${darkLightTheme}`} onClick={historicDataSwitchOnClick}> 
                     {(historicData === 'enabled')?'Click here to hide historical maps':'Click here to show historical maps'}
                 </button>
                 <SelectDataEntry
-                    title={dataFields.historical_status.title}
+                    title={dataFields.survival_status.title}
-                    slug="historical_status"
+                    slug="survival_status"
-                    value={""}
+                    value={props.building.survival_status}
-                    tooltip={dataFields.historical_status.tooltip}
+                    tooltip={dataFields.survival_status.tooltip}
-                    options={HistoricalStatusOptions}
+                    options={SurvivalStatusOptions}
                     mode={props.mode}
                     copy={props.copy}
                     onChange={props.onChange}
                 />
-                <DataEntry
+                <SelectDataEntry
-                    title="Historical land use change"
+                    title={dataFields.survival_source.title}
-                    slug=""
+                    slug="survival_source"
-                    value=""
+                    value={props.building.survival_source}
-                    mode='view'
+                    mode={props.mode}
+                    copy={props.copy}
+                    onChange={props.onChange}
+                    tooltip={dataFields.survival_source.tooltip}
+                    placeholder={dataFields.survival_source.example}
+                    options={dataFields.survival_source.items}
                     />
             </DataEntryGroup>
           </Fragment>
@@ -496,26 +503,32 @@ const AgeView: React.FunctionComponent<CategoryViewProps> = (props) => {
             </DataEntryGroup>
             <DataEntryGroup name="Survival and Loss tracked using Historical Maps" collapsed={true} >
                 <InfoBox>
-                    This section is under development.
+                    Can you help us create a map that shows how many buildings in London have survived since the 1890s? 
+                    Choose a colour to indicate whether the building has survived.
                 </InfoBox>
-                <button className={`map-switcher-inline ${historicData} btn btn-outline btn-outline-dark ${darkLightTheme}`} onClick={historicDataSwitchOnClick}> 
+                <button className={`map-switcher-inline ${historicData}-state btn btn-outline btn-outline-dark ${darkLightTheme}`} onClick={historicDataSwitchOnClick}> 
                     {(historicData === 'enabled')?'Click here to hide historical maps':'Click here to show historical maps'}
                 </button>
                 <SelectDataEntry
-                    title={dataFields.historical_status.title}
+                    title={dataFields.survival_status.title}
-                    slug="historical_status"
+                    slug="survival_status"
-                    value={""}
+                    value={props.building.survival_status}
-                    tooltip={dataFields.historical_status.tooltip}
+                    tooltip={dataFields.survival_status.tooltip}
-                    options={HistoricalStatusOptions}
+                    options={SurvivalStatusOptions}
                     mode={props.mode}
                     copy={props.copy}
                     onChange={props.onChange}
                 />
-                <DataEntry
+                <SelectDataEntry
-                    title="Historical land use change"
+                    title={dataFields.survival_source.title}
-                    slug=""
+                    slug="survival_source"
-                    value=""
+                    value={props.building.survival_source}
-                    mode='view'
+                    mode={props.mode}
+                    copy={props.copy}
+                    onChange={props.onChange}
+                    tooltip={dataFields.survival_source.tooltip}
+                    placeholder={dataFields.survival_source.example}
+                    options={dataFields.survival_source.items}
                     />
             </DataEntryGroup>
         </Fragment>

app/src/frontend/config/categories-config.ts

@@ -54,7 +54,7 @@ export const categoriesConfig: {[key in Category]: CategoryDefinition} = {
     },
     [Category.Size]: {
         slug: 'size',
-        name: 'Size & Form',
+        name: 'Size',
         aboutUrl: 'https://pages.colouring.london/shapeandsize',
         intro: 'How big are buildings?',
     },

app/src/frontend/config/category-maps-config.ts

@@ -24,7 +24,8 @@ export interface CategoryMapDefinition {
 export const defaultMapCategory = Category.Age;
 
 export const categoryMapsConfig: {[key in Category]: CategoryMapDefinition[]} = {
-    [Category.Age]: [{
+    [Category.Age]: [
+        {
             mapStyle: 'date_year',
             legend: {
                 title: 'Age',
@@ -46,8 +47,22 @@ export const categoryMapsConfig: {[key in Category]: CategoryMapDefinition[]} =
                     { color: '#d0c291', text: '<1700' },
                 ]
             },
-    }],
+        },
-    [Category.Size]: [{
+        {
+            mapStyle: 'survival_status',
+            legend: {
+                title: 'Survival status',
+                elements: [
+                    { color: '#6ded45', text: 'Same as Historical Map (Unchanged)' },
+                    { color: '#f7c725', text: 'Similar to Historical Map (Some Changes)' },
+                    { color: '#ff2121', text: 'Historical Building Demolished' },
+                    { color: '#CF26DF', text: 'Current Building on Previous Green Space' },
+                ]
+            }
+        },
+    ],
+    [Category.Size]: [
+        {
             mapStyle: 'size_height',
             legend: {
                 title: 'Height to apex',
@@ -62,7 +77,8 @@ export const categoryMapsConfig: {[key in Category]: CategoryMapDefinition[]} =
                     { color: '#980043', text: '≥152'}
                 ]
             },
-    }],
+        }
+    ],
     [Category.Team]: [{
         mapStyle: 'team',
         legend: {

app/src/frontend/config/data-fields-config.ts

@@ -444,7 +444,7 @@ export const dataFields = { /* eslint-disable @typescript-eslint/camelcase */
     sust_aggregate_estimate_epc: {
         category: Category.Sustainability,
         title: "Domestic Building Energy Rating",
-        tooltip: "Energy Performance Certificate (EPC) Any premises sold or rented is required to have an EPC to show how energy efficient it is. Only buildings rate grade E or higher maybe rented",
+        tooltip: "Energy Performance Certificate (EPC) Any premises sold or rented is required to have an EPC to show how energy efficient it is. Only buildings rate grade E or higher may be rented",
         example: "",
     },
     sust_retrofit_date: {
@@ -460,10 +460,37 @@ export const dataFields = { /* eslint-disable @typescript-eslint/camelcase */
         //tooltip: ,
     },
 
-    historical_status: {
+    survival_status: {
         category: Category.Age,
-        title: "Historical Status",
+        title: "Survival status",
         tooltip: "Survival and Loss tracked using Historical Maps",
+        items: [
+            "Same as Historical Map (Unchanged)",
+            "Similar to Historical Map (Some Changes)",
+            "Historical Building Demolished",
+            "Current Building on Previous Green Space"
+        ],
+        example: "",
+    },
+
+    survival_source: {
+        category: Category.Age,
+        title: "Source of survival information",
+        tooltip: "Source for the survival status",
+        items: [
+            "Matched by comparing maps",
+            "Checked using streetview images",
+            "Historical publication or archive document",
+            "Other"
+        ],
+        example: "",
+    },
+
+    survival_link: {
+        category: Category.Age,
+        title: "Please add any additional text or image links providing historical information on this building",
+        tooltip: "URL for age and date reference",
+        example: ["", "", ""],
     },
 
     edit_history: {

app/src/frontend/config/tileserver-config.ts

@@ -21,7 +21,8 @@ export type BuildingMapTileset = 'date_year' |
     'building_attachment_form' |
     'landuse' |
     'dynamics_demolished_count' |
-    'team';
+    'team' |
+    'survival_status';
 
 export type SpecialMapTileset = 'base_light' | 'base_night' | 'base_night_outlines' | 'highlight' | 'number_labels' | 'base_boroughs';
 

app/src/frontend/map/layers/historic-data-layer.tsx

@@ -3,14 +3,19 @@ import { TileLayer } from 'react-leaflet';
 import { LayerEnablementState } from '../../config/map-config';
 import { BuildingBaseLayerAllZoom } from './building-base-layer-all-zoom';
 import { useDisplayPreferences } from '../../displayPreferences-context';
+import { BuildingDataLayer } from './building-data-layer';
 
 export function HistoricDataLayer({}) {
     const { historicData } = useDisplayPreferences();
     if(historicData == "enabled") {
-        return <><TileLayer
+        return <>
+                <TileLayer
                     url="https://mapseries-tilesets.s3.amazonaws.com/london_1890s/{z}/{x}/{y}.png"
                     attribution='&copy; CC BY 4.0 - Reproduced with the permission of the <a href="https://maps.nls.uk/">National Library of Scotland</a>'
-        /><BuildingBaseLayerAllZoom theme="night_outlines" /></>
+                />
+                <BuildingBaseLayerAllZoom theme="night_outlines" />
+                <BuildingDataLayer tileset={'survival_status'} revisionId={''} />
+            </>
 	} else {
         return null;
     }

app/src/tiles/dataDefinition.ts

@@ -71,6 +71,8 @@ const LAYER_QUERIES = {
         WHERE
             construction_core_material IS NOT NULL`,
     location: `
+        SELECT blds_with_data.* 
+        FROM (
             SELECT
                     geometry_id,
                     (
@@ -86,8 +88,12 @@ const LAYER_QUERIES = {
                         case when ref_osm_id IS NULL then 0 else 1 end
                     ) AS location_info_count
                 FROM
-            buildings`,
+                    buildings
+            ) AS blds_with_data
+        WHERE blds_with_data.location_info_count > 0`,
     team: `
+        SELECT blds_with_data.* 
+        FROM (
             SELECT
                     geometry_id,
                     (
@@ -108,7 +114,9 @@ const LAYER_QUERIES = {
                         case when date_year IS NULL then 0 else 1 end
                     ) AS team_info_count
                 FROM
-            buildings`,
+                    buildings
+            ) AS blds_with_data
+        WHERE blds_with_data.team_info_count > 0`,
     is_domestic: `
         SELECT
             geometry_id,
@@ -117,6 +125,22 @@ const LAYER_QUERIES = {
             buildings
         WHERE
             is_domestic IS NOT NULL`,
+    survival_status: `
+        SELECT
+            geometry_id,
+            survival_status
+        FROM
+            buildings
+        WHERE
+        survival_status IS NOT NULL`,
+    survival_source: `
+        SELECT
+            geometry_id,
+            survival_source
+        FROM
+            buildings
+        WHERE
+        survival_source IS NOT NULL`,
     likes: `
         SELECT
             geometry_id,

docs/working-with-colouring-core.md

@@ -0,0 +1,25 @@
+# Working with the Colouring Cities Core Platform
+
+To make Colouring Cities a better and more collaborative platform, we strongly encourage all of our partners to keep their code as unified with the [Core Platform](https://github.com/colouring-cities/colouring-core) as possible. This will help ensure consitency and enable data sharing between the various Colouring Cities projects, and will also allow us all to continue to improve the Core Platform.
+
+## Working with Forks
+By using Forks, GitHub will maintain a link between your project and the Core plaform. This will allow you to see any changes that are made to the Core respository and easily syncrhonise those changes into your own code, as shown below (as well as allowing you to contribute some of your code back to the Core platform).
+
+## Forking the Core Respository
+We recommend that you create a "Fork" of the [colouring-core](https://github.com/colouring-cities/colouring-core) to create the respository for your Colouring Cities project, and then customise the code from the core platform to suit the needs of your projects.  
+  
+This can be done by going to the [Core Repository](https://github.com/colouring-cities/colouring-core), clicking the 'Fork' button, as show below and then following the instructions.
+
+![image](https://user-images.githubusercontent.com/21125422/224355892-7cc76027-6402-4fb1-b12b-7c41e3e4e55d.png)
+
+## Checking Project Status
+On your project repository, you will then see that your project is forked from Colouring Core, and you will also be notified when changes are made to the Core platform, as shown in the example below.
+
+![image](https://user-images.githubusercontent.com/21125422/224358529-4ef5ac1d-50e2-467b-95aa-033e44cce7af.png)
+
+You can click on "Sync Fork" and then "Update Branch" to pull the latest changes from Colouring Core into your own project.
+
+## Customising your Colouring Cities Project
+When it comes to customising the Colouring Core platform for use in your country/city, we want to make this as easy as possible, while still maintaining the brand identity of the platform.
+
+Where possible, customisations should be made using configuration files, rather than by modifying the source code. More information on the current configuration file setup can be found [here](configuring-colouring-cities.md). Please let us know (ideally by creating issues on the [Core respository](https://github.com/colouring-cities/colouring-core) (if there aree other elements that you would like us to add into this configuration framework.

etl/planning_data/README.MD

@@ -1,4 +1,8 @@
-Following instructions assume that code is placed within `~/colouring-london/etl/planning_data/`
+Planning data was written to be extendable to other locations but currently is setup only for London and relies on GLA data.
+
+Extending it to other locations will require some custom work. At least connecting to other external API and parsing their data. Some additional custom data processing will be likely needed like it was needed for London data.
+
+Following instructions assume that code is placed within `~/colouring-core/etl/planning_data/` and tiles are within `/srv/colouring-london/tilecache/` Note that this is affected by location specified in ecosystem config file which may be for example at `/var/www/colouring-london/ecosystem.config.js`
 
 To install necessary dependencies use `cd ~/colouring-london/etl/planning_data/ && pip3 install -r requirements.txt`
 

maintenance/extract_data/export_attributes.sql

@@ -56,7 +56,9 @@ COPY (SELECT
     planning_historic_area_assessment_url,
     is_domestic,
     community_type_worth_keeping_total,
-    likes_total
+    likes_total,
+    survival_status,
+    survival_source
 FROM buildings)
 TO '/tmp/building_attributes.csv'
 WITH CSV HEADER

migrations/038.survival_status.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE buildings DROP COLUMN IF EXISTS survival_status;
+ALTER TABLE buildings DROP COLUMN IF EXISTS survival_source;

migrations/038.survival_status.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE buildings ADD COLUMN IF NOT EXISTS survival_status text;
+ALTER TABLE buildings ADD COLUMN IF NOT EXISTS survival_source text;