a_adli/colouring-montreal
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'develop' into feature/513-activate-land-use

Browse Source
This commit is contained in:
Maciej Ziarkowski 2020-03-08 16:05:55 +00:00
commit af64c4ca58
41 changed files with 913 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@ -1,6 +1,6 @@
language: node_js
node_js:
- 8
- 12
cache: npm
before_script:
- cd $TRAVIS_BUILD_DIR/app && npm ci

13
app/.babelrc
View File

@ -1,13 +0,0 @@
{
"plugins": [
"@babel/plugin-syntax-jsx",
"@babel/plugin-syntax-typescript",
[
"babel-plugin-typescript-to-proptypes",
{
"implicitChildren": true,
"typeCheck": true
}
]
]
}

34
app/map_styles/polygon.xml
View File

@ -89,6 +89,40 @@
<PolygonSymbolizer fill="#800026" />
</Rule>
</Style>
<Style name="size_height">
<Rule>
<Filter>[size_height] &lt; 5.55</Filter>
<PolygonSymbolizer fill="#f7f4f9" />
</Rule>
<Rule>
<Filter>[size_height] &gt;= 5.55 and [size_height] &lt; 7.73</Filter>
<PolygonSymbolizer fill="#e7e1ef" />
</Rule>
<Rule>
<Filter>[size_height] &gt;= 7.73 and [size_height] &lt; 11.38</Filter>
<PolygonSymbolizer fill="#d4b9da" />
</Rule>
<Rule>
<Filter>[size_height] &gt;= 11.38 and [size_height] &lt; 18.45</Filter>
<PolygonSymbolizer fill="#c994c7" />
</Rule>
<Rule>
<Filter>[size_height] &gt;= 18.45 and [size_height] &lt; 35.05</Filter>
<PolygonSymbolizer fill="#df65b0" />
</Rule>
<Rule>
<Filter>[size_height] &gt;= 35.05 and [size_height] &lt; 89.30</Filter>
<PolygonSymbolizer fill="#e7298a" />
</Rule>
<Rule>
<Filter>[size_height] &gt;= 89.30 and [size_height] &lt; 152</Filter>
<PolygonSymbolizer fill="#ce1256" />
</Rule>
<Rule>
<Filter>[size_height] &gt;= 152</Filter>
<PolygonSymbolizer fill="#980043" />
</Rule>
</Style>
<Style name="date_year">
<Rule>
<Filter>[date_year] &gt;= 2000</Filter>

26
app/package-lock.json generated
View File

@ -425,15 +425,6 @@
"@babel/helper-plugin-utils": "^7.0.0"
}
},
"@babel/plugin-syntax-typescript": {
"version": "7.7.4",
"resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.7.4.tgz",
"integrity": "sha512-77blgY18Hud4NM1ggTA8xVT/dBENQf17OpiToSa2jSmEY3fWXD2jwrdVlO4kq5yzUTeF15WSQ6b4fByNvJcjpQ==",
"dev": true,
"requires": {
"@babel/helper-plugin-utils": "^7.0.0"
}
},
"@babel/plugin-transform-arrow-functions": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.2.0.tgz",
@ -1198,9 +1189,9 @@
"dev": true
},
"@types/node": {
"version": "8.10.59",
"resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.59.tgz",
"integrity": "sha512-8RkBivJrDCyPpBXhVZcjh7cQxVBSmRk9QM7hOketZzp6Tg79c0N8kkpAIito9bnJ3HCVCHVYz+KHTEbfQNfeVQ==",
"version": "12.12.25",
"resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.25.tgz",
"integrity": "sha512-nf1LMGZvgFX186geVZR1xMZKKblJiRfiASTHw85zED2kI1yDKHDwTKMdkaCbTlXoRKlGKaDfYywt+V0As30q3w==",
"dev": true
},
"@types/nodemailer": {
@ -2429,17 +2420,6 @@
"integrity": "sha512-f49NsaohQ1ByY20nUrpc30QFdbeT4ntV4PAL2vSZe6uCB5nqAcqXS/qzU+aI6ZfYhWASx5eIsTFvFrs1B2ffGg==",
"dev": true
},
"babel-plugin-typescript-to-proptypes": {
"version": "0.17.1",
"resolved": "https://registry.npmjs.org/babel-plugin-typescript-to-proptypes/-/babel-plugin-typescript-to-proptypes-0.17.1.tgz",
"integrity": "sha512-yREUfvDlmn6QjM0QbywXUkXBQMD/iFfLVTl+jig4X7ZLUg9lq8ZLuex8HIM2SQ4X3vcjGnWPFowodlMcXhwxdQ==",
"dev": true,
"requires": {
"@babel/helper-module-imports": "^7.0.0",
"@babel/helper-plugin-utils": "^7.0.0",
"@babel/plugin-syntax-typescript": "^7.2.0"
}
},
"babel-preset-jest": {
"version": "23.2.0",
"resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-23.2.0.tgz",

7
app/package.json
View File

@ -28,7 +28,6 @@
"node-fs": "^0.1.7",
"nodemailer": "^6.3.0",
"pg-promise": "^8.7.5",
"prop-types": "^15.7.2",
"query-string": "^6.8.2",
"react": "^16.9.0",
"react-dom": "^16.9.0",
@ -40,17 +39,14 @@
"use-throttle": "0.0.3"
},
"devDependencies": {
"@babel/plugin-syntax-jsx": "^7.7.4",
"@babel/plugin-syntax-typescript": "^7.7.4",
"@types/express": "^4.17.2",
"@types/express-session": "^1.15.16",
"@types/jest": "^24.0.23",
"@types/lodash": "^4.14.149",
"@types/lodash.isequal": "^4.5.5",
"@types/mapbox__sphericalmercator": "^1.1.3",
"@types/node": "^8.10.59",
"@types/node": "^12.12.25",
"@types/nodemailer": "^6.2.2",
"@types/prop-types": "^15.7.3",
"@types/react": "^16.9.16",
"@types/react-dom": "^16.9.4",
"@types/react-leaflet": "^2.5.0",
@ -58,7 +54,6 @@
"@types/sharp": "^0.22.3",
"@types/webpack-env": "^1.14.1",
"babel-eslint": "^10.0.3",
"babel-plugin-typescript-to-proptypes": "^0.17.1",
"eslint": "^5.16.0",
"eslint-plugin-jest": "^22.21.0",
"eslint-plugin-react": "^7.17.0",

155
app/public/openapi.yml Normal file
View File

@ -0,0 +1,155 @@
openapi: 3.0.0
info:
title: Colouring London API
version: 1.0.0
servers:
- url: https://colouring.london/api
description: Production server (uses live data)
paths:
/extracts:
get:
summary: Returns a list of bulk data extracts
responses:
'200':
description: A list of bulk extracts, from newest to oldest
content:
application/json:
schema:
properties:
extracts:
type: array
items:
$ref: '#/components/schemas/BulkExtract'
example:
extracts:
- extract_id: 1
extracted_on: 2019-10-03T05:33:00.000Z
download_path: /downloads/data-extract-2019-10-03-06_33_00.zip
'400':
$ref: '#/components/responses/BadRequest'
'500':
$ref: '#/components/responses/ServerError'
/history:
get:
summary: Returns a paginated list of edits (latest edits if no relevant parameters specified)
parameters:
- name: before_id
description: Returned edits will be ones made directly before the specified revision ID
in: query
schema:
$ref: '#/components/schemas/RevisionId'
required: false
- name: after_id
description: Returned edits will be ones made directly after the specified revision ID
in: query
schema:
$ref: '#/components/schemas/RevisionId'
required: false
- name: count
description: The desired number of records to return
in: query
schema:
type: number
minimum: 1
maximum: 100
default: 100
required: false
responses:
'200':
description: A list of edit history records
content:
application/json:
schema:
properties:
history:
type: array
items:
$ref: '#/components/schemas/BuildingEditHistoryEntry'
paging:
type: object
properties:
id_for_older_query:
allOf:
- $ref: '#/components/schemas/RevisionId'
- description: If older records exist - ID to use for querying them (use as before_id param), otherwise null
nullable: true
id_for_newer_query:
allOf:
- $ref: '#/components/schemas/RevisionId'
- description: If newer records exist - ID to use for querying them (use as after_id param), otherwise null
nullable: true
'400':
$ref: '#/components/responses/BadRequest'
'500':
$ref: '#/components/responses/ServerError'
components:
responses:
BadRequest:
description: Invalid request submitted by user
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
ServerError:
description: Unexpected server error
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
example:
error: Database error
schemas:
Error:
type: object
properties:
error:
type: string
description: Error message
BulkExtract:
type: object
properties:
extract_id:
type: integer
description: Unique sequential ID for the extract
extracted_on:
type: string
format: date-time
description: UTC timestamp at which the extract was generated
download_path:
type: string
description: Download path for the extract. Contains only URL path (should be used with the same hostname as the API).
RevisionId:
description: Unique sequential ID for an edit history entry (positive big integer)
type: string
pattern: ^[1-9]\d*&
BuildingEditHistoryEntry:
type: object
properties:
revision_id:
$ref: '#/components/schemas/RevisionId'
forward_patch:
type: object
description: Forward diff of the building attribute data
reverse_patch:
type: object
description: Reverse diff of the building attribute data
revision_timestamp:
type: string
format: date-time
description: UTC timestamp at which the building data was edited
username:
type: string
description: Username of the editor
building_id:
type: number
description: Unique ID of the edited building

11
app/razzle.config.js
View File

@ -9,17 +9,6 @@ module.exports = {
})
config.module.rules = rules;
// find module rule that runs ts-loader for TS(X) files
const tsRule = config.module.rules.find(r =>
new RegExp(r.test).test('test.tsx') && Array.isArray(r.use) && r.use.some(u => u.loader.includes('ts-loader')));
// run babel-loader before ts-loader to generate propTypes
tsRule.use.push({
loader: 'babel-loader',
options: {
babelrc: true
}
})
return config;
},
};

22
app/src/api/api.ts
View File

@ -3,6 +3,8 @@ import express from 'express';
import autofillController from './controllers/autofillController';
import * as editHistoryController from './controllers/editHistoryController';
import { ApiParamError, ApiUserError } from './errors/api';
import { DatabaseError } from './errors/general';
import buildingsRouter from './routes/buildingsRouter';
import extractsRouter from './routes/extractsRouter';
import usersRouter from './routes/usersRouter';
@ -95,14 +97,30 @@ server.get('/search', function (req, res) {
});
});
server.use((err, req, res, next) => {
server.use((err: any, req: express.Request, res: express.Response, next: express.NextFunction) => {
if (res.headersSent) {
return next(err);
}
if (err != undefined) {
console.log('Global error handler: ', err);
res.status(500).send({ error: 'Server error' });
if (err instanceof ApiUserError) {
let errorMessage: string;
if(err instanceof ApiParamError) {
errorMessage = `Problem with parameter ${err.paramName}: ${err.message}`;
} else {
errorMessage = err.message;
}
res.status(400).send({ error: errorMessage });
} else if(err instanceof DatabaseError){
res.status(500).send({ error: 'Database error' });
} else {
res.status(500).send({ error: 'Server error' });
}
}
});

14
app/src/api/controllers/buildingController.ts
View File

@ -1,6 +1,6 @@
import express from 'express';
import { parseIntParam } from '../helpers';
import { parsePositiveIntParam, processParam } from '../parameters';
import asyncController from '../routes/asyncController';
import * as buildingService from '../services/building';
import * as userService from '../services/user';
@ -35,7 +35,7 @@ const getBuildingsByReference = asyncController(async (req: express.Request, res
// GET individual building, POST building updates
const getBuildingById = asyncController(async (req: express.Request, res: express.Response) => {
const buildingId = parseIntParam(req.params.building_id);
const buildingId = processParam(req.params, 'building_id', parsePositiveIntParam, true);
try {
const result = await buildingService.getBuildingById(buildingId);
@ -63,7 +63,7 @@ const updateBuildingById = asyncController(async (req: express.Request, res: exp
});
async function updateBuilding(req: express.Request, res: express.Response, userId: string) {
const buildingId = parseIntParam(req.params.building_id);
const buildingId = processParam(req.params, 'building_id', parsePositiveIntParam, true);
const buildingUpdate = req.body;
@ -84,7 +84,7 @@ async function updateBuilding(req: express.Request, res: express.Response, userI
// GET building UPRNs
const getBuildingUPRNsById = asyncController(async (req: express.Request, res: express.Response) => {
const buildingId = parseIntParam(req.params.building_id);
const buildingId = processParam(req.params, 'building_id', parsePositiveIntParam, true);
try {
const result = await buildingService.getBuildingUPRNsById(buildingId);
@ -105,7 +105,7 @@ const getBuildingLikeById = asyncController(async (req: express.Request, res: ex
return res.send({ like: false }); // not logged in, so cannot have liked
}
const buildingId = parseIntParam(req.params.building_id);
const buildingId = processParam(req.params, 'building_id', parsePositiveIntParam, true);
try {
const like = await buildingService.getBuildingLikeById(buildingId, req.session.user_id);
@ -118,7 +118,7 @@ const getBuildingLikeById = asyncController(async (req: express.Request, res: ex
});
const getBuildingEditHistoryById = asyncController(async (req: express.Request, res: express.Response) => {
const buildingId = parseIntParam(req.params.building_id);
const buildingId = processParam(req.params, 'building_id', parsePositiveIntParam, true);
try {
const editHistory = await buildingService.getBuildingEditHistory(buildingId);
@ -134,7 +134,7 @@ const updateBuildingLikeById = asyncController(async (req: express.Request, res:
return res.send({ error: 'Must be logged in' });
}
const buildingId = parseIntParam(req.params.building_id);
const buildingId = processParam(req.params, 'building_id', parsePositiveIntParam, true);
const { like } = req.body;
try {

27
app/src/api/controllers/editHistoryController.ts
View File

@ -1,17 +1,32 @@
import express from 'express';
import { ApiParamError, ApiUserError } from '../errors/api';
import { ArgumentError } from '../errors/general';
import { checkRegexParam, parsePositiveIntParam, processParam } from '../parameters';
import asyncController from "../routes/asyncController";
import * as editHistoryService from '../services/editHistory';
const getGlobalEditHistory = asyncController(async (req: express.Request, res: express.Response) => {
const revisionIdRegex = /^[1-9]\d*$/;
const afterId: string = processParam(req.query, 'after_id', x => checkRegexParam(x, revisionIdRegex));
const beforeId: string = processParam(req.query, 'before_id', x => checkRegexParam(x, revisionIdRegex));
const count: number = processParam(req.query, 'count', parsePositiveIntParam);
if(afterId != undefined && beforeId != undefined) {
throw new ApiUserError('Cannot specify both after_id and before_id parameters');
}
try {
const result = await editHistoryService.getGlobalEditHistory();
res.send({
history: result
});
const result = await editHistoryService.getGlobalEditHistory(beforeId, afterId, count);
res.send(result);
} catch(error) {
console.error(error);
res.send({ error: 'Database error' });
if(error instanceof ArgumentError && error.argumentName === 'count') {
const apiErr = new ApiParamError(error.message, 'count');
throw apiErr;
}
throw error;
}
});

5
app/src/api/controllers/extractController.ts
View File

@ -1,6 +1,6 @@
import express from 'express';
import { parseIntParam } from '../helpers';
import { parsePositiveIntParam, processParam } from '../parameters';
import asyncController from '../routes/asyncController';
import * as dataExtractService from '../services/dataExtract';
@ -15,8 +15,9 @@ const getAllDataExtracts = asyncController(async function(req: express.Request,
});
const getDataExtract = asyncController(async function(req: express.Request, res: express.Response) {
const extractId = processParam(req.params, 'extract_id', parsePositiveIntParam, true);
try {
const extractId = parseIntParam(req.params.extract_id);
const extract = await dataExtractService.getDataExtractById(extractId);
res.send({ extract: extract });
} catch (err) {

62
app/src/api/dataAccess/__mocks__/editHistory.ts Normal file
View File

@ -0,0 +1,62 @@
import { EditHistoryEntry } from '../../../frontend/models/edit-history-entry';
import { numAsc, numDesc } from '../../../helpers';
/**
* Create an object mocking all method of editHistory dataAccess
* The type is set to reflect the type of that module, with added methods
* used when testing
*/
const mockEditHistory =
jest.genMockFromModule('../editHistory') as typeof import('../editHistory') & {
__setHistory: (mockHistoryData: EditHistoryEntry[]) => void
};
let mockData: EditHistoryEntry[] = [];
mockEditHistory.__setHistory = function(mockHistoryData: EditHistoryEntry[]) {
mockData = mockHistoryData.sort(numDesc(x => BigInt(x.revision_id)));
};
mockEditHistory.getHistoryAfterId = function(id: string, count: number): Promise<EditHistoryEntry[]> {
return Promise.resolve(
mockData
.filter(x => BigInt(x.revision_id) > BigInt(id))
.sort(numAsc(x => BigInt(x.revision_id)))
.slice(0, count)
.sort(numDesc(x => BigInt(x.revision_id)))
);
};
mockEditHistory.getHistoryBeforeId = function(id: string, count: number): Promise<EditHistoryEntry[]> {
let filteredData = id == undefined ? mockData : mockData.filter(x => BigInt(x.revision_id) < BigInt(id));
return Promise.resolve(
filteredData
.slice(0, count)
);
};
mockEditHistory.getIdNewerThan = async function(id: string): Promise<string> {
const historyAfterId = await mockEditHistory.getHistoryAfterId(id, 1);
return historyAfterId[historyAfterId.length - 1]?.revision_id;
};
mockEditHistory.getIdOlderThan = async function(id: string): Promise<string> {
const historyBeforeId = await mockEditHistory.getHistoryBeforeId(id, 1);
return historyBeforeId[0]?.revision_id;
};
const {
__setHistory,
getHistoryAfterId,
getHistoryBeforeId,
getIdNewerThan,
getIdOlderThan
} = mockEditHistory;
export {
__setHistory,
getHistoryAfterId,
getHistoryBeforeId,
getIdNewerThan,
getIdOlderThan
};

88
app/src/api/dataAccess/editHistory.ts Normal file
View File

@ -0,0 +1,88 @@
import db from '../../db';
import { EditHistoryEntry } from '../../frontend/models/edit-history-entry';
import { DatabaseError } from '../errors/general';
const baseQuery = `
SELECT
log_id as revision_id,
forward_patch,
reverse_patch,
date_trunc('minute', log_timestamp) as revision_timestamp,
username,
building_id
FROM logs
JOIN users ON logs.user_id = users.user_id`;
export function getHistoryAfterId(id: string, count: number): Promise<EditHistoryEntry[]> {
/**
* SQL with lower time bound specified (records after ID).
* The outer SELECT is so that final results are sorted by descending ID
* (like the other queries). The inner select is sorted in ascending order
* so that the right rows are returned when limiting the result set.
*/
try {
return db.any(`
SELECT * FROM (
${baseQuery}
WHERE log_id > $1
ORDER BY revision_id ASC
LIMIT $2
) AS result_asc ORDER BY revision_id DESC`,
[id, count]
);
} catch(err) {
throw new DatabaseError(err);
}
}
export function getHistoryBeforeId(id: string, count: number): Promise<EditHistoryEntry[]> {
try {
if(id == undefined) {
return db.any(`
${baseQuery}
ORDER BY revision_id DESC
LIMIT $1
`, [count]);
} else {
return db.any(`
${baseQuery}
WHERE log_id < $1
ORDER BY revision_id DESC
LIMIT $2
`, [id, count]);
}
} catch(err) {
throw new DatabaseError(err);
}
}
export async function getIdOlderThan(id: string): Promise<string> {
try {
const result = await db.oneOrNone<{revision_id:string}>(`
SELECT MAX(log_id) as revision_id
FROM logs
WHERE log_id < $1
`, [id]);
return result?.revision_id;
} catch(err) {
throw new DatabaseError(err);
}
}
export async function getIdNewerThan(id: string): Promise<string> {
try {
const result = await db.oneOrNone<{revision_id:string}>(`
SELECT MIN(log_id) as revision_id
FROM logs
WHERE log_id > $1
`, [id]);
return result?.revision_id;
} catch(err) {
throw new DatabaseError(err);
}
}

44
app/src/api/errors/api.ts Normal file
View File

@ -0,0 +1,44 @@
/**
* Note that custom errors and the instanceof operator in TS work together
* only when transpiling to ES2015 and up.
* For earier target versions (ES5), a workaround is required:
* https://stackoverflow.com/questions/41102060/typescript-extending-error-class
*/
export class ApiUserError extends Error {
constructor(message?: string) {
super(message);
this.name = 'ApiUserError';
}
}
export class ApiParamError extends ApiUserError {
public paramName: string;
constructor(message?: string, paramName?: string) {
super(message);
this.name = 'ApiParamError';
this.paramName = paramName;
}
}
export class ApiParamRequiredError extends ApiParamError {
constructor(message?: string) {
super(message);
this.name = 'ApiParamRequiredError';
}
}
export class ApiParamOutOfBoundsError extends ApiParamError {
constructor(message?: string) {
super(message);
this.name = 'ApiParamOutOfBoundsError';
}
}
export class ApiParamInvalidFormatError extends ApiParamError {
constructor(message?: string) {
super(message);
this.name = 'ApiParamInvalidFormatError';
}
}

17
app/src/api/errors/general.ts Normal file
View File

@ -0,0 +1,17 @@
export class ArgumentError extends Error {
public argumentName: string;
constructor(message?: string, argumentName?: string) {
super(message);
this.name = 'ArgumentError';
this.argumentName = argumentName;
}
}
export class DatabaseError extends Error {
public detail: any;
constructor(detail?: string) {
super();
this.name = 'DatabaseError';
this.detail = detail;
}
}

9
app/src/api/helpers.ts
View File

@ -1,9 +0,0 @@
import { strictParseInt } from '../parse';
export function parseIntParam(param: string) {
const result = strictParseInt(param);
if (isNaN(result)) {
throw new Error('Invalid parameter format: not an integer');
}
return result;
}

44
app/src/api/parameters.ts Normal file
View File

@ -0,0 +1,44 @@
import { strictParseInt } from '../parse';
import { ApiParamError, ApiParamInvalidFormatError, ApiParamRequiredError } from './errors/api';
export function processParam<T>(params: object, paramName: string, processingFn: (x: string) => T, required: boolean = false) {
const stringValue = params[paramName];
if(stringValue == undefined && required) {
const err = new ApiParamRequiredError('Parameter required but not supplied');
err.paramName = paramName;
throw err;
}
try {
return processingFn(stringValue);
} catch(error) {
if(error instanceof ApiParamError) {
error.paramName = paramName;
}
throw error;
}
}
export function parsePositiveIntParam(param: string) {
if(param == undefined) return undefined;
const result = strictParseInt(param);
if (isNaN(result)) {
throw new ApiParamInvalidFormatError('Invalid format: not a positive integer');
}
return result;
}
export function checkRegexParam(param: string, regex: RegExp): string {
if(param == undefined) return undefined;
if(param.match(regex) == undefined) {
throw new ApiParamInvalidFormatError(`Invalid format: does not match regular expression ${regex}`);
}
return param;
}

176
app/src/api/services/__tests__/editHistory.test.ts Normal file
View File

@ -0,0 +1,176 @@
import { EditHistoryEntry } from '../../../frontend/models/edit-history-entry';
import * as editHistoryData from '../../dataAccess/editHistory'; // manually mocked
import { ArgumentError } from '../../errors/general';
import { getGlobalEditHistory } from '../editHistory';
jest.mock('../../dataAccess/editHistory');
const mockedEditHistoryData = editHistoryData as typeof import('../../dataAccess/__mocks__/editHistory');
function generateHistory(n: number, firstId: number = 100) {
return [...Array(n).keys()].map<EditHistoryEntry>(i => ({
revision_id: (firstId + i) + '',
revision_timestamp: new Date(2019, 10, 1, 17, 20 + i).toISOString(),
username: 'testuser',
building_id: 1234567,
forward_patch: {},
reverse_patch: {}
}));
}
describe('getGlobalEditHistory()', () => {
beforeEach(() => mockedEditHistoryData.__setHistory(generateHistory(20)));
afterEach(() => jest.clearAllMocks());
it.each([
[null, null],
['100', null],
[null, '100']
])('Should error when requesting non-positive number of records', async (beforeId: string, afterId: string) => {
let resultPromise = getGlobalEditHistory(beforeId, afterId, 0);
await expect(resultPromise).rejects.toBeInstanceOf(ArgumentError);
await expect(resultPromise).rejects.toHaveProperty('argumentName', 'count');
});
describe('getting history before a point', () => {
it('should return latest history if no ID specified', async () => {
const result = await getGlobalEditHistory(null, null, 5);
expect(result.history.map(x => x.revision_id)).toEqual(['119', '118', '117', '116', '115']);
});
it.each(
[
[null, 3, ['119', '118', '117']],
[null, 6, ['119', '118', '117', '116', '115', '114']],
['118', 1, ['117']],
['104', 10, ['103', '102','101', '100']],
['100', 2, []]
]
)('should return the N records before the specified ID in descending order [beforeId: %p, count: %p]', async (
beforeId: string, count: number, ids: string[]
) => {
const result = await getGlobalEditHistory(beforeId, null, count);
expect(result.history.map(h => h.revision_id)).toEqual(ids);
});
it.each([
[null, 4, null],
[null, 10, null],
[null, 20, null],
[null, 30, null],
['50', 10, '99'],
['100', 10, '99'],
['130', 10, null],
['105', 2, '104'],
['120', 20, null],
])('should detect if there are any newer records left [beforeId: %p, count: %p]', async (
beforeId: string, count: number, idForNewerQuery: string
) => {
const result = await getGlobalEditHistory(beforeId, null, count);
expect(result.paging.id_for_newer_query).toBe(idForNewerQuery);
});
it.each([
[null, 4, '116'],
[null, 10, '110'],
[null, 20, null],
[null, 30, null],
['50', 10, null],
['100', 10, null],
['130', 10, '110'],
['105', 2, '103'],
['120', 20, null],
])('should detect if there are any older records left [beforeId: %p, count: %p]', async (
beforeId: string, count: number, idForOlderQuery: string
) => {
const result = await getGlobalEditHistory(beforeId, null, count);
expect(result.paging.id_for_older_query).toBe(idForOlderQuery);
});
});
describe('getting history after a point', () => {
it.each([
['100', 7, ['107', '106', '105', '104', '103', '102', '101']],
['115', 3, ['118', '117', '116']],
['120', 10, []]
])('should return N records after requested ID in descending order [afterId: %p, count: %p]', async (
afterId: string, count: number, expected: string[]
) => {
const result = await getGlobalEditHistory(null, afterId, count);
expect(result.history.map(x => x.revision_id)).toEqual(expected);
});
it.each([
['99', 10, '109'],
['110', 5, '115'],
['119', 20, null],
['99', 20, null],
])('should detect if there are any newer records left [afterId: %p, count: %p]', async (
afterId: string, count: number, idForNewerQuery: string
) => {
const result = await getGlobalEditHistory(null, afterId, count);
expect(result.paging.id_for_newer_query).toBe(idForNewerQuery);
});
it.each([
['99', 10, null],
['110', 5, '111'],
['119', 20, '120'],
['99', 20, null],
])('should detect if there are any older records left [afterId: %p, count: %p]', async (
afterId: string, count: number, idForOlderQuery: string
) => {
const result = await getGlobalEditHistory(null, afterId, count);
expect(result.paging.id_for_older_query).toBe(idForOlderQuery);
});
});
describe('result count limit', () => {
it.each([
[null, null],
[null, '100'],
['300', null]
])('should not return more than 100 entries (beforeId: %p, afterId: %p)', async (
beforeId: string, afterId: string
) => {
mockedEditHistoryData.__setHistory(
generateHistory(200)
);
const result = await getGlobalEditHistory(beforeId, afterId, 200);
expect(result.history.length).toBe(100);
});
it.each([
[null, null],
[null, '100'],
['300', null]
])('should default to 100 entries', async (
beforeId: string, afterId: string
) => {
mockedEditHistoryData.__setHistory(
generateHistory(200)
);
const result = await getGlobalEditHistory(beforeId, afterId);
expect(result.history.length).toBe(100);
});
});
});

2
app/src/api/services/building.ts
View File

@ -118,7 +118,7 @@ async function getBuildingById(id: number) {
async function getBuildingEditHistory(id: number) {
try {
return await db.manyOrNone(
`SELECT log_id as revision_id, forward_patch, reverse_patch, date_trunc('minute', log_timestamp), username
`SELECT log_id as revision_id, forward_patch, reverse_patch, date_trunc('minute', log_timestamp) as revision_timestamp, username
FROM logs, users
WHERE building_id = $1 AND logs.user_id = users.user_id
ORDER BY log_timestamp DESC`,

3
app/src/api/services/dataExtract.ts
View File

@ -19,7 +19,8 @@ async function listDataExtracts(): Promise<DataExtract[]> {
const extractRecords = await db.manyOrNone<DataExtractRow>(
`SELECT
extract_id, extracted_on, extract_path
FROM bulk_extracts`
FROM bulk_extracts
ORDER BY extracted_on DESC`
);
return extractRecords.map(getDataExtractFromRow);

46
app/src/api/services/editHistory.ts
View File

@ -1,20 +1,38 @@
import db from '../../db';
import { EditHistoryEntry } from '../../frontend/models/edit-history-entry';
import { decBigInt, incBigInt } from '../../helpers';
import { getHistoryAfterId, getHistoryBeforeId, getIdNewerThan, getIdOlderThan } from '../dataAccess/editHistory';
import { ArgumentError } from '../errors/general';
async function getGlobalEditHistory() {
try {
return await db.manyOrNone(
`SELECT log_id as revision_id, forward_patch, reverse_patch, date_trunc('minute', log_timestamp), username, building_id
FROM logs, users
WHERE logs.user_id = users.user_id
AND log_timestamp >= now() - interval '7 days'
ORDER BY log_timestamp DESC`
);
} catch (error) {
console.error(error);
return [];
async function getGlobalEditHistory(beforeId?: string, afterId?: string, count: number = 100) {
if(count <= 0) throw new ArgumentError('cannot request less than 1 history record', 'count');
if(count > 100) count = 100;
// limited set of records. Expected to be already ordered from newest to oldest
let editHistoryRecords: EditHistoryEntry[];
if(afterId != undefined) {
editHistoryRecords = await getHistoryAfterId(afterId, count);
} else {
editHistoryRecords = await getHistoryBeforeId(beforeId, count);
}
}
const currentBatchMaxId = editHistoryRecords[0]?.revision_id ?? decBigInt(beforeId);
const newer = currentBatchMaxId && await getIdNewerThan(currentBatchMaxId);
const currentBatchMinId = editHistoryRecords[editHistoryRecords.length-1]?.revision_id ?? incBigInt(afterId);
const older = currentBatchMinId && await getIdOlderThan(currentBatchMinId);
const idForOlderQuery = older != undefined ? incBigInt(older) : null;
const idForNewerQuery = newer != undefined ? decBigInt(newer) : null;
return {
history: editHistoryRecords,
paging: {
id_for_newer_query: idForNewerQuery,
id_for_older_query: idForOlderQuery
}
};
}
export {
getGlobalEditHistory

4
app/src/frontend/building/data-containers/size.tsx
View File

@ -13,7 +13,7 @@ import { CategoryViewProps } from './category-view-props';
*/
const SizeView: React.FunctionComponent<CategoryViewProps> = (props) => (
<Fragment>
<DataEntryGroup name="Storeys" collapsed={false}>
<DataEntryGroup name="Storeys">
<NumericDataEntry
title={dataFields.size_storeys_core.title}
@ -49,7 +49,7 @@ const SizeView: React.FunctionComponent<CategoryViewProps> = (props) => (
min={0}
/>
</DataEntryGroup>
<DataEntryGroup name="Height">
<DataEntryGroup name="Height" collapsed={false}>
<NumericDataEntry
title={dataFields.size_height_apex.title}
slug="size_height_apex"

2
app/src/frontend/building/edit-history/building-edit-summary.tsx
View File

@ -54,7 +54,7 @@ const BuildingEditSummary: React.FunctionComponent<BuildingEditSummaryProps> = (
return (
<div className="edit-history-entry">
<h2 className="edit-history-timestamp">Edited on {formatDate(parseDate(historyEntry.date_trunc))}</h2>
<h2 className="edit-history-timestamp">Edited on {formatDate(parseDate(historyEntry.revision_timestamp))}</h2>
<h3 className="edit-history-username">By {historyEntry.username}</h3>
{
showBuildingId && historyEntry.building_id != undefined &&

15
app/src/frontend/map/legend.tsx
View File

@ -67,13 +67,16 @@ const LEGEND_CONFIG = {
]
},
size: {
title: 'Number of storeys',
title: 'Height to apex',
elements: [
{ color: '#ffffcc', text: '≥40' },
{ color: '#fed976', text: '20–39' },
{ color: '#fd8d3c', text: '10–19' },
{ color: '#e31a1c', text: '6–9' },
{ color: '#800026', text: '1–5' },
{ color: '#f7f4f9', text: '0-5.55'},
{ color: '#e7e1ef', text: '5.55-7.73'},
{ color: '#d4b9da', text: '7.73-11.38'},
{ color: '#c994c7', text: '11.38-18.45'},
{ color: '#df65b0', text: '18.45-35.05'},
{ color: '#e7298a', text: '35.05-89.30'},
{ color: '#ce1256', text: '89.30-152'},
{ color: '#980043', text: '≥152'}
]
},
construction: {

2
app/src/frontend/map/map.tsx
View File

@ -130,7 +130,7 @@ class ColouringMap extends Component<ColouringMapProps, ColouringMapState> {
const cat = this.props.category;
const tilesetByCat = {
age: 'date_year',
size: 'size_storeys',
size: 'size_height',
location: 'location',
like: 'likes',
planning: 'conservation_area',

2
app/src/frontend/models/edit-history-entry.ts
View File

@ -1,5 +1,5 @@
export interface EditHistoryEntry {
date_trunc: string;
revision_timestamp: string;
username: string;
revision_id: string;
forward_patch: object;

7
app/src/frontend/pages/changes.css Normal file
View File

@ -0,0 +1,7 @@
.edit-history-link {
margin-top: 4em;
}
.edit-history-latest-link {
float: right;
}

75
app/src/frontend/pages/changes.tsx
View File

@ -1,31 +1,88 @@
import { parse } from 'query-string';
import React, { useEffect, useState } from 'react';
import { Link, RouteComponentProps } from 'react-router-dom';
import './changes.css';
import { apiGet } from '../apiHelpers';
import { BuildingEditSummary } from '../building/edit-history/building-edit-summary';
import ErrorBox from '../components/error-box';
import InfoBox from '../components/info-box';
import { EditHistoryEntry } from '../models/edit-history-entry';
const ChangesPage = () => {
const [history, setHistory] = useState<EditHistoryEntry[]>(undefined);
interface PagingInfo {
id_for_older_query: string;
id_for_newer_query: string;
}
const recordsPerPage = 20;
const ChangesPage = (props: RouteComponentProps) => {
const { after_id, before_id } = parse(props.location.search);
const [history, setHistory] = useState<EditHistoryEntry[]>();
const [paging, setPaging] = useState<PagingInfo>();
const [error, setError] = useState();
useEffect(() => {
const fetchData = async () => {
const {history} = await apiGet(`/api/history`);
setHistory(null);
setPaging(null);
let url = `/api/history?count=${recordsPerPage}`;
if(after_id) {
url = `${url}&after_id=${after_id}`;
}
if (before_id) {
url = `${url}&before_id=${before_id}`;
}
try {
const {history, paging, error} = await apiGet(url);
if(error) {
setError(error);
} else {
setHistory(history);
setPaging(paging);
}
} catch (err) {
setError('Connection problem. Please try again later...');
}
setHistory(history);
};
fetchData();
}, []);
}, [props.location.search]);
return (
<article>
<section className="main-col">
<h1>Global edit history</h1>
{
paging?.id_for_newer_query &&
<Link className='edit-history-link' to={`?after_id=${paging.id_for_newer_query}`}>Show more recent changes</Link>
}
{
(after_id || before_id) &&
<Link className='edit-history-latest-link' to='?'>Show latest changes</Link>
}
<ul className="edit-history-list">
{(history == undefined || history.length == 0) ?
<InfoBox msg="No changes in the last week"></InfoBox> :
{
error &&
<ErrorBox msg={error}></ErrorBox>
}
{
error == undefined && history == undefined &&
<InfoBox msg="Loading history..."></InfoBox>
}
{
(history?.length === 0) &&
<InfoBox msg="No changes so far"></InfoBox>
}
{
(history != undefined && history.length > 0) &&
history.map(entry => (
<li key={`${entry.revision_id}`} className="edit-history-list-element">
<BuildingEditSummary
@ -37,6 +94,10 @@ const ChangesPage = () => {
))
}
</ul>
{
paging?.id_for_older_query &&
<Link to={`?before_id=${paging.id_for_older_query}`}>Show older changes</Link>
}
</section>
</article>
);

5
app/src/frontend/pages/data-extracts.tsx
View File

@ -30,10 +30,7 @@ export default class DataExtracts extends React.Component<{}, DataExtractsState>
async componentDidMount() {
let data = await apiGet('/api/extracts', { jsonReviver: dateReviver});
const extracts = (data.extracts as ExtractViewModel[])
.sort((a, b) => b.extracted_on.valueOf() - a.extracted_on.valueOf());
const extracts = (data.extracts as ExtractViewModel[]);
this.setState({ extracts: extracts, latestExtract: extracts[0], previousExtracts: extracts.slice(1) });
}

5
app/src/frontend/styles/layout.css
View File

@ -49,6 +49,11 @@ article .color-block {
padding-left: 1em;
padding-right: 1em;
}
@media(min-width: 768px) {
.main-col {
min-width: 48em;
}
}
hr {
display: block;
height: 1px;

24
app/src/helpers.ts
View File

@ -34,3 +34,27 @@ export function isNullishOrEmpty(obj: any) {
export function isEmptyArray(obj: any) {
return Array.isArray(obj) && obj.length === 0;
}
type AccessorFunction<T, V> = (obj: T) => V;
type CompareFunction<T> = (a: T, b: T) => number;
export function numAsc<T, V extends number | bigint>(accessor: AccessorFunction<T, V>): CompareFunction<T>{
return (a: T, b: T) => Number(accessor(a) - accessor(b));
}
export function numDesc<T, V extends number | bigint>(accessor: AccessorFunction<T, V>): CompareFunction<T> {
return (a: T, b: T) => Number(accessor(b) - accessor(a));
}
/**
* As of Jan 2020, bigint literals e.g. 1n can only be used with TS target esnext
* which then doesn't transpile the null conditional/coalescing operators and breaks on Node 12
* So BigInt(1) needs to be used here
* */
export function incBigInt(bigStr: string): string {
return bigStr == undefined ? bigStr : String(BigInt(bigStr) + BigInt(1));
}
export function decBigInt(bigStr: string): string {
return bigStr == undefined ? bigStr : String(BigInt(bigStr) - BigInt(1));
}

9
app/src/tiles/dataDefinition.ts
View File

@ -46,6 +46,15 @@ const BUILDING_LAYER_DEFINITIONS = {
WHERE
g.geometry_id = b.geometry_id
) as size_stories`,
size_height: `(
SELECT
b.size_height_apex as size_height,
g.geometry_geom
FROM
geometries as g,
buildings as b
WHERE g.geometry_id = b.geometry_id
) as size_height`,
location: `(
SELECT
(

5
app/tsconfig.json
View File

@ -10,7 +10,8 @@
"esModuleInterop": true,
"noImplicitAny": false
},
"include": [
"./src/**/*"
"files": [
"./src/index.ts",
"./src/client.tsx"
]
}

35
docs/adding-new-fields.md Normal file
View File

@ -0,0 +1,35 @@
## Adding new building attribute fields
This document is a checklist for adding a new building attribute to the system. It's split into two sections - actions that apply when adding any field, and additional steps to add a field that will be visualised on the map.
The second section would be required when adding a new category or when changing which field should be visualised for a category.
### Adding any attribute
#### In database
1. Add a column to the `buildings` table in the database.
2. Add any check constraints or foreign key constraints on the column, if necessary (if the foreign key constraint is used to restrict the column to a set of values, the table with the values might need to be created from scratch)
#### In API
1. Add field name to `BUILDING_FIELD_WHITELIST` in the building service to allow saving changes to the field
2. Add any special domain logic for processing updates to the field in the `processBuildingUpdate()` function
#### In frontend
1. Add the field description to the `dataFields` object in `data_fields.ts`
2. Add the data entry React component of the appropriate type (text, numeric etc) to the appropriate category view component in the `building/data-containers` folder. Link to `dataFields` for static string values (field name, description etc)
#### In data extracts
1. Add the field to the select list in the COPY query in `maintenance/extract_data/export_attributes.sql`
2. Add a description of the field to the `README.txt` file
### Adding an attribute which is used to colour the map
All steps from the previous section need to be carried out first.
#### In tileserver
1. Add a SQL query for calculating the value to be visualised to `BUILDING_LAYER_DEFINITIONS` in `app/tiles/dataDefinition.ts`
2. Add Mapnik rendering style in `app/map_styles/polygon.xml`
#### In frontend
1. Update the category to field name mapping in the `tilesetByCat` object inside the `ColouringMap` React component (`map.tsx` file)
2. Add an entry for the field to the `LEGEND_CONFIG` object in `legend.tsx` file

4
docs/setup-dev-environment.md
View File

@ -37,7 +37,7 @@ Now clone the colouring london codebase.
Now install Node. It is helpful to define some local variables.
```
NODE_VERSION=v8.11.3
NODE_VERSION=v12.14.1
DISTRO=linux-x64
wget -nc https://nodejs.org/dist/$NODE_VERSION/node-$NODE_VERSION-$DISTRO.tar.xz
sudo mkdir /usr/local/lib/node
@ -137,7 +137,7 @@ root user profile. Don't forget to exit from root at the end.
```
sudo su root
export NODEJS_HOME=/usr/local/lib/node/node-v8.11.3/bin/`
export NODEJS_HOME=/usr/local/lib/node/node-v12.14.1/bin/`
export PATH=$NODEJS_HOME:$PATH`
npm install -g npm@next`
exit

25
docs/setup-production-environment.md
View File

@ -32,6 +32,10 @@ Install Postgres and associated tools
`sudo apt-get install -y gdal-bin libspatialindex-dev libgeos-dev libproj-dev`
Install Python 3 and pip
`sudo apt-get install python3 python3-pip`
Install Nginx
@ -68,7 +72,7 @@ Now set appropriate permissions on the `colouring-london` directory
First define a couple of convenience variables:
`NODE_VERSION=v8.11.3`
`NODE_VERSION=v12.14.1`
`DISTRO=linux-x64`
@ -107,7 +111,7 @@ Now upgrade the `npm` package manager to the most recent release with global pri
`sudo su root`
`export NODEJS_HOME=/usr/local/lib/node/node-v8.11.3/bin/`
`export NODEJS_HOME=/usr/local/lib/node/node-v12.14.1/bin/`
`export PATH=$NODEJS_HOME:$PATH`
@ -240,7 +244,7 @@ Perform a global install of PM2
`sudo su root`
`export NODEJS_HOME=/usr/local/lib/node/node-v8.11.3/bin/`
`export NODEJS_HOME=/usr/local/lib/node/node-v12.14.1/bin/`
`export PATH=$NODEJS_HOME:$PATH`
@ -303,6 +307,21 @@ To stop the colouring-london app type:
***
#### Set up data extracts
Install requirements for the maintenance Python scripts
`cd /var/www/colouring-london/maintenance`
`sudo pip3 install -r requirements.txt`
The maintenance scripts might need environment variables present at the time of execution, notably the database connection details.
If running the scripts manually, the variables can be provided just before execution, for example
`PGHOST=localhost PGPORT=5432 PGDATABASE=dbname PGUSER=username PGPASSWORD=secretpassword EXTRACTS_DIRECTORY=/var/www/colouring-london/downloads python3 maintenance/extract_data/extract_data.py`
If the maintenance script is to be run on a schedule, the variables should be loaded before running the script, for example from a `.env` file.
#### Set up SSL - TO DO

4
maintenance/extract_data/README.txt
View File

@ -41,7 +41,11 @@ This is the main table, containing almost all data collected by Colouring London
- `location_postcode`: postcode
- `location_latitude`: latitude
- `location_longitude`: longitude
- `current_landuse_class`: current land use class
- `current_landuse_group`: current land use group
- `current_landuse_order`: current land use order
- `building_attachment_form`: building attachment form
- `date_change_building_use`: year of last building use change
- `date_year`: year built
- `date_lower`: lower bound on year built
- `date_upper`: upper bound on year built

4
maintenance/extract_data/export_attributes.sql
View File

@ -11,7 +11,11 @@ COPY (SELECT
location_postcode,
location_latitude,
location_longitude,
current_landuse_class,
current_landuse_group,
current_landuse_order,
building_attachment_form,
date_change_building_use,
date_year,
date_lower,
date_upper,

4
maintenance/extract_data/extract_data.py
View File

@ -64,9 +64,11 @@ def make_data_extract(current_time, connection, zip_file_path):
zip_file_path.parent.mkdir(parents=True, exist_ok=True)
source_dir_path = Path(__file__).parent
try:
with zipfile.ZipFile(zip_file_path, mode='w') as newzip:
newzip.write('README.txt')
newzip.write(source_dir_path / 'README.txt', arcname='README.txt')
newzip.write('/tmp/building_attributes.csv', arcname='building_attributes.csv')
newzip.write('/tmp/building_uprns.csv', arcname='building_uprns.csv')
newzip.write('/tmp/edit_history.csv', arcname='edit_history.csv')

1
maintenance/tasks.cron.txt
View File

@ -1 +0,0 @@
0 5 * * * /var/www/colouringlondon/maintenance/extract_data/extract_data.py

2
provision/vm_provision.sh
View File

@ -40,7 +40,7 @@ apt-get install -y \
#
# node version and platform
NODE_VERSION=v8.11.3
NODE_VERSION=v12.14.1
DISTRO=linux-x64
# download