Merge branch 'master' into dominic-patch-1

This commit is contained in:
Tom Russell 2020-04-09 10:34:56 +01:00 committed by GitHub
commit e17419e017
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
43 changed files with 1098 additions and 141 deletions

2
.gitignore vendored
View File

@ -18,6 +18,8 @@ etl/**/*.xls
etl/**/*.xlsx
etl/**/*.zip
.DS_Store
# Cache
app/tilecache/**/*.png
app/tilecache/**/*.mbtiles

View File

@ -1,6 +1,6 @@
language: node_js
node_js:
- 8
- 12
cache: npm
before_script:
- cd $TRAVIS_BUILD_DIR/app && npm ci

View File

@ -1,13 +0,0 @@
{
"plugins": [
"@babel/plugin-syntax-jsx",
"@babel/plugin-syntax-typescript",
[
"babel-plugin-typescript-to-proptypes",
{
"implicitChildren": true,
"typeCheck": true
}
]
]
}

View File

@ -99,33 +99,29 @@
<PolygonSymbolizer fill="#e7e1ef" />
</Rule>
<Rule>
<Filter>[size_height] &gt;= 7.73 and [size_height] &lt; 1.138</Filter>
<Filter>[size_height] &gt;= 7.73 and [size_height] &lt; 11.38</Filter>
<PolygonSymbolizer fill="#d4b9da" />
</Rule>
<Rule>
<Filter>[size_height] &gt;= 1.138 and [size_height] &lt; 1.845</Filter>
<Filter>[size_height] &gt;= 11.38 and [size_height] &lt; 18.45</Filter>
<PolygonSymbolizer fill="#c994c7" />
</Rule>
<Rule>
<Filter>[size_height] &gt;= 1.845 and [size_height] &lt; 3.505</Filter>
<Filter>[size_height] &gt;= 18.45 and [size_height] &lt; 35.05</Filter>
<PolygonSymbolizer fill="#df65b0" />
</Rule>
<Rule>
<Filter>[size_height] &gt;= 3.505 and [size_height] &lt; 8.930</Filter>
<Filter>[size_height] &gt;= 35.05 and [size_height] &lt; 89.30</Filter>
<PolygonSymbolizer fill="#e7298a" />
</Rule>
<Rule>
<Filter>[size_height] &gt;= 8.930 and [size_height] &lt; 15.179</Filter>
<Filter>[size_height] &gt;= 89.30 and [size_height] &lt; 152</Filter>
<PolygonSymbolizer fill="#ce1256" />
</Rule>
<Rule>
<Filter>[size_height] &gt;= 15.179 and [size_height] &lt; 99.999</Filter>
<Filter>[size_height] &gt;= 152</Filter>
<PolygonSymbolizer fill="#980043" />
</Rule>
<Rule>
<Filter>[size_height] &gt;= 99.999</Filter>
<PolygonSymbolizer fill="#67001f" />
</Rule>
</Style>
<Style name="date_year">
<Rule>
@ -299,7 +295,7 @@
</Rule>
<Rule>
<Filter>[current_landuse_order] = "Transport"</Filter>
<PolygonSymbolizer fill="#b3de69" />
<PolygonSymbolizer fill="#bee8ff" />
</Rule>
<Rule>
<Filter>[current_landuse_order] = "Utilities And Infrastructure"</Filter>

26
app/package-lock.json generated
View File

@ -425,15 +425,6 @@
"@babel/helper-plugin-utils": "^7.0.0"
}
},
"@babel/plugin-syntax-typescript": {
"version": "7.7.4",
"resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.7.4.tgz",
"integrity": "sha512-77blgY18Hud4NM1ggTA8xVT/dBENQf17OpiToSa2jSmEY3fWXD2jwrdVlO4kq5yzUTeF15WSQ6b4fByNvJcjpQ==",
"dev": true,
"requires": {
"@babel/helper-plugin-utils": "^7.0.0"
}
},
"@babel/plugin-transform-arrow-functions": {
"version": "7.2.0",
"resolved": "https://registry.npmjs.org/@babel/plugin-transform-arrow-functions/-/plugin-transform-arrow-functions-7.2.0.tgz",
@ -1189,9 +1180,9 @@
"dev": true
},
"@types/node": {
"version": "8.10.59",
"resolved": "https://registry.npmjs.org/@types/node/-/node-8.10.59.tgz",
"integrity": "sha512-8RkBivJrDCyPpBXhVZcjh7cQxVBSmRk9QM7hOketZzp6Tg79c0N8kkpAIito9bnJ3HCVCHVYz+KHTEbfQNfeVQ==",
"version": "12.12.25",
"resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.25.tgz",
"integrity": "sha512-nf1LMGZvgFX186geVZR1xMZKKblJiRfiASTHw85zED2kI1yDKHDwTKMdkaCbTlXoRKlGKaDfYywt+V0As30q3w==",
"dev": true
},
"@types/nodemailer": {
@ -2420,17 +2411,6 @@
"integrity": "sha512-f49NsaohQ1ByY20nUrpc30QFdbeT4ntV4PAL2vSZe6uCB5nqAcqXS/qzU+aI6ZfYhWASx5eIsTFvFrs1B2ffGg==",
"dev": true
},
"babel-plugin-typescript-to-proptypes": {
"version": "0.17.1",
"resolved": "https://registry.npmjs.org/babel-plugin-typescript-to-proptypes/-/babel-plugin-typescript-to-proptypes-0.17.1.tgz",
"integrity": "sha512-yREUfvDlmn6QjM0QbywXUkXBQMD/iFfLVTl+jig4X7ZLUg9lq8ZLuex8HIM2SQ4X3vcjGnWPFowodlMcXhwxdQ==",
"dev": true,
"requires": {
"@babel/helper-module-imports": "^7.0.0",
"@babel/helper-plugin-utils": "^7.0.0",
"@babel/plugin-syntax-typescript": "^7.2.0"
}
},
"babel-preset-jest": {
"version": "23.2.0",
"resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-23.2.0.tgz",

View File

@ -26,7 +26,6 @@
"node-fs": "^0.1.7",
"nodemailer": "^6.3.0",
"pg-promise": "^8.7.5",
"prop-types": "^15.7.2",
"query-string": "^6.8.2",
"react": "^16.9.0",
"react-dom": "^16.9.0",
@ -37,16 +36,13 @@
"sharp": "^0.22.1"
},
"devDependencies": {
"@babel/plugin-syntax-jsx": "^7.7.4",
"@babel/plugin-syntax-typescript": "^7.7.4",
"@types/express": "^4.17.2",
"@types/express-session": "^1.15.16",
"@types/jest": "^24.0.23",
"@types/lodash": "^4.14.149",
"@types/mapbox__sphericalmercator": "^1.1.3",
"@types/node": "^8.10.59",
"@types/node": "^12.12.25",
"@types/nodemailer": "^6.2.2",
"@types/prop-types": "^15.7.3",
"@types/react": "^16.9.16",
"@types/react-dom": "^16.9.4",
"@types/react-leaflet": "^2.5.0",
@ -54,7 +50,6 @@
"@types/sharp": "^0.22.3",
"@types/webpack-env": "^1.14.1",
"babel-eslint": "^10.0.3",
"babel-plugin-typescript-to-proptypes": "^0.17.1",
"eslint": "^5.16.0",
"eslint-plugin-jest": "^22.21.0",
"eslint-plugin-react": "^7.17.0",

155
app/public/openapi.yml Normal file
View File

@ -0,0 +1,155 @@
openapi: 3.0.0
info:
title: Colouring London API
version: 1.0.0
servers:
- url: https://colouring.london/api
description: Production server (uses live data)
paths:
/extracts:
get:
summary: Returns a list of bulk data extracts
responses:
'200':
description: A list of bulk extracts, from newest to oldest
content:
application/json:
schema:
properties:
extracts:
type: array
items:
$ref: '#/components/schemas/BulkExtract'
example:
extracts:
- extract_id: 1
extracted_on: 2019-10-03T05:33:00.000Z
download_path: /downloads/data-extract-2019-10-03-06_33_00.zip
'400':
$ref: '#/components/responses/BadRequest'
'500':
$ref: '#/components/responses/ServerError'
/history:
get:
summary: Returns a paginated list of edits (latest edits if no relevant parameters specified)
parameters:
- name: before_id
description: Returned edits will be ones made directly before the specified revision ID
in: query
schema:
$ref: '#/components/schemas/RevisionId'
required: false
- name: after_id
description: Returned edits will be ones made directly after the specified revision ID
in: query
schema:
$ref: '#/components/schemas/RevisionId'
required: false
- name: count
description: The desired number of records to return
in: query
schema:
type: number
minimum: 1
maximum: 100
default: 100
required: false
responses:
'200':
description: A list of edit history records
content:
application/json:
schema:
properties:
history:
type: array
items:
$ref: '#/components/schemas/BuildingEditHistoryEntry'
paging:
type: object
properties:
id_for_older_query:
allOf:
- $ref: '#/components/schemas/RevisionId'
- description: If older records exist - ID to use for querying them (use as before_id param), otherwise null
nullable: true
id_for_newer_query:
allOf:
- $ref: '#/components/schemas/RevisionId'
- description: If newer records exist - ID to use for querying them (use as after_id param), otherwise null
nullable: true
'400':
$ref: '#/components/responses/BadRequest'
'500':
$ref: '#/components/responses/ServerError'
components:
responses:
BadRequest:
description: Invalid request submitted by user
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
ServerError:
description: Unexpected server error
content:
application/json:
schema:
$ref: '#/components/schemas/Error'
example:
error: Database error
schemas:
Error:
type: object
properties:
error:
type: string
description: Error message
BulkExtract:
type: object
properties:
extract_id:
type: integer
description: Unique sequential ID for the extract
extracted_on:
type: string
format: date-time
description: UTC timestamp at which the extract was generated
download_path:
type: string
description: Download path for the extract. Contains only URL path (should be used with the same hostname as the API).
RevisionId:
description: Unique sequential ID for an edit history entry (positive big integer)
type: string
pattern: ^[1-9]\d*&
BuildingEditHistoryEntry:
type: object
properties:
revision_id:
$ref: '#/components/schemas/RevisionId'
forward_patch:
type: object
description: Forward diff of the building attribute data
reverse_patch:
type: object
description: Reverse diff of the building attribute data
revision_timestamp:
type: string
format: date-time
description: UTC timestamp at which the building data was edited
username:
type: string
description: Username of the editor
building_id:
type: number
description: Unique ID of the edited building

View File

@ -9,17 +9,6 @@ module.exports = {
})
config.module.rules = rules;
// find module rule that runs ts-loader for TS(X) files
const tsRule = config.module.rules.find(r =>
new RegExp(r.test).test('test.tsx') && Array.isArray(r.use) && r.use.some(u => u.loader.includes('ts-loader')));
// run babel-loader before ts-loader to generate propTypes
tsRule.use.push({
loader: 'babel-loader',
options: {
babelrc: true
}
})
return config;
},
};

View File

@ -2,13 +2,15 @@ import bodyParser from 'body-parser';
import express from 'express';
import * as editHistoryController from './controllers/editHistoryController';
import { ApiParamError, ApiUserError } from './errors/api';
import { DatabaseError } from './errors/general';
import buildingsRouter from './routes/buildingsRouter';
import extractsRouter from './routes/extractsRouter';
import leaderboardRouter from './routes/leaderboardRouter';
import usersRouter from './routes/usersRouter';
import { queryLocation } from './services/search';
import { authUser, getNewUserAPIKey, logout } from './services/user';
const server = express.Router();
// parse POSTed json body
@ -17,6 +19,7 @@ server.use(bodyParser.json());
server.use('/buildings', buildingsRouter);
server.use('/users', usersRouter);
server.use('/extracts', extractsRouter);
server.use('/leaderboard', leaderboardRouter);
server.get('/history', editHistoryController.getGlobalEditHistory);
@ -93,14 +96,30 @@ server.get('/search', function (req, res) {
});
});
server.use((err, req, res, next) => {
server.use((err: any, req: express.Request, res: express.Response, next: express.NextFunction) => {
if (res.headersSent) {
return next(err);
}
if (err != undefined) {
console.log('Global error handler: ', err);
res.status(500).send({ error: 'Server error' });
if (err instanceof ApiUserError) {
let errorMessage: string;
if(err instanceof ApiParamError) {
errorMessage = `Problem with parameter ${err.paramName}: ${err.message}`;
} else {
errorMessage = err.message;
}
res.status(400).send({ error: errorMessage });
} else if(err instanceof DatabaseError){
res.status(500).send({ error: 'Database error' });
} else {
res.status(500).send({ error: 'Server error' });
}
}
});

View File

@ -1,6 +1,6 @@
import express from 'express';
import { parseIntParam } from '../helpers';
import { parsePositiveIntParam, processParam } from '../parameters';
import asyncController from '../routes/asyncController';
import * as buildingService from '../services/building';
import * as userService from '../services/user';
@ -35,7 +35,7 @@ const getBuildingsByReference = asyncController(async (req: express.Request, res
// GET individual building, POST building updates
const getBuildingById = asyncController(async (req: express.Request, res: express.Response) => {
const buildingId = parseIntParam(req.params.building_id);
const buildingId = processParam(req.params, 'building_id', parsePositiveIntParam, true);
try {
const result = await buildingService.getBuildingById(buildingId);
@ -63,7 +63,7 @@ const updateBuildingById = asyncController(async (req: express.Request, res: exp
});
async function updateBuilding(req: express.Request, res: express.Response, userId: string) {
const buildingId = parseIntParam(req.params.building_id);
const buildingId = processParam(req.params, 'building_id', parsePositiveIntParam, true);
const buildingUpdate = req.body;
@ -84,7 +84,7 @@ async function updateBuilding(req: express.Request, res: express.Response, userI
// GET building UPRNs
const getBuildingUPRNsById = asyncController(async (req: express.Request, res: express.Response) => {
const buildingId = parseIntParam(req.params.building_id);
const buildingId = processParam(req.params, 'building_id', parsePositiveIntParam, true);
try {
const result = await buildingService.getBuildingUPRNsById(buildingId);
@ -105,7 +105,7 @@ const getBuildingLikeById = asyncController(async (req: express.Request, res: ex
return res.send({ like: false }); // not logged in, so cannot have liked
}
const buildingId = parseIntParam(req.params.building_id);
const buildingId = processParam(req.params, 'building_id', parsePositiveIntParam, true);
try {
const like = await buildingService.getBuildingLikeById(buildingId, req.session.user_id);
@ -118,7 +118,7 @@ const getBuildingLikeById = asyncController(async (req: express.Request, res: ex
});
const getBuildingEditHistoryById = asyncController(async (req: express.Request, res: express.Response) => {
const buildingId = parseIntParam(req.params.building_id);
const buildingId = processParam(req.params, 'building_id', parsePositiveIntParam, true);
try {
const editHistory = await buildingService.getBuildingEditHistory(buildingId);
@ -134,7 +134,7 @@ const updateBuildingLikeById = asyncController(async (req: express.Request, res:
return res.send({ error: 'Must be logged in' });
}
const buildingId = parseIntParam(req.params.building_id);
const buildingId = processParam(req.params, 'building_id', parsePositiveIntParam, true);
const { like } = req.body;
try {

View File

@ -1,17 +1,32 @@
import express from 'express';
import { ApiParamError, ApiUserError } from '../errors/api';
import { ArgumentError } from '../errors/general';
import { checkRegexParam, parsePositiveIntParam, processParam } from '../parameters';
import asyncController from "../routes/asyncController";
import * as editHistoryService from '../services/editHistory';
const getGlobalEditHistory = asyncController(async (req: express.Request, res: express.Response) => {
const revisionIdRegex = /^[1-9]\d*$/;
const afterId: string = processParam(req.query, 'after_id', x => checkRegexParam(x, revisionIdRegex));
const beforeId: string = processParam(req.query, 'before_id', x => checkRegexParam(x, revisionIdRegex));
const count: number = processParam(req.query, 'count', parsePositiveIntParam);
if(afterId != undefined && beforeId != undefined) {
throw new ApiUserError('Cannot specify both after_id and before_id parameters');
}
try {
const result = await editHistoryService.getGlobalEditHistory();
res.send({
history: result
});
const result = await editHistoryService.getGlobalEditHistory(beforeId, afterId, count);
res.send(result);
} catch(error) {
console.error(error);
res.send({ error: 'Database error' });
if(error instanceof ArgumentError && error.argumentName === 'count') {
const apiErr = new ApiParamError(error.message, 'count');
throw apiErr;
}
throw error;
}
});

View File

@ -1,6 +1,6 @@
import express from 'express';
import { parseIntParam } from '../helpers';
import { parsePositiveIntParam, processParam } from '../parameters';
import asyncController from '../routes/asyncController';
import * as dataExtractService from '../services/dataExtract';
@ -15,8 +15,9 @@ const getAllDataExtracts = asyncController(async function(req: express.Request,
});
const getDataExtract = asyncController(async function(req: express.Request, res: express.Response) {
const extractId = processParam(req.params, 'extract_id', parsePositiveIntParam, true);
try {
const extractId = parseIntParam(req.params.extract_id);
const extract = await dataExtractService.getDataExtractById(extractId);
res.send({ extract: extract });
} catch (err) {

View File

@ -0,0 +1,22 @@
import express from 'express';
import asyncController from "../routes/asyncController";
import * as leadersService from '../services/leaderboard';
const getLeaders = asyncController(async (req: express.Request, res: express.Response) => {
try {
const number_limit = req.query.number_limit;
const time_limit = req.query.time_limit;
const result = await leadersService.getLeaders(number_limit, time_limit);
res.send({
leaders: result
});
} catch(error) {
console.error(error);
res.send({ error: 'Database error' });
}
});
export default{
getLeaders
};

View File

@ -0,0 +1,62 @@
import { EditHistoryEntry } from '../../../frontend/models/edit-history-entry';
import { numAsc, numDesc } from '../../../helpers';
/**
* Create an object mocking all method of editHistory dataAccess
* The type is set to reflect the type of that module, with added methods
* used when testing
*/
const mockEditHistory =
jest.genMockFromModule('../editHistory') as typeof import('../editHistory') & {
__setHistory: (mockHistoryData: EditHistoryEntry[]) => void
};
let mockData: EditHistoryEntry[] = [];
mockEditHistory.__setHistory = function(mockHistoryData: EditHistoryEntry[]) {
mockData = mockHistoryData.sort(numDesc(x => BigInt(x.revision_id)));
};
mockEditHistory.getHistoryAfterId = function(id: string, count: number): Promise<EditHistoryEntry[]> {
return Promise.resolve(
mockData
.filter(x => BigInt(x.revision_id) > BigInt(id))
.sort(numAsc(x => BigInt(x.revision_id)))
.slice(0, count)
.sort(numDesc(x => BigInt(x.revision_id)))
);
};
mockEditHistory.getHistoryBeforeId = function(id: string, count: number): Promise<EditHistoryEntry[]> {
let filteredData = id == undefined ? mockData : mockData.filter(x => BigInt(x.revision_id) < BigInt(id));
return Promise.resolve(
filteredData
.slice(0, count)
);
};
mockEditHistory.getIdNewerThan = async function(id: string): Promise<string> {
const historyAfterId = await mockEditHistory.getHistoryAfterId(id, 1);
return historyAfterId[historyAfterId.length - 1]?.revision_id;
};
mockEditHistory.getIdOlderThan = async function(id: string): Promise<string> {
const historyBeforeId = await mockEditHistory.getHistoryBeforeId(id, 1);
return historyBeforeId[0]?.revision_id;
};
const {
__setHistory,
getHistoryAfterId,
getHistoryBeforeId,
getIdNewerThan,
getIdOlderThan
} = mockEditHistory;
export {
__setHistory,
getHistoryAfterId,
getHistoryBeforeId,
getIdNewerThan,
getIdOlderThan
};

View File

@ -0,0 +1,88 @@
import db from '../../db';
import { EditHistoryEntry } from '../../frontend/models/edit-history-entry';
import { DatabaseError } from '../errors/general';
const baseQuery = `
SELECT
log_id as revision_id,
forward_patch,
reverse_patch,
date_trunc('minute', log_timestamp) as revision_timestamp,
username,
building_id
FROM logs
JOIN users ON logs.user_id = users.user_id`;
export function getHistoryAfterId(id: string, count: number): Promise<EditHistoryEntry[]> {
/**
* SQL with lower time bound specified (records after ID).
* The outer SELECT is so that final results are sorted by descending ID
* (like the other queries). The inner select is sorted in ascending order
* so that the right rows are returned when limiting the result set.
*/
try {
return db.any(`
SELECT * FROM (
${baseQuery}
WHERE log_id > $1
ORDER BY revision_id ASC
LIMIT $2
) AS result_asc ORDER BY revision_id DESC`,
[id, count]
);
} catch(err) {
throw new DatabaseError(err);
}
}
export function getHistoryBeforeId(id: string, count: number): Promise<EditHistoryEntry[]> {
try {
if(id == undefined) {
return db.any(`
${baseQuery}
ORDER BY revision_id DESC
LIMIT $1
`, [count]);
} else {
return db.any(`
${baseQuery}
WHERE log_id < $1
ORDER BY revision_id DESC
LIMIT $2
`, [id, count]);
}
} catch(err) {
throw new DatabaseError(err);
}
}
export async function getIdOlderThan(id: string): Promise<string> {
try {
const result = await db.oneOrNone<{revision_id:string}>(`
SELECT MAX(log_id) as revision_id
FROM logs
WHERE log_id < $1
`, [id]);
return result?.revision_id;
} catch(err) {
throw new DatabaseError(err);
}
}
export async function getIdNewerThan(id: string): Promise<string> {
try {
const result = await db.oneOrNone<{revision_id:string}>(`
SELECT MIN(log_id) as revision_id
FROM logs
WHERE log_id > $1
`, [id]);
return result?.revision_id;
} catch(err) {
throw new DatabaseError(err);
}
}

44
app/src/api/errors/api.ts Normal file
View File

@ -0,0 +1,44 @@
/**
* Note that custom errors and the instanceof operator in TS work together
* only when transpiling to ES2015 and up.
* For earier target versions (ES5), a workaround is required:
* https://stackoverflow.com/questions/41102060/typescript-extending-error-class
*/
export class ApiUserError extends Error {
constructor(message?: string) {
super(message);
this.name = 'ApiUserError';
}
}
export class ApiParamError extends ApiUserError {
public paramName: string;
constructor(message?: string, paramName?: string) {
super(message);
this.name = 'ApiParamError';
this.paramName = paramName;
}
}
export class ApiParamRequiredError extends ApiParamError {
constructor(message?: string) {
super(message);
this.name = 'ApiParamRequiredError';
}
}
export class ApiParamOutOfBoundsError extends ApiParamError {
constructor(message?: string) {
super(message);
this.name = 'ApiParamOutOfBoundsError';
}
}
export class ApiParamInvalidFormatError extends ApiParamError {
constructor(message?: string) {
super(message);
this.name = 'ApiParamInvalidFormatError';
}
}

View File

@ -0,0 +1,17 @@
export class ArgumentError extends Error {
public argumentName: string;
constructor(message?: string, argumentName?: string) {
super(message);
this.name = 'ArgumentError';
this.argumentName = argumentName;
}
}
export class DatabaseError extends Error {
public detail: any;
constructor(detail?: string) {
super();
this.name = 'DatabaseError';
this.detail = detail;
}
}

View File

@ -1,9 +0,0 @@
import { strictParseInt } from '../parse';
export function parseIntParam(param: string) {
const result = strictParseInt(param);
if (isNaN(result)) {
throw new Error('Invalid parameter format: not an integer');
}
return result;
}

44
app/src/api/parameters.ts Normal file
View File

@ -0,0 +1,44 @@
import { strictParseInt } from '../parse';
import { ApiParamError, ApiParamInvalidFormatError, ApiParamRequiredError } from './errors/api';
export function processParam<T>(params: object, paramName: string, processingFn: (x: string) => T, required: boolean = false) {
const stringValue = params[paramName];
if(stringValue == undefined && required) {
const err = new ApiParamRequiredError('Parameter required but not supplied');
err.paramName = paramName;
throw err;
}
try {
return processingFn(stringValue);
} catch(error) {
if(error instanceof ApiParamError) {
error.paramName = paramName;
}
throw error;
}
}
export function parsePositiveIntParam(param: string) {
if(param == undefined) return undefined;
const result = strictParseInt(param);
if (isNaN(result)) {
throw new ApiParamInvalidFormatError('Invalid format: not a positive integer');
}
return result;
}
export function checkRegexParam(param: string, regex: RegExp): string {
if(param == undefined) return undefined;
if(param.match(regex) == undefined) {
throw new ApiParamInvalidFormatError(`Invalid format: does not match regular expression ${regex}`);
}
return param;
}

View File

@ -0,0 +1,9 @@
import express from 'express';
import leaderboardController from '../controllers/leaderboardController';
const router = express.Router();
router.get('/leaders', leaderboardController.getLeaders);
export default router;

View File

@ -0,0 +1,176 @@
import { EditHistoryEntry } from '../../../frontend/models/edit-history-entry';
import * as editHistoryData from '../../dataAccess/editHistory'; // manually mocked
import { ArgumentError } from '../../errors/general';
import { getGlobalEditHistory } from '../editHistory';
jest.mock('../../dataAccess/editHistory');
const mockedEditHistoryData = editHistoryData as typeof import('../../dataAccess/__mocks__/editHistory');
function generateHistory(n: number, firstId: number = 100) {
return [...Array(n).keys()].map<EditHistoryEntry>(i => ({
revision_id: (firstId + i) + '',
revision_timestamp: new Date(2019, 10, 1, 17, 20 + i).toISOString(),
username: 'testuser',
building_id: 1234567,
forward_patch: {},
reverse_patch: {}
}));
}
describe('getGlobalEditHistory()', () => {
beforeEach(() => mockedEditHistoryData.__setHistory(generateHistory(20)));
afterEach(() => jest.clearAllMocks());
it.each([
[null, null],
['100', null],
[null, '100']
])('Should error when requesting non-positive number of records', async (beforeId: string, afterId: string) => {
let resultPromise = getGlobalEditHistory(beforeId, afterId, 0);
await expect(resultPromise).rejects.toBeInstanceOf(ArgumentError);
await expect(resultPromise).rejects.toHaveProperty('argumentName', 'count');
});
describe('getting history before a point', () => {
it('should return latest history if no ID specified', async () => {
const result = await getGlobalEditHistory(null, null, 5);
expect(result.history.map(x => x.revision_id)).toEqual(['119', '118', '117', '116', '115']);
});
it.each(
[
[null, 3, ['119', '118', '117']],
[null, 6, ['119', '118', '117', '116', '115', '114']],
['118', 1, ['117']],
['104', 10, ['103', '102','101', '100']],
['100', 2, []]
]
)('should return the N records before the specified ID in descending order [beforeId: %p, count: %p]', async (
beforeId: string, count: number, ids: string[]
) => {
const result = await getGlobalEditHistory(beforeId, null, count);
expect(result.history.map(h => h.revision_id)).toEqual(ids);
});
it.each([
[null, 4, null],
[null, 10, null],
[null, 20, null],
[null, 30, null],
['50', 10, '99'],
['100', 10, '99'],
['130', 10, null],
['105', 2, '104'],
['120', 20, null],
])('should detect if there are any newer records left [beforeId: %p, count: %p]', async (
beforeId: string, count: number, idForNewerQuery: string
) => {
const result = await getGlobalEditHistory(beforeId, null, count);
expect(result.paging.id_for_newer_query).toBe(idForNewerQuery);
});
it.each([
[null, 4, '116'],
[null, 10, '110'],
[null, 20, null],
[null, 30, null],
['50', 10, null],
['100', 10, null],
['130', 10, '110'],
['105', 2, '103'],
['120', 20, null],
])('should detect if there are any older records left [beforeId: %p, count: %p]', async (
beforeId: string, count: number, idForOlderQuery: string
) => {
const result = await getGlobalEditHistory(beforeId, null, count);
expect(result.paging.id_for_older_query).toBe(idForOlderQuery);
});
});
describe('getting history after a point', () => {
it.each([
['100', 7, ['107', '106', '105', '104', '103', '102', '101']],
['115', 3, ['118', '117', '116']],
['120', 10, []]
])('should return N records after requested ID in descending order [afterId: %p, count: %p]', async (
afterId: string, count: number, expected: string[]
) => {
const result = await getGlobalEditHistory(null, afterId, count);
expect(result.history.map(x => x.revision_id)).toEqual(expected);
});
it.each([
['99', 10, '109'],
['110', 5, '115'],
['119', 20, null],
['99', 20, null],
])('should detect if there are any newer records left [afterId: %p, count: %p]', async (
afterId: string, count: number, idForNewerQuery: string
) => {
const result = await getGlobalEditHistory(null, afterId, count);
expect(result.paging.id_for_newer_query).toBe(idForNewerQuery);
});
it.each([
['99', 10, null],
['110', 5, '111'],
['119', 20, '120'],
['99', 20, null],
])('should detect if there are any older records left [afterId: %p, count: %p]', async (
afterId: string, count: number, idForOlderQuery: string
) => {
const result = await getGlobalEditHistory(null, afterId, count);
expect(result.paging.id_for_older_query).toBe(idForOlderQuery);
});
});
describe('result count limit', () => {
it.each([
[null, null],
[null, '100'],
['300', null]
])('should not return more than 100 entries (beforeId: %p, afterId: %p)', async (
beforeId: string, afterId: string
) => {
mockedEditHistoryData.__setHistory(
generateHistory(200)
);
const result = await getGlobalEditHistory(beforeId, afterId, 200);
expect(result.history.length).toBe(100);
});
it.each([
[null, null],
[null, '100'],
['300', null]
])('should default to 100 entries', async (
beforeId: string, afterId: string
) => {
mockedEditHistoryData.__setHistory(
generateHistory(200)
);
const result = await getGlobalEditHistory(beforeId, afterId);
expect(result.history.length).toBe(100);
});
});
});

View File

@ -118,7 +118,7 @@ async function getBuildingById(id: number) {
async function getBuildingEditHistory(id: number) {
try {
return await db.manyOrNone(
`SELECT log_id as revision_id, forward_patch, reverse_patch, date_trunc('minute', log_timestamp), username
`SELECT log_id as revision_id, forward_patch, reverse_patch, date_trunc('minute', log_timestamp) as revision_timestamp, username
FROM logs, users
WHERE building_id = $1 AND logs.user_id = users.user_id
ORDER BY log_timestamp DESC`,

View File

@ -1,20 +1,38 @@
import db from '../../db';
import { EditHistoryEntry } from '../../frontend/models/edit-history-entry';
import { decBigInt, incBigInt } from '../../helpers';
import { getHistoryAfterId, getHistoryBeforeId, getIdNewerThan, getIdOlderThan } from '../dataAccess/editHistory';
import { ArgumentError } from '../errors/general';
async function getGlobalEditHistory() {
try {
return await db.manyOrNone(
`SELECT log_id as revision_id, forward_patch, reverse_patch, date_trunc('minute', log_timestamp), username, building_id
FROM logs, users
WHERE logs.user_id = users.user_id
AND log_timestamp >= now() - interval '7 days'
ORDER BY log_timestamp DESC`
);
} catch (error) {
console.error(error);
return [];
async function getGlobalEditHistory(beforeId?: string, afterId?: string, count: number = 100) {
if(count <= 0) throw new ArgumentError('cannot request less than 1 history record', 'count');
if(count > 100) count = 100;
// limited set of records. Expected to be already ordered from newest to oldest
let editHistoryRecords: EditHistoryEntry[];
if(afterId != undefined) {
editHistoryRecords = await getHistoryAfterId(afterId, count);
} else {
editHistoryRecords = await getHistoryBeforeId(beforeId, count);
}
}
const currentBatchMaxId = editHistoryRecords[0]?.revision_id ?? decBigInt(beforeId);
const newer = currentBatchMaxId && await getIdNewerThan(currentBatchMaxId);
const currentBatchMinId = editHistoryRecords[editHistoryRecords.length-1]?.revision_id ?? incBigInt(afterId);
const older = currentBatchMinId && await getIdOlderThan(currentBatchMinId);
const idForOlderQuery = older != undefined ? incBigInt(older) : null;
const idForNewerQuery = newer != undefined ? decBigInt(newer) : null;
return {
history: editHistoryRecords,
paging: {
id_for_newer_query: idForNewerQuery,
id_for_older_query: idForOlderQuery
}
};
}
export {
getGlobalEditHistory

View File

@ -0,0 +1,39 @@
import db from '../../db';
async function getLeaders(number_limit: number, time_limit: number) {
try {
if(time_limit > 0){
return await db.manyOrNone(
`SELECT count(log_id) as number_edits, username
FROM logs, users
WHERE logs.user_id=users.user_id
AND CURRENT_TIMESTAMP::DATE - log_timestamp::DATE <= $1
AND NOT (users.username = 'casa_friendly_robot')
AND NOT (users.username = 'colouringlondon')
GROUP by users.username
ORDER BY number_edits DESC
LIMIT $2`, [time_limit, number_limit]
);
}else{
return await db.manyOrNone(
`SELECT count(log_id) as number_edits, username
FROM logs, users
WHERE logs.user_id=users.user_id
AND NOT (users.username = 'casa_friendly_robot')
AND NOT (users.username = 'colouringlondon')
GROUP by users.username
ORDER BY number_edits DESC
LIMIT $1`, [number_limit]
);
}
} catch(error) {
console.error(error);
return [];
}
}
export {
getLeaders
};

View File

@ -14,6 +14,7 @@ import ContactPage from './pages/contact';
import ContributorAgreementPage from './pages/contributor-agreement';
import DataAccuracyPage from './pages/data-accuracy';
import DataExtracts from './pages/data-extracts';
import LeaderboardPage from './pages/leaderboard';
import OrdnanceSurveyLicencePage from './pages/ordnance-survey-licence';
import OrdnanceSurveyUprnPage from './pages/ordnance-survey-uprn';
import PrivacyPolicyPage from './pages/privacy-policy';
@ -113,6 +114,7 @@ class App extends React.Component<AppProps, AppState> {
<Route exact path="/data-accuracy.html" component={DataAccuracyPage} />
<Route exact path="/data-extracts.html" component={DataExtracts} />
<Route exact path="/contact.html" component={ContactPage} />
<Route exact path="/leaderboard.html" component={LeaderboardPage} />
<Route exact path="/history.html" component={ChangesPage} />
<Route exact path={App.mapAppPaths} render={(props) => (
<MapApp

View File

@ -54,7 +54,7 @@ const BuildingEditSummary: React.FunctionComponent<BuildingEditSummaryProps> = (
return (
<div className="edit-history-entry">
<h2 className="edit-history-timestamp">Edited on {formatDate(parseDate(historyEntry.date_trunc))}</h2>
<h2 className="edit-history-timestamp">Edited on {formatDate(parseDate(historyEntry.revision_timestamp))}</h2>
<h3 className="edit-history-username">By {historyEntry.username}</h3>
{
showBuildingId && historyEntry.building_id != undefined &&

View File

@ -89,6 +89,11 @@ class Header extends React.Component<HeaderProps, HeaderState> {
Downloads
</NavLink>
</li>
<li className="nav-item">
<NavLink to="/leaderboard.html" className="nav-link" onClick={this.handleNavigate}>
Leaderboard
</NavLink>
</li>
<li className="nav-item">
<NavLink to="/contact.html" className="nav-link" onClick={this.handleNavigate}>
Contact

View File

@ -26,7 +26,7 @@ const LEGEND_CONFIG = {
{ color: '#f5f58f', text: 'Industry And Business' },
{ color: '#73ccd1', text: 'Community Services' },
{ color: '#ffbfbf', text: 'Recreation And Leisure' },
{ color: '#b3de69', text: 'Transport' },
{ color: '#bee8ff', text: 'Transport' },
{ color: '#cccccc', text: 'Utilities And Infrastructure' },
{ color: '#52403C', text: 'Agriculture And Fisheries' },
{ color: '#898944', text: 'Defence' },
@ -71,13 +71,12 @@ const LEGEND_CONFIG = {
elements: [
{ color: '#f7f4f9', text: '0-5.55'},
{ color: '#e7e1ef', text: '5.55-7.73'},
{ color: '#d4b9da', text: '7.73-1.138'},
{ color: '#c994c7', text: '1.138-1.845'},
{ color: '#df65b0', text: '1.845-3.505'},
{ color: '#e7298a', text: '3.505-8.930'},
{ color: '#ce1256', text: '8.930-15.179'},
{ color: '#980043', text: '15.179-99.999'},
{ color: '#67001f', text: '≥99.999'}
{ color: '#d4b9da', text: '7.73-11.38'},
{ color: '#c994c7', text: '11.38-18.45'},
{ color: '#df65b0', text: '18.45-35.05'},
{ color: '#e7298a', text: '35.05-89.30'},
{ color: '#ce1256', text: '89.30-152'},
{ color: '#980043', text: '≥152'}
]
},
construction: {

View File

@ -1,5 +1,5 @@
export interface EditHistoryEntry {
date_trunc: string;
revision_timestamp: string;
username: string;
revision_id: string;
forward_patch: object;

View File

@ -0,0 +1,7 @@
.edit-history-link {
margin-top: 4em;
}
.edit-history-latest-link {
float: right;
}

View File

@ -1,31 +1,88 @@
import { parse } from 'query-string';
import React, { useEffect, useState } from 'react';
import { Link, RouteComponentProps } from 'react-router-dom';
import './changes.css';
import { apiGet } from '../apiHelpers';
import { BuildingEditSummary } from '../building/edit-history/building-edit-summary';
import ErrorBox from '../components/error-box';
import InfoBox from '../components/info-box';
import { EditHistoryEntry } from '../models/edit-history-entry';
const ChangesPage = () => {
const [history, setHistory] = useState<EditHistoryEntry[]>(undefined);
interface PagingInfo {
id_for_older_query: string;
id_for_newer_query: string;
}
const recordsPerPage = 20;
const ChangesPage = (props: RouteComponentProps) => {
const { after_id, before_id } = parse(props.location.search);
const [history, setHistory] = useState<EditHistoryEntry[]>();
const [paging, setPaging] = useState<PagingInfo>();
const [error, setError] = useState();
useEffect(() => {
const fetchData = async () => {
const {history} = await apiGet(`/api/history`);
setHistory(null);
setPaging(null);
let url = `/api/history?count=${recordsPerPage}`;
if(after_id) {
url = `${url}&after_id=${after_id}`;
}
if (before_id) {
url = `${url}&before_id=${before_id}`;
}
try {
const {history, paging, error} = await apiGet(url);
if(error) {
setError(error);
} else {
setHistory(history);
setPaging(paging);
}
} catch (err) {
setError('Connection problem. Please try again later...');
}
setHistory(history);
};
fetchData();
}, []);
}, [props.location.search]);
return (
<article>
<section className="main-col">
<h1>Global edit history</h1>
{
paging?.id_for_newer_query &&
<Link className='edit-history-link' to={`?after_id=${paging.id_for_newer_query}`}>Show more recent changes</Link>
}
{
(after_id || before_id) &&
<Link className='edit-history-latest-link' to='?'>Show latest changes</Link>
}
<ul className="edit-history-list">
{(history == undefined || history.length == 0) ?
<InfoBox msg="No changes in the last week"></InfoBox> :
{
error &&
<ErrorBox msg={error}></ErrorBox>
}
{
error == undefined && history == undefined &&
<InfoBox msg="Loading history..."></InfoBox>
}
{
(history?.length === 0) &&
<InfoBox msg="No changes so far"></InfoBox>
}
{
(history != undefined && history.length > 0) &&
history.map(entry => (
<li key={`${entry.revision_id}`} className="edit-history-list-element">
<BuildingEditSummary
@ -37,6 +94,10 @@ const ChangesPage = () => {
))
}
</ul>
{
paging?.id_for_older_query &&
<Link to={`?before_id=${paging.id_for_older_query}`}>Show older changes</Link>
}
</section>
</article>
);

View File

@ -0,0 +1,34 @@
table {
table-layout: fixed;
width: 60%;
margin-left: 20%;
margin-right: 20%;
border: 1px solid black;
}
table th, td {
border: 1px solid black;
text-align: left;
padding-left: 1%;
}
table tr:nth-child(odd) {
background: #f6f8fa;
}
table tr:nth-child(1) {
background: #fff;
}
#title {
text-align: center;
padding-bottom: 1%;
}
#radiogroup {
padding: 1%;
}
input[type="radio"] {
margin: 0 2px 0 10px;
}

View File

@ -0,0 +1,149 @@
import React, { Component } from 'react';
import './leaderboard.css';
interface Leader {
number_edits: string;
username: string;
}
interface LeaderboardProps {
}
interface LeaderboardState {
leaders: Leader[];
fetching: boolean;
//We need to track the state of the radio buttons to ensure their current state is shown correctly when the view is (re)rendered
number_limit: number;
time_limit: number;
}
class LeaderboardPage extends Component<LeaderboardProps, LeaderboardState> {
constructor(props) {
super(props);
this.state = {
leaders: [],
fetching: false,
number_limit: 10,
time_limit: -1
};
this.getLeaders = this.getLeaders.bind(this);
this.renderTableData = this.renderTableData.bind(this);
this.handleChange = this.handleChange.bind(this);
}
handleChange(e) {
if(e.target.name == 'number_limit'){
this.getLeaders(e.target.value, this.state.time_limit);
this.setState({number_limit: e.target.value});
}else {
this.getLeaders(this.state.number_limit, e.target.value);
this.setState({time_limit: e.target.value});
}
}
componentDidMount() {
this.getLeaders(this.state.number_limit, this.state.time_limit);
}
componentWillUnmount() {}
getLeaders(number_limit, time_limit) {
this.setState({
fetching: true
});
fetch(
'/api/leaderboard/leaders?number_limit=' + number_limit + '&time_limit='+time_limit
).then(
(res) => res.json()
).then((data) => {
if (data && data.leaders){
this.setState({
leaders: data.leaders,
fetching: false
});
} else {
console.error(data);
this.setState({
leaders: [],
fetching: false
});
}
}).catch((err) => {
console.error(err);
this.setState({
leaders: [],
fetching: false
});
});
}
renderTableData() {
return this.state.leaders.map((u, i) => {
const username = u.username;
const number_edits = u.number_edits;
return (
<tr key={username}>
<td>{i+1}</td>
<td>{username}</td>
<td>{number_edits}</td>
</tr>
);
});
}
render() {
return(
<div>
<form id="radiogroup">
<div id="number-radiogroup" >
<p>Select number of users to be displayed: <br/>
<input type="radio" name="number_limit" value="10" onChange={this.handleChange} checked={10 == this.state.number_limit} />10
<input type="radio" name="number_limit" value="100" onChange={this.handleChange} checked={100 == this.state.number_limit} />100
</p>
</div>
<div id="time-radiogroup" >
<p>Select time period: <br/>
<input type="radio" name="time_limit" value="-1" onChange={this.handleChange} checked={-1 == this.state.time_limit} /> All time
<input type="radio" name="time_limit" value="7" onChange={this.handleChange} checked={7 == this.state.time_limit} /> Last 7 days
<input type="radio" name="time_limit" value="30" onChange={this.handleChange} checked={30 == this.state.time_limit} /> Last 30 days
</p>
</div>
</form>
<h1 id='title'>Leader Board</h1>
<table id='leaderboard'>
<tbody>
<tr>
<th>Rank</th>
<th>Username</th>
<th>Contributions</th>
</tr>
{this.renderTableData()}
</tbody>
</table>
</div>
);
}
}
export default LeaderboardPage;

View File

@ -49,6 +49,11 @@ article .color-block {
padding-left: 1em;
padding-right: 1em;
}
@media(min-width: 768px) {
.main-col {
min-width: 48em;
}
}
hr {
display: block;
height: 1px;

View File

@ -34,3 +34,27 @@ export function isNullishOrEmpty(obj: any) {
export function isEmptyArray(obj: any) {
return Array.isArray(obj) && obj.length === 0;
}
type AccessorFunction<T, V> = (obj: T) => V;
type CompareFunction<T> = (a: T, b: T) => number;
export function numAsc<T, V extends number | bigint>(accessor: AccessorFunction<T, V>): CompareFunction<T>{
return (a: T, b: T) => Number(accessor(a) - accessor(b));
}
export function numDesc<T, V extends number | bigint>(accessor: AccessorFunction<T, V>): CompareFunction<T> {
return (a: T, b: T) => Number(accessor(b) - accessor(a));
}
/**
* As of Jan 2020, bigint literals e.g. 1n can only be used with TS target esnext
* which then doesn't transpile the null conditional/coalescing operators and breaks on Node 12
* So BigInt(1) needs to be used here
* */
export function incBigInt(bigStr: string): string {
return bigStr == undefined ? bigStr : String(BigInt(bigStr) + BigInt(1));
}
export function decBigInt(bigStr: string): string {
return bigStr == undefined ? bigStr : String(BigInt(bigStr) - BigInt(1));
}

View File

@ -1,4 +1,4 @@
## Adding new building attribute fields
# Adding new building attribute fields
This document is a checklist for adding a new building attribute to the system. It's split into three sections - actions that apply when adding any field, and additional steps to add a field that will be visualised on the map.
The second section would be required when adding a new category or when changing which field should be visualised for a category.
@ -9,42 +9,42 @@ When adding a new attribute a set of seed data should be identified, the base da
- Test the API and database elements.
### Adding any attribute
## Adding any attribute
#### In database
### In database
1. Add a column to the `buildings` table in the database.
2. Add any check constraints or foreign key constraints on the column, if necessary (if the foreign key constraint is used to restrict the column to a set of values, the table with the values might need to be created from scratch)
3. If a source is being collected for field. Add a column `fieldName_source` to the `sources` table.
4. If verfication is being enabled. Add a column `bieldName_verifications` to the `verfication` table.
#### In API
### In API
1. Add field name to `BUILDING_FIELD_WHITELIST` in the building service to allow saving changes to the field
2. Add any special domain logic for processing updates to the field in the `processBuildingUpdate()` function
#### In frontend
### In frontend
1. Add the field description to the `dataFields` object in `data_fields.ts`
2. Add the data entry React component of the appropriate type (text, numeric etc) to the appropriate category view component in the `building/data-containers` folder. Link to `dataFields` for static string values (field name, description etc)
#### In data extracts
### In data extracts
1. Add the field to the select list in the COPY query in `maintenance/extract_data/export_attributes.sql`
2. Add a description of the field to the `README.txt` file
### Adding an attribute which is used to colour the map
## Adding an attribute which is used to colour the map
All steps from the previous section need to be carried out first.
#### In tileserver
1. Add a SQL query for calculating the value to be visualised to `BUILDING_LAYER_DEFINITIONS` in `app/tiles/dataDefinition.ts`
### In tileserver
1. Add a SQL query for calculating the value to be visualised to `BUILDING_LAYER_DEFINITIONS` in `app/src/tiles/dataDefinition.ts`
2. Add Mapnik rendering style in `app/map_styles/polygon.xml`
#### In frontend
### In frontend
1. Update the category to field name mapping in the `tilesetByCat` object inside the `ColouringMap` React component (`map.tsx` file)
2. Add an entry for the field to the `LEGEND_CONFIG` object in `legend.tsx` file
### Testing
## Testing
Run tests on staging to confirm;
- Database changes accepted

View File

@ -37,7 +37,7 @@ Now clone the colouring london codebase.
Now install Node. It is helpful to define some local variables.
```
NODE_VERSION=v8.11.3
NODE_VERSION=v12.14.1
DISTRO=linux-x64
wget -nc https://nodejs.org/dist/$NODE_VERSION/node-$NODE_VERSION-$DISTRO.tar.xz
sudo mkdir /usr/local/lib/node
@ -137,7 +137,7 @@ root user profile. Don't forget to exit from root at the end.
```
sudo su root
export NODEJS_HOME=/usr/local/lib/node/node-v8.11.3/bin/`
export NODEJS_HOME=/usr/local/lib/node/node-v12.14.1/bin/`
export PATH=$NODEJS_HOME:$PATH`
npm install -g npm@next`
exit

View File

@ -72,7 +72,7 @@ Now set appropriate permissions on the `colouring-london` directory
First define a couple of convenience variables:
`NODE_VERSION=v8.11.3`
`NODE_VERSION=v12.14.1`
`DISTRO=linux-x64`
@ -111,7 +111,7 @@ Now upgrade the `npm` package manager to the most recent release with global pri
`sudo su root`
`export NODEJS_HOME=/usr/local/lib/node/node-v8.11.3/bin/`
`export NODEJS_HOME=/usr/local/lib/node/node-v12.14.1/bin/`
`export PATH=$NODEJS_HOME:$PATH`
@ -244,7 +244,7 @@ Perform a global install of PM2
`sudo su root`
`export NODEJS_HOME=/usr/local/lib/node/node-v8.11.3/bin/`
`export NODEJS_HOME=/usr/local/lib/node/node-v12.14.1/bin/`
`export PATH=$NODEJS_HOME:$PATH`

View File

@ -60,9 +60,14 @@ def main(base_url, api_key, source_file, json_columns):
if building_id is None:
continue
if 'sust_dec' in line and line['sust_dec'] == '':
del line['sust_dec']
response_code, response_data = update_building(building_id, line, api_key, base_url)
if response_code != 200:
print('ERROR', building_id, response_code, response_data)
else:
print('DEBUG', building_id, response_code, response_data)
def update_building(building_id, data, api_key, base_url):
@ -82,6 +87,7 @@ def find_building(data, base_url):
if building_id is not None:
print("match_by_building_id", building_id)
return building_id
if 'toid' in data:
building_id = find_by_reference(base_url, 'toid', data['toid'])
if building_id is not None:

View File

@ -61,6 +61,11 @@ def update_building(building_id, data, api_key, base_url):
def find_building(data, base_url):
if 'building_id' in data:
building_id = data['building_id']
if building_id is not None:
print("match_by_building_id", building_id)
return building_id
if 'toid' in data:
building_id = find_by_reference(base_url, 'toid', data['toid'])
if building_id is not None:

View File

@ -0,0 +1,3 @@
-- Remove team fields, update in paralell with adding new fields
-- Award or awards (may be multiple) stored as json b object
ALTER TABLE buildings DROP COLUMN IF EXISTS team_awards;

View File

@ -0,0 +1,8 @@
--Storing as json b, create column defined as jsonb
--This contains the award name and award year
ALTER TABLE buildings
ADD COLUMN IF NOT EXISTS team_awards jsonb;
--To validate this input, the following confirms it's an valid object but not that the items in the object are validated agains those we will acccept
ALTER TABLE buildings
ADD CONSTRAINT data_is_valid CHECK (is_jsonb_valid ('{"type": "object"}', team_awards));

View File

@ -40,7 +40,7 @@ apt-get install -y \
#
# node version and platform
NODE_VERSION=v8.11.3
NODE_VERSION=v12.14.1
DISTRO=linux-x64
# download