Merge pull request #423 from mz8i/feature/tile-server-refactor

Tile server refactor, highlight colour, retina tiles
This commit is contained in:
mz8i 2019-09-30 14:29:21 +01:00 committed by GitHub
commit a83a027f14
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 887 additions and 681 deletions

View File

@ -35,6 +35,12 @@
</Style>
<Style name="highlight">
<Rule>
<Filter>[base_layer] = 'location' or [base_layer] = 'conservation_area'</Filter>
<LineSymbolizer stroke="#ff0000aa" stroke-width="4.5" />
<LineSymbolizer stroke="#ff0000ff" stroke-width="2.5" />
</Rule>
<Rule>
<ElseFilter />
<LineSymbolizer stroke="#00ffffaa" stroke-width="4.5" />
<LineSymbolizer stroke="#00ffffff" stroke-width="2.5" />
</Rule>

206
app/package-lock.json generated
View File

@ -1088,6 +1088,12 @@
"@types/geojson": "*"
}
},
"@types/mapbox__sphericalmercator": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/@types/mapbox__sphericalmercator/-/mapbox__sphericalmercator-1.1.3.tgz",
"integrity": "sha512-HjseLEStbhgZdPd6FG8TKrJxCRv2zy4NBNjQwpE3DQVE8yQMJK1p3Xq25cM9FAVe4cGBaGbv4Un4kBJTvs4H9g==",
"dev": true
},
"@types/mime": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.1.tgz",
@ -1193,6 +1199,15 @@
"@types/mime": "*"
}
},
"@types/sharp": {
"version": "0.22.2",
"resolved": "https://registry.npmjs.org/@types/sharp/-/sharp-0.22.2.tgz",
"integrity": "sha512-oH49f42h3nf/qys0weYsaTGiMv67wPB769ynCoPfBAVwjjxFF3QtIPEe3MfhwyNjQAhQhTEfnmMKvVZfcFkhIw==",
"dev": true,
"requires": {
"@types/node": "*"
}
},
"@types/webpack-env": {
"version": "1.14.0",
"resolved": "https://registry.npmjs.org/@types/webpack-env/-/webpack-env-1.14.0.tgz",
@ -2577,21 +2592,24 @@
"integrity": "sha512-Un7MIEDdUC5gNpcGDV97op1Ywk748MpHcFTHoYs6qnj1Z3j7I53VG3nwZhKzoBZmbdRNnb6WRdFlwl7tSDuZGw==",
"dev": true
},
"bindings": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/bindings/-/bindings-1.4.0.tgz",
"integrity": "sha512-7znEVX22Djn+nYjxCWKDne0RRloa9XfYa84yk3s+HkE3LpDYZmhArYr9O9huBoHY3/oXispx5LorIX7Sl2CgSQ==",
"requires": {
"file-uri-to-path": "1.0.0"
}
},
"bl": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/bl/-/bl-1.2.2.tgz",
"integrity": "sha512-e8tQYnZodmebYDWGH7KMRvtzKXaJHx3BbilrgZCfvyLUYdKpK1t5PSPmpkny/SgiTSCnjfLW7v5rlONXVFkQEA==",
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/bl/-/bl-3.0.0.tgz",
"integrity": "sha512-EUAyP5UHU5hxF8BPT0LKW8gjYLhq1DQIcneOX/pL/m2Alo+OYDQAJlHq+yseMP50Os2nHXOSic6Ss3vSQeyf4A==",
"requires": {
"readable-stream": "^2.3.5",
"safe-buffer": "^5.1.1"
"readable-stream": "^3.0.1"
},
"dependencies": {
"readable-stream": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.4.0.tgz",
"integrity": "sha512-jItXPLmrSR8jmTRmRWJXCnGJsfy85mB3Wd/uINMXA65yrnFo0cPClFIUWzo2najVNSl+mx7/4W8ttlLWJe99pQ==",
"requires": {
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
}
}
}
},
"bluebird": {
@ -2866,25 +2884,6 @@
"isarray": "^1.0.0"
}
},
"buffer-alloc": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz",
"integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==",
"requires": {
"buffer-alloc-unsafe": "^1.1.0",
"buffer-fill": "^1.0.0"
}
},
"buffer-alloc-unsafe": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz",
"integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg=="
},
"buffer-fill": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz",
"integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw="
},
"buffer-from": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
@ -3213,9 +3212,9 @@
}
},
"chownr": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.0.1.tgz",
"integrity": "sha1-4qdQQqlVGQi+vSW4Uj1fl2nXkYE="
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.2.tgz",
"integrity": "sha512-GkfeAQh+QNy3wquu9oIZr6SS5x7wGdSgNQvD10X3r+AZr1Oys22HW8kAmDMvNg2+Dm0TeGaEuO8gFwdBXxwO8A=="
},
"chrome-trace-event": {
"version": "1.0.2",
@ -3350,7 +3349,6 @@
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/color/-/color-3.1.1.tgz",
"integrity": "sha512-PvUltIXRjehRKPSy89VnDWFKY58xyhTLyxIg21vwQBI6qLwZNPmC8k3C1uytIgFKEpOIzN4y32iPm8231zFHIg==",
"dev": true,
"requires": {
"color-convert": "^1.9.1",
"color-string": "^1.5.2"
@ -3373,7 +3371,6 @@
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/color-string/-/color-string-1.5.3.tgz",
"integrity": "sha512-dC2C5qeWoYkxki5UAXapdjqO672AM4vZuPGRQfO8b5HKuKGBbKWpITyDYN7TOFKvRW7kOgAn3746clDBMDJyQw==",
"dev": true,
"requires": {
"color-name": "^1.0.0",
"simple-swizzle": "^0.2.2"
@ -5800,11 +5797,6 @@
"schema-utils": "^1.0.0"
}
},
"file-uri-to-path": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
"integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw=="
},
"filename-regex": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/filename-regex/-/filename-regex-2.0.1.tgz",
@ -9652,7 +9644,9 @@
"nan": {
"version": "2.12.1",
"resolved": "https://registry.npmjs.org/nan/-/nan-2.12.1.tgz",
"integrity": "sha512-JY7V6lRkStKcKTvHO5NVSQRv+RV+FIL5pvDoLiAtSL9pKlC5x9PKQcZDsq7m4FO4d57mkhC6Z+QhAh3Jdk5JFw=="
"integrity": "sha512-JY7V6lRkStKcKTvHO5NVSQRv+RV+FIL5pvDoLiAtSL9pKlC5x9PKQcZDsq7m4FO4d57mkhC6Z+QhAh3Jdk5JFw==",
"dev": true,
"optional": true
},
"nanomatch": {
"version": "1.2.13",
@ -9733,9 +9727,9 @@
"dev": true
},
"node-abi": {
"version": "2.7.1",
"resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.7.1.tgz",
"integrity": "sha512-OV8Bq1OrPh6z+Y4dqwo05HqrRL9YNF7QVMRfq1/pguwKLG+q9UB/Lk0x5qXjO23JjJg+/jqCHSTaG1P3tfKfuw==",
"version": "2.11.0",
"resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.11.0.tgz",
"integrity": "sha512-kuy/aEg75u40v378WRllQ4ZexaXJiCvB68D2scDXclp/I4cRq6togpbOoKhmN07tns9Zldu51NNERo0wehfX9g==",
"requires": {
"semver": "^5.4.1"
}
@ -13201,9 +13195,9 @@
}
},
"prebuild-install": {
"version": "5.2.4",
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-5.2.4.tgz",
"integrity": "sha512-CG3JnpTZXdmr92GW4zbcba4jkDha6uHraJ7hW4Fn8j0mExxwOKK20hqho8ZuBDCKYCHYIkFM1P2jhtG+KpP4fg==",
"version": "5.3.2",
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-5.3.2.tgz",
"integrity": "sha512-INDfXzTPnhT+WYQemqnAXlP7SvfiFMopMozSgXCZ+RDLb279gKfIuLk4o7PgEawLp3WrMgIYGBpkxpraROHsSA==",
"requires": {
"detect-libc": "^1.0.3",
"expand-template": "^2.0.3",
@ -13214,11 +13208,10 @@
"node-abi": "^2.7.0",
"noop-logger": "^0.1.1",
"npmlog": "^4.0.1",
"os-homedir": "^1.0.1",
"pump": "^2.0.1",
"pump": "^3.0.0",
"rc": "^1.2.7",
"simple-get": "^2.7.0",
"tar-fs": "^1.13.0",
"simple-get": "^3.0.3",
"tar-fs": "^2.0.0",
"tunnel-agent": "^0.6.0",
"which-pm-runs": "^1.0.0"
},
@ -13228,14 +13221,13 @@
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
"integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ="
},
"simple-get": {
"version": "2.8.1",
"resolved": "https://registry.npmjs.org/simple-get/-/simple-get-2.8.1.tgz",
"integrity": "sha512-lSSHRSw3mQNUGPAYRqo7xy9dhKmxFXIjLjp4KHpf99GEH2VH7C3AM+Qfx6du6jhfUi6Vm7XnbEVEf7Wb6N8jRw==",
"pump": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
"integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
"requires": {
"decompress-response": "^3.3.0",
"once": "^1.3.1",
"simple-concat": "^1.0.0"
"end-of-stream": "^1.1.0",
"once": "^1.3.1"
}
}
}
@ -13378,6 +13370,7 @@
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz",
"integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==",
"dev": true,
"requires": {
"end-of-stream": "^1.1.0",
"once": "^1.3.1"
@ -15130,45 +15123,31 @@
}
},
"sharp": {
"version": "0.21.3",
"resolved": "https://registry.npmjs.org/sharp/-/sharp-0.21.3.tgz",
"integrity": "sha512-5qZk8r+YgfyztLEKkNez20Wynq/Uh1oNyP5T/3gTYwt2lBYGs9iDs5m0yVsZEPm8eVBbAJhS08J1wp/g+Ai1Qw==",
"version": "0.22.1",
"resolved": "https://registry.npmjs.org/sharp/-/sharp-0.22.1.tgz",
"integrity": "sha512-lXzSk/FL5b/MpWrT1pQZneKe25stVjEbl6uhhJcTULm7PhmJgKKRbTDM/vtjyUuC/RLqL2PRyC4rpKwbv3soEw==",
"requires": {
"bindings": "^1.3.1",
"color": "^3.1.0",
"color": "^3.1.1",
"detect-libc": "^1.0.3",
"fs-copy-file-sync": "^1.1.1",
"nan": "^2.12.1",
"nan": "^2.13.2",
"npmlog": "^4.1.2",
"prebuild-install": "^5.2.2",
"semver": "^5.6.0",
"prebuild-install": "^5.3.0",
"semver": "^6.0.0",
"simple-get": "^3.0.3",
"tar": "^4.4.8",
"tunnel-agent": "^0.6.0"
},
"dependencies": {
"color": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/color/-/color-3.1.0.tgz",
"integrity": "sha512-CwyopLkuRYO5ei2EpzpIh6LqJMt6Mt+jZhO5VI5f/wJLZriXQE32/SSqzmrh+QB+AZT81Cj8yv+7zwToW8ahZg==",
"requires": {
"color-convert": "^1.9.1",
"color-string": "^1.5.2"
}
},
"color-string": {
"version": "1.5.3",
"resolved": "https://registry.npmjs.org/color-string/-/color-string-1.5.3.tgz",
"integrity": "sha512-dC2C5qeWoYkxki5UAXapdjqO672AM4vZuPGRQfO8b5HKuKGBbKWpITyDYN7TOFKvRW7kOgAn3746clDBMDJyQw==",
"requires": {
"color-name": "^1.0.0",
"simple-swizzle": "^0.2.2"
}
"nan": {
"version": "2.14.0",
"resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz",
"integrity": "sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg=="
},
"semver": {
"version": "5.6.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz",
"integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg=="
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
"integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw=="
}
}
},
@ -16010,20 +15989,20 @@
}
},
"tar-fs": {
"version": "1.16.3",
"resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-1.16.3.tgz",
"integrity": "sha512-NvCeXpYx7OsmOh8zIOP/ebG55zZmxLE0etfWRbWok+q2Qo8x/vOR/IJT1taADXPe+jsiu9axDb3X4B+iIgNlKw==",
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.0.tgz",
"integrity": "sha512-vaY0obB6Om/fso8a8vakQBzwholQ7v5+uy+tF3Ozvxv1KNezmVQAiWtcNmMHFSFPqL3dJA8ha6gdtFbfX9mcxA==",
"requires": {
"chownr": "^1.0.1",
"chownr": "^1.1.1",
"mkdirp": "^0.5.1",
"pump": "^1.0.0",
"tar-stream": "^1.1.2"
"pump": "^3.0.0",
"tar-stream": "^2.0.0"
},
"dependencies": {
"pump": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/pump/-/pump-1.0.3.tgz",
"integrity": "sha512-8k0JupWme55+9tCVE+FS5ULT3K6AbgqrGa58lTT49RpyfwwcGedHqaC5LlQNdEAumn/wFsu6aPwkuPMioy8kqw==",
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
"integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
"requires": {
"end-of-stream": "^1.1.0",
"once": "^1.3.1"
@ -16032,17 +16011,27 @@
}
},
"tar-stream": {
"version": "1.6.2",
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz",
"integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==",
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.0.tgz",
"integrity": "sha512-+DAn4Nb4+gz6WZigRzKEZl1QuJVOLtAwwF+WUxy1fJ6X63CaGaUAxJRD2KEn1OMfcbCjySTYpNC6WmfQoIEOdw==",
"requires": {
"bl": "^1.0.0",
"buffer-alloc": "^1.2.0",
"end-of-stream": "^1.0.0",
"bl": "^3.0.0",
"end-of-stream": "^1.4.1",
"fs-constants": "^1.0.0",
"readable-stream": "^2.3.0",
"to-buffer": "^1.1.1",
"xtend": "^4.0.0"
"inherits": "^2.0.3",
"readable-stream": "^3.1.1"
},
"dependencies": {
"readable-stream": {
"version": "3.4.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.4.0.tgz",
"integrity": "sha512-jItXPLmrSR8jmTRmRWJXCnGJsfy85mB3Wd/uINMXA65yrnFo0cPClFIUWzo2najVNSl+mx7/4W8ttlLWJe99pQ==",
"requires": {
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
}
}
}
},
"terser": {
@ -16182,11 +16171,6 @@
"integrity": "sha1-fSKbH8xjfkZsoIEYCDanqr/4P0M=",
"dev": true
},
"to-buffer": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz",
"integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg=="
},
"to-fast-properties": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz",

View File

@ -34,12 +34,13 @@
"react-leaflet-universal": "^1.2.0",
"react-router-dom": "^5.0.1",
"serialize-javascript": "^1.7.0",
"sharp": "^0.21.3"
"sharp": "^0.22.1"
},
"devDependencies": {
"@types/express": "^4.17.0",
"@types/express-session": "^1.15.13",
"@types/jest": "^24.0.17",
"@types/mapbox__sphericalmercator": "^1.1.3",
"@types/node": "^8.10.52",
"@types/nodemailer": "^6.2.1",
"@types/prop-types": "^15.7.1",
@ -47,6 +48,7 @@
"@types/react-dom": "^16.8.5",
"@types/react-leaflet": "^2.4.0",
"@types/react-router-dom": "^4.3.4",
"@types/sharp": "^0.22.2",
"@types/webpack-env": "^1.14.0",
"babel-eslint": "^10.0.2",
"eslint": "^5.16.0",

View File

@ -0,0 +1,17 @@
import { Request, Response, NextFunction } from 'express';
/**
* A wrapper for controller functions that return a Promise, enabling them to be used with Express
* Without this wrapper, Promise rejections caused by an error in the controller will not be passed properly
* to subsequent middleware layers.
* @param fn the async controller function to be wrapped
* @returns controller function which handles async errors correctly
*/
function asyncController(fn: (req: Request, res: Response, next: NextFunction) => Promise<any>) {
return (req: Request, res: Response, next: NextFunction) => {
Promise.resolve(fn(req, res, next))
.catch(next);
};
}
export default asyncController;

View File

@ -3,7 +3,8 @@
*
*/
import db from '../../db';
import { removeAllAtBbox } from '../../tiles/cache';
import { tileCache } from '../../tiles/rendererDefinition';
import { BoundingBox } from '../../tiles/types';
// data type note: PostgreSQL bigint (64-bit) is handled as string in JavaScript, because of
// JavaScript numerics are 64-bit double, giving only partial coverage.
@ -314,8 +315,8 @@ function privateQueryBuildingBBOX(buildingId){
function expireBuildingTileCache(buildingId) {
privateQueryBuildingBBOX(buildingId).then((bbox) => {
const buildingBbox = [bbox.xmax, bbox.ymax, bbox.xmin, bbox.ymin]
removeAllAtBbox(buildingBbox);
const buildingBbox: BoundingBox = [bbox.xmax, bbox.ymax, bbox.xmin, bbox.ymin];
tileCache.removeAllAtBbox(buildingBbox);
})
}

View File

@ -109,7 +109,7 @@ class ColouringMap extends Component<ColouringMapProps, ColouringMapState> { //
attribution={attribution}
/>;
const buildingsBaseUrl = `/tiles/base_${this.state.theme}/{z}/{x}/{y}.png`;
const buildingsBaseUrl = `/tiles/base_${this.state.theme}/{z}/{x}/{y}{r}.png`;
const buildingBaseLayer = <TileLayer url={buildingsBaseUrl} minZoom={14} />;
// colour-data tiles
@ -127,7 +127,7 @@ class ColouringMap extends Component<ColouringMapProps, ColouringMapState> { //
const dataLayer = tileset != undefined ?
<TileLayer
key={tileset}
url={`/tiles/${tileset}/{z}/{x}/{y}.png?rev=${rev}`}
url={`/tiles/${tileset}/{z}/{x}/{y}{r}.png?rev=${rev}`}
minZoom={9}
/>
: null;
@ -136,7 +136,7 @@ class ColouringMap extends Component<ColouringMapProps, ColouringMapState> { //
const highlightLayer = this.props.building != undefined ?
<TileLayer
key={this.props.building.building_id}
url={`/tiles/highlight/{z}/{x}/{y}.png?highlight=${this.props.building.geometry_id}`}
url={`/tiles/highlight/{z}/{x}/{y}{r}.png?highlight=${this.props.building.geometry_id}&base=${tileset}`}
minZoom={14}
zIndex={100}
/>
@ -155,6 +155,7 @@ class ColouringMap extends Component<ColouringMapProps, ColouringMapState> { //
zoomControl={false}
attributionControl={false}
onClick={this.handleClick}
detectRetina={true}
>
{ baseLayer }
{ buildingBaseLayer }

View File

@ -1,171 +0,0 @@
/**
* Cache tiles (PNG images generated from database)
*
* Frequency of change:
* - base layer tiles change rarely - on changes to underlying geometry table
* - visualisation layer tiles change frequently - with almost any edit to the buildings table
*
* Cost of generation and storage:
* - low zoom tiles are more expensive to render, containing more features from the database
* - high zoom tiles are cheaper to rerender, and changes are more visible
* - there are many more high zoom tiles than low: 4 tiles at zoom level n+1 for each tile
* at zoom level n
*
*/
// Using node-fs package to patch fs
// for node >10 we could drop this in favour of fs.mkdir (which has recursive option)
// and then use stdlib `import fs from 'fs';`
import fs from 'node-fs';
import { getXYZ } from './tile';
// Use an environment variable to configure the cache location, somewhere we can read/write to.
const CACHE_PATH = process.env.TILECACHE_PATH
/**
* Get a tile from the cache
*
* @param {String} tileset
* @param {number} z zoom level
* @param {number} x
* @param {number} y
*/
function get(tileset, z, x, y) {
if (!shouldTryCache(tileset, z)) {
return Promise.reject(`Skip cache get ${tileset}/${z}/${x}/${y}`);
}
const location = cacheLocation(tileset, z, x, y);
return new Promise((resolve, reject) => {
fs.readFile(location.fname, (err, data) => {
if (err) {
reject(err)
} else {
resolve(data)
}
})
});
}
/**
* Put a tile in the cache
*
* @param {Buffer} im image data
* @param {String} tileset
* @param {number} z zoom level
* @param {number} x
* @param {number} y
*/
function put(im, tileset, z, x, y) {
if (!shouldTryCache(tileset, z)) {
return Promise.reject(`Skip cache put ${tileset}/${z}/${x}/${y}`);
}
const location = cacheLocation(tileset, z, x, y);
return new Promise((resolve, reject) => {
fs.writeFile(location.fname, im, 'binary', (err) => {
if (err && err.code === 'ENOENT') {
// recursively create tile directory if it didn't previously exist
fs.mkdir(location.dir, 0o755, true, (err) => {
if (err) {
reject(err);
} else {
// then write the file
fs.writeFile(location.fname, im, 'binary', (err) => {
(err)? reject(err): resolve()
});
}
});
} else {
(err)? reject(err): resolve()
}
});
})
}
/**
* Remove a single cached tile
*
* @param {String} tileset
* @param {number} z zoom level
* @param {number} x
* @param {number} y
*/
function remove(tileset, z, x, y) {
const location = cacheLocation(tileset, z, x, y)
return new Promise(resolve => {
fs.unlink(location.fname, (err) => {
if(err){
// pass
} else {
console.log('Expire cache', tileset, z, x, y)
}
resolve()
})
})
}
/**
* Remove all cached data-visualising tiles which intersect a bbox
* - initially called directly after edits; may be better on a worker process?
*
* @param {String} tileset
* @param {Array} bbox [w, s, e, n] in EPSG:3857 coordinates
*/
function removeAllAtBbox(bbox) {
// magic numbers for min/max zoom
const minZoom = 9;
const maxZoom = 18;
// magic list of tilesets - see tileserver, other cache rules
const tilesets = ['date_year', 'size_storeys', 'location', 'likes', 'conservation_area'];
let tileBounds;
const removePromises = [];
for (let ti = 0; ti < tilesets.length; ti++) {
const tileset = tilesets[ti];
for (let z = minZoom; z <= maxZoom; z++) {
tileBounds = getXYZ(bbox, z)
for (let x = tileBounds.minX; x <= tileBounds.maxX; x++){
for (let y = tileBounds.minY; y <= tileBounds.maxY; y++){
removePromises.push(remove(tileset, z, x, y))
}
}
}
}
Promise.all(removePromises)
}
/**
* Cache location for a tile
*
* @param {String} tileset
* @param {number} z zoom level
* @param {number} x
* @param {number} y
* @returns {object} { dir: <directory>, fname: <full filepath> }
*/
function cacheLocation(tileset, z, x, y) {
const dir = `${CACHE_PATH}/${tileset}/${z}/${x}`
const fname = `${dir}/${y}.png`
return {dir, fname}
}
/**
* Check rules for caching tiles
*
* @param {String} tileset
* @param {number} z zoom level
* @returns {boolean} whether to use the cache (or not)
*/
function shouldTryCache(tileset, z) {
if (tileset === 'date_year') {
// cache high zoom because of front page hits
return z <= 16
}
if (tileset === 'base_light' || tileset === 'base_night') {
// cache for higher zoom levels (unlikely to change)
return z <= 17
}
// else cache for lower zoom levels (change slowly)
return z <= 13
}
export { get, put, remove, removeAllAtBbox };

View File

@ -0,0 +1,135 @@
import { strictParseInt } from "../parse";
import { DataConfig } from "./renderers/datasourceRenderer";
const BUILDING_LAYER_DEFINITIONS = {
base_light: `(
SELECT
b.location_number as location_number,
g.geometry_geom
FROM
geometries as g,
buildings as b
WHERE
g.geometry_id = b.geometry_id
) as outline`,
base_night: `(
SELECT
b.location_number as location_number,
g.geometry_geom
FROM
geometries as g,
buildings as b
WHERE
g.geometry_id = b.geometry_id
) as outline`,
date_year: `(
SELECT
b.date_year as date_year,
g.geometry_geom
FROM
geometries as g,
buildings as b
WHERE
g.geometry_id = b.geometry_id
) as outline`,
size_storeys: `(
SELECT
(
coalesce(b.size_storeys_attic, 0) +
coalesce(b.size_storeys_core, 0)
) as size_storeys,
g.geometry_geom
FROM
geometries as g,
buildings as b
WHERE
g.geometry_id = b.geometry_id
) as outline`,
location: `(
SELECT
(
case when b.location_name is null then 0 else 1 end +
case when b.location_number is null then 0 else 1 end +
case when b.location_street is null then 0 else 1 end +
case when b.location_line_two is null then 0 else 1 end +
case when b.location_town is null then 0 else 1 end +
case when b.location_postcode is null then 0 else 1 end +
case when b.location_latitude is null then 0 else 1 end +
case when b.location_longitude is null then 0 else 1 end +
case when b.ref_toid is null then 0 else 1 end +
case when b.ref_osm_id is null then 0 else 1 end
) as location_info_count,
g.geometry_geom
FROM
geometries as g,
buildings as b
WHERE
g.geometry_id = b.geometry_id
) as location`,
likes: `(
SELECT
g.geometry_geom,
b.likes_total as likes
FROM
geometries as g,
buildings as b
WHERE
g.geometry_id = b.geometry_id
AND b.likes_total > 0
) as location`,
conservation_area: `(
SELECT
g.geometry_geom
FROM
geometries as g,
buildings as b
WHERE
g.geometry_id = b.geometry_id
AND b.planning_in_conservation_area = true
) as conservation_area`
};
const GEOMETRY_FIELD = 'geometry_geom';
function getBuildingsDataConfig(tileset: string, dataParams: any): DataConfig {
const table = BUILDING_LAYER_DEFINITIONS[tileset];
if(table == undefined) {
throw new Error('Invalid tileset requested');
}
return {
geometry_field: GEOMETRY_FIELD,
table: table
};
}
function getHighlightDataConfig(tileset: string, dataParams: any): DataConfig {
let { highlight, base } = dataParams;
highlight = strictParseInt(highlight);
base = base || 'default';
if(isNaN(highlight) || base.match(/^\w+$/) == undefined) {
throw new Error('Bad parameters for highlight layer');
}
return {
geometry_field: GEOMETRY_FIELD,
table: `(
SELECT
g.geometry_geom,
'${base}' as base_layer
FROM
geometries as g
WHERE
g.geometry_id = ${highlight}
) as highlight`
};
}
export {
BUILDING_LAYER_DEFINITIONS,
getBuildingsDataConfig,
getHighlightDataConfig
};

View File

@ -0,0 +1,78 @@
import { TileCache } from "./tileCache";
import { BoundingBox, TileParams } from "./types";
import { StitchRenderer } from "./renderers/stitchRenderer";
import { CachedRenderer } from "./renderers/cachedRenderer";
import { BranchingRenderer } from "./renderers/branchingRenderer";
import { WindowedRenderer } from "./renderers/windowedRenderer";
import { BlankRenderer } from "./renderers/blankRenderer";
import { DatasourceRenderer } from "./renderers/datasourceRenderer";
import { getBuildingsDataConfig, getHighlightDataConfig, BUILDING_LAYER_DEFINITIONS } from "./dataDefinition";
/**
* A list of all tilesets handled by the tile server
*/
const allTilesets = ['highlight', ...Object.keys(BUILDING_LAYER_DEFINITIONS)];
const buildingDataRenderer = new DatasourceRenderer(getBuildingsDataConfig);
const stitchRenderer = new StitchRenderer(undefined); // depends recurrently on cache, so parameter will be set later
/**
* Zoom level when we switch from rendering direct from database to instead composing tiles
* from the zoom level below - gets similar effect, with much lower load on Postgres
*/
const STITCH_THRESHOLD = 12;
const renderOrStitchRenderer = new BranchingRenderer(
({ z }) => z <= STITCH_THRESHOLD,
stitchRenderer, // refer to the prepared stitch renderer
buildingDataRenderer
);
const tileCache = new TileCache(
process.env.TILECACHE_PATH,
{
tilesets: ['date_year', 'size_storeys', 'location', 'likes', 'conservation_area'],
minZoom: 9,
maxZoom: 18,
scales: [1, 2]
},
({ tileset, z }: TileParams) => (tileset === 'date_year' && z <= 16) ||
((tileset === 'base_light' || tileset === 'base_night') && z <= 17) ||
z <= 13
);
const cachedRenderer = new CachedRenderer(
tileCache,
renderOrStitchRenderer
);
// set up stitch renderer to use the data renderer with caching
stitchRenderer.tileRenderer = cachedRenderer;
const highlightRenderer = new DatasourceRenderer(getHighlightDataConfig);
const highlightOrBuildingRenderer = new BranchingRenderer(
({ tileset }) => tileset === 'highlight',
highlightRenderer,
cachedRenderer
);
const blankRenderer = new BlankRenderer();
/**
* Hard-code extent so we can short-circuit rendering and return empty/transparent tiles outside the area of interest
* bbox in CRS epsg:3857 in form: [w, s, e, n]
*/
const EXTENT_BBOX: BoundingBox = [-61149.622628, 6667754.851372, 28128.826409, 6744803.375884];
const mainRenderer = new WindowedRenderer(
EXTENT_BBOX,
highlightOrBuildingRenderer,
blankRenderer
);
export {
allTilesets,
mainRenderer,
tileCache
};

View File

@ -0,0 +1,21 @@
import { Image } from "mapnik";
import sharp from 'sharp';
import { TileParams, TileRenderer } from "../types";
class BlankRenderer implements TileRenderer {
getTile(tileParams: TileParams): Promise<Image> {
return sharp({
create: {
width: 1,
height: 1,
channels: 4,
background: { r: 0, g: 0, b: 0, alpha: 0 }
}
}).png().toBuffer();
}
}
export {
BlankRenderer
};

View File

@ -0,0 +1,23 @@
import { Image } from "mapnik";
import { TileParams, TileRenderer } from "../types";
class BranchingRenderer {
constructor(
public branchTestFn: (tileParams: TileParams) => boolean,
public trueResultTileRenderer: TileRenderer,
public falseResultTileRenderer: TileRenderer
) {}
getTile(tileParams: TileParams, dataParams: any): Promise<Image> {
if(this.branchTestFn(tileParams)) {
return this.trueResultTileRenderer.getTile(tileParams, dataParams);
} else {
return this.falseResultTileRenderer.getTile(tileParams, dataParams);
}
}
}
export {
BranchingRenderer
};

View File

@ -0,0 +1,32 @@
import { Image } from "mapnik";
import { TileParams, TileRenderer } from "../types";
import { TileCache } from "../tileCache";
import { formatParams } from "../util";
class CachedRenderer implements TileRenderer {
constructor(
/** Cache to use for tiles */
public tileCache: TileCache,
/** Renderer to use when tile hasn't been cached yet */
public tileRenderer: TileRenderer
) {}
async getTile(tileParams: TileParams, dataParams: any): Promise<Image> {
try {
const tile = await this.tileCache.get(tileParams);
return tile;
} catch(err) {
const im = await this.tileRenderer.getTile(tileParams, dataParams);
try {
await this.tileCache.put(im, tileParams);
} catch (err) {}
return im;
}
}
}
export {
CachedRenderer
};

View File

@ -0,0 +1,105 @@
import path from 'path';
import mapnik from "mapnik";
import { TileParams, TileRenderer } from "../types";
import { getBbox, TILE_SIZE } from "../util";
import { promisify } from "util";
interface DataConfig {
table: string;
geometry_field: string;
}
const TILE_BUFFER_SIZE = 64;
const PROJ4_STRING = '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over';
// connection details from environment variables
const DATASOURCE_CONFIG = {
'host': process.env.PGHOST,
'dbname': process.env.PGDATABASE,
'user': process.env.PGUSER,
'password': process.env.PGPASSWORD,
'port': process.env.PGPORT,
'extent': '-20005048.4188,-9039211.13765,19907487.2779,17096598.5401',
'srid': 3857,
'type': 'postgis'
};
// register datasource adapters for mapnik database connection
if (mapnik.register_default_input_plugins) {
mapnik.register_default_input_plugins();
}
// register fonts for text rendering
mapnik.register_default_fonts();
class DatasourceRenderer implements TileRenderer {
constructor(private getTableDefinitionFn: (tileset: string, dataParams: any) => DataConfig) {}
async getTile({tileset, z, x, y, scale}: TileParams, dataParams: any): Promise<mapnik.Image> {
const bbox = getBbox(z, x, y);
const tileSize = TILE_SIZE * scale;
let map = new mapnik.Map(tileSize, tileSize, PROJ4_STRING);
map.bufferSize = TILE_BUFFER_SIZE;
const layer = new mapnik.Layer('tile', PROJ4_STRING);
const dataSourceConfig = this.getTableDefinitionFn(tileset, dataParams);
const conf = Object.assign(dataSourceConfig, DATASOURCE_CONFIG);
const postgis = new mapnik.Datasource(conf);
layer.datasource = postgis;
layer.styles = [tileset];
const stylePath = path.join(__dirname, '..', 'map_styles', 'polygon.xml');
map = await promisify(map.load.bind(map))(stylePath, {strict: true});
map.add_layer(layer);
const im = new mapnik.Image(map.width, map.height);
map.extent = bbox;
const rendered = await promisify(map.render.bind(map))(im, {});
return await promisify(rendered.encode.bind(rendered))('png');
}
}
function promiseHandler(resolve, reject) {
return function(err, result) {
if(err) reject(err);
else resolve(result);
}
}
/**
* Utility function which promisifies a method of an object and binds it to the object
* This makes it easier to use callback-based object methods in a promise-based way
* @param obj Object containing the target method
* @param methodName Method name to promisify and return
*/
function promisifyMethod<T, F>(obj: T, methodName: keyof T);
/**
* @param methodGetter accessor function to get the method from the object
*/
function promisifyMethod<T, S>(obj: T, methodGetter: (o: T) => S);
function promisifyMethod<T, S>(obj: T, paramTwo: keyof T | ((o: T) => S)) {
let method;
if (typeof paramTwo === 'string') {
method = obj[paramTwo];
} else if (typeof paramTwo === 'function') {
method = paramTwo(obj);
}
if (typeof method === 'function') {
return promisify(method.bind(obj));
} else {
throw new Error(`Cannot promisify non-function property '${paramTwo}'`);
}
}
export {
DatasourceRenderer,
DataConfig
};

View File

@ -0,0 +1,67 @@
import sharp from 'sharp';
import { Image } from 'mapnik';
import { TileParams, TileRenderer } from "../types";
import { getBbox, getXYZ, TILE_SIZE, formatParams } from "../util";
class StitchRenderer implements TileRenderer {
constructor(
/** Renderer to use when retrieving tiles to be stitched together */
public tileRenderer: TileRenderer
) {}
getTile(tileParams: TileParams, dataParams: any): Promise<Image> {
console.log(`Stitching tile ${formatParams(tileParams)}`);
return this.stitchTile(tileParams, dataParams, this.tileRenderer);
}
private async stitchTile({ tileset, z, x, y, scale }: TileParams, dataParams: any, tileRenderer: TileRenderer) {
const bbox = getBbox(z, x, y);
const nextZ = z + 1;
const nextXY = getXYZ(bbox, nextZ);
const tileSize = TILE_SIZE * scale;
const [topLeft, topRight, bottomLeft, bottomRight] = await Promise.all([
[nextXY.minX, nextXY.minY],
[nextXY.maxX, nextXY.minY],
[nextXY.minX, nextXY.maxY],
[nextXY.maxX, nextXY.maxY]
].map(([x, y]) => tileRenderer.getTile({ tileset, z: nextZ, x, y, scale }, dataParams)));
// not possible to chain overlays in a single pipeline, but there may still be a better
// way to create image buffer here (four tiles resize to one at the next zoom level)
// instead of repeatedly creating `sharp` objects, to png, to buffer...
return sharp({
create: {
width: tileSize * 2,
height: tileSize * 2,
channels: 4,
background: { r: 0, g: 0, b: 0, alpha: 0 }
}
}).overlayWith(
topLeft, { gravity: sharp.gravity.northwest }
).png().toBuffer().then((buf) => {
return sharp(buf).overlayWith(
topRight, { gravity: sharp.gravity.northeast }
).png().toBuffer()
}).then((buf) => {
return sharp(buf).overlayWith(
bottomLeft, { gravity: sharp.gravity.southwest }
).png().toBuffer()
}).then((buf) => {
return sharp(buf).overlayWith(
bottomRight, { gravity: sharp.gravity.southeast }
).png().toBuffer()
}).then((buf) => {
return sharp(buf
).resize(tileSize, tileSize, { fit: 'inside' }
).png().toBuffer()
})
}
}
export {
StitchRenderer
};

View File

@ -0,0 +1,34 @@
import { Image } from "mapnik";
import { BoundingBox, TileParams, TileRenderer } from "../types";
import { getXYZ } from "../util";
class WindowedRenderer implements TileRenderer {
constructor(
/** Bounding box defining the renderer window */
public bbox: BoundingBox,
/** Renderer to use for tile requests inside window */
public insideWindowRenderer: TileRenderer,
/** Renderer to use for tile requests outside window */
public outsideWindowRenderer: TileRenderer
) {}
getTile(tileParams: TileParams, dataParams: any): Promise<Image> {
if(this.isOutsideExtent(tileParams)) {
return this.outsideWindowRenderer.getTile(tileParams, dataParams);
} else {
return this.insideWindowRenderer.getTile(tileParams, dataParams);
}
}
private isOutsideExtent({x, y, z}: TileParams) {
const xy = getXYZ(this.bbox, z);
return xy.minY > y || xy.maxY < y || xy.minX > x || xy.maxX < x;
}
}
export {
WindowedRenderer
};

View File

@ -1,201 +0,0 @@
/**
* Render tiles
*
* Use mapnik to render map tiles from the database
*
* Styles have two sources of truth for colour ranges (could generate from single source?)
* - XML style definitions in app/map_styles/polygon.xml
* - front-end legend in app/src/frontend/legend.js
*
* Data is provided by the queries in MAP_STYLE_TABLE_DEFINITIONS below.
*
*/
import path from 'path';
import mapnik from 'mapnik';
import SphericalMercator from '@mapbox/sphericalmercator';
// connection details from environment variables
const DATASOURCE_CONFIG = {
'host': process.env.PGHOST,
'dbname': process.env.PGDATABASE,
'user': process.env.PGUSER,
'password': process.env.PGPASSWORD,
'port': process.env.PGPORT,
'geometry_field': 'geometry_geom',
'extent': '-20005048.4188,-9039211.13765,19907487.2779,17096598.5401',
'srid': 3857,
'type': 'postgis'
}
const TILE_SIZE = 256
const TILE_BUFFER_SIZE = 64
const PROJ4_STRING = '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over';
// Mapnik uses table definitions to query geometries and attributes from PostGIS.
// The queries here are eventually used as subqueries when Mapnik fetches data to render a
// tile - so given a table definition like:
// (SELECT geometry_geom FROM geometries) as def
// Mapnik will wrap it in a bbox query and PostGIS will eventually see something like:
// SELECT AsBinary("geometry") AS geom from
// (SELECT geometry_geom FROM geometries) as def
// WHERE "geometry" && SetSRID('BOX3D(0,1,2,3)'::box3d, 3857)
// see docs: https://github.com/mapnik/mapnik/wiki/OptimizeRenderingWithPostGIS
const MAP_STYLE_TABLE_DEFINITIONS = {
base_light: `(
SELECT
b.location_number as location_number,
g.geometry_geom
FROM
geometries as g,
buildings as b
WHERE
g.geometry_id = b.geometry_id
) as outline`,
base_night: `(
SELECT
b.location_number as location_number,
g.geometry_geom
FROM
geometries as g,
buildings as b
WHERE
g.geometry_id = b.geometry_id
) as outline`,
date_year: `(
SELECT
b.date_year as date_year,
g.geometry_geom
FROM
geometries as g,
buildings as b
WHERE
g.geometry_id = b.geometry_id
) as outline`,
size_storeys: `(
SELECT
(
coalesce(b.size_storeys_attic, 0) +
coalesce(b.size_storeys_core, 0)
) as size_storeys,
g.geometry_geom
FROM
geometries as g,
buildings as b
WHERE
g.geometry_id = b.geometry_id
) as outline`,
location: `(
SELECT
(
case when b.location_name is null then 0 else 1 end +
case when b.location_number is null then 0 else 1 end +
case when b.location_street is null then 0 else 1 end +
case when b.location_line_two is null then 0 else 1 end +
case when b.location_town is null then 0 else 1 end +
case when b.location_postcode is null then 0 else 1 end +
case when b.location_latitude is null then 0 else 1 end +
case when b.location_longitude is null then 0 else 1 end +
case when b.ref_toid is null then 0 else 1 end +
case when b.ref_osm_id is null then 0 else 1 end
) as location_info_count,
g.geometry_geom
FROM
geometries as g,
buildings as b
WHERE
g.geometry_id = b.geometry_id
) as location`,
likes: `(
SELECT
g.geometry_geom,
b.likes_total as likes
FROM
geometries as g,
buildings as b
WHERE
g.geometry_id = b.geometry_id
AND b.likes_total > 0
) as location`,
conservation_area: `(
SELECT
g.geometry_geom
FROM
geometries as g,
buildings as b
WHERE
g.geometry_id = b.geometry_id
AND b.planning_in_conservation_area = true
) as conservation_area`
}
// register datasource adapters for mapnik database connection
if (mapnik.register_default_input_plugins) {
mapnik.register_default_input_plugins();
}
// register fonts for text rendering
mapnik.register_default_fonts();
const mercator = new SphericalMercator({
size: TILE_SIZE
});
function getBbox(z, x, y) {
return mercator.bbox(x, y, z, false, '900913');
}
function getXYZ(bbox, z) {
return mercator.xyz(bbox, z, false, '900913')
}
function renderTile(tileset, z, x, y, geometryId, cb) {
const bbox = getBbox(z, x, y)
const map = new mapnik.Map(TILE_SIZE, TILE_SIZE, PROJ4_STRING);
map.bufferSize = TILE_BUFFER_SIZE;
const layer = new mapnik.Layer('tile', PROJ4_STRING);
const tableDefinition = (tileset === 'highlight') ?
getHighlightTableDefinition(geometryId)
: MAP_STYLE_TABLE_DEFINITIONS[tileset];
const conf = Object.assign({ table: tableDefinition }, DATASOURCE_CONFIG)
var postgis;
try {
postgis = new mapnik.Datasource(conf);
layer.datasource = postgis;
layer.styles = [tileset]
map.load(
path.join(__dirname, '..', 'map_styles', 'polygon.xml'),
{ strict: true },
function (err, map) {
if (err) {throw err}
map.add_layer(layer)
const im = new mapnik.Image(map.width, map.height)
map.extent = bbox
map.render(im, {}, (err, rendered) => {
if (err) {throw err}
rendered.encode('png', cb)
});
}
)
} catch (err) {
console.error(err);
}
}
// highlight single geometry, requires geometryId in the table query
function getHighlightTableDefinition(geometryId) {
return `(
SELECT
g.geometry_geom
FROM
geometries as g
WHERE
g.geometry_id = ${geometryId}
) as highlight`
}
export { getBbox, getXYZ, renderTile, TILE_SIZE };

143
app/src/tiles/tileCache.ts Normal file
View File

@ -0,0 +1,143 @@
/**
* Cache tiles (PNG images generated from database)
*
* Frequency of change:
* - base layer tiles change rarely - on changes to underlying geometry table
* - visualisation layer tiles change frequently - with almost any edit to the buildings table
*
* Cost of generation and storage:
* - low zoom tiles are more expensive to render, containing more features from the database
* - high zoom tiles are cheaper to rerender, and changes are more visible
* - there are many more high zoom tiles than low: 4 tiles at zoom level n+1 for each tile
* at zoom level n
*
*/
// Using node-fs package to patch fs
// for node >10 we could drop this in favour of fs.mkdir (which has recursive option)
// and then use stdlib `import fs from 'fs';`
import fs from 'node-fs';
import { promisify } from 'util'
import { Image } from 'mapnik';
import { TileParams, BoundingBox } from './types';
import { getXYZ, formatParams } from './util';
// TODO: switch to modern node and use built-in fs with promise-based API
const readFile = promisify(fs.readFile),
writeFile = promisify(fs.writeFile),
mkdir = promisify(fs.mkdir),
unlink = promisify(fs.unlink);
interface CacheLocation {
/**
* Cache file directory path
*/
dir: string;
/**
* Full path to cache file
*/
fname: string;
}
interface CacheDomain {
/**
* An array of tileset names to cache
*/
tilesets: string[];
/**
* The lowest zoom level to cache
*/
minZoom: number;
/**
* The highest zoom level to cache
*/
maxZoom: number;
/**
* An array of scale factors to cache
*/
scales: number[];
}
class TileCache {
constructor(
/** Base path in filesystem to store the cache */
private basePath: string,
/** Domain definition for the cache */
private cacheDomain: CacheDomain,
/** Function for defining custom caching rules (optional) */
private shouldCacheFn?: (TileParams) => boolean
) {}
async get(tileParams: TileParams): Promise<Image> {
if (!this.shouldUseCache(tileParams)) {
throw new Error(`Skip cache get ${formatParams(tileParams)}`);
}
const location = this.cacheLocation(tileParams);
return readFile(location.fname);
}
async put(im: Image, tileParams: TileParams): Promise<void> {
if (!this.shouldUseCache(tileParams)) {
throw new Error(`Skip cache put ${formatParams(tileParams)}`);
}
const location = this.cacheLocation(tileParams);
try {
await writeFile(location.fname, im, 'binary');
} catch(err) {
if(err.code === 'ENOENT') {
await mkdir(location.dir, 0o755, true);
await writeFile(location.fname, im, 'binary');
} else throw err;
}
}
async remove(tileParams: TileParams): Promise<void> {
const location = this.cacheLocation(tileParams);
try {
await unlink(location.fname);
} catch(err) {}
console.log(`Expire cache ${formatParams(tileParams)}`);
}
async removeAllAtBbox(bbox: BoundingBox): Promise<void[]> {
const removePromises: Promise<void>[] = [];
for (const tileset of this.cacheDomain.tilesets) {
for (let z = this.cacheDomain.minZoom; z <= this.cacheDomain.maxZoom; z++) {
let tileBounds = getXYZ(bbox, z)
for (let x = tileBounds.minX; x <= tileBounds.maxX; x++) {
for (let y = tileBounds.minY; y <= tileBounds.maxY; y++) {
for (const scale of this.cacheDomain.scales) {
removePromises.push(this.remove({tileset, z, x, y, scale}));
}
}
}
}
}
return Promise.all(removePromises);
}
private cacheLocation({tileset, z, x, y, scale}: TileParams): CacheLocation {
const dir = `${this.basePath}/${tileset}/${z}/${x}`;
const fname = `${dir}/${y}@${scale}x.png`;
return { dir, fname };
}
private shouldUseCache(tileParams: TileParams): boolean {
return this.cacheDomain.tilesets.includes(tileParams.tileset) &&
this.cacheDomain.minZoom <= tileParams.z &&
this.cacheDomain.maxZoom >= tileParams.z &&
this.cacheDomain.scales.includes(tileParams.scale) &&
(this.shouldCacheFn == undefined || this.shouldCacheFn(tileParams));
}
}
export {
TileCache
};

View File

@ -1,215 +1,73 @@
/**
* Tileserver
* - routes for Express app
* - stitch tiles above a certain zoom level (compositing from sharply-rendered lower zooms)
* - render empty tile outside extent of geographical area of interest
*
* - see rendererDefinition for actual rules of rendering
*/
import express from 'express';
import sharp from 'sharp';
import { get, put } from './cache';
import { renderTile, getBbox, getXYZ, TILE_SIZE } from './tile';
import { strictParseInt } from '../parse';
import { TileParams } from './types';
import { mainRenderer, allTilesets } from './rendererDefinition';
import asyncController from '../api/routes/asyncController';
// zoom level when we switch from rendering direct from database to instead composing tiles
// from the zoom level below - gets similar effect, with much lower load on Postgres
const STITCH_THRESHOLD = 12
// Hard-code extent so we can short-circuit rendering and return empty/transparent tiles outside the area of interest
// bbox in CRS espg:3957 in form: [w, s, e, n]
const EXTENT_BBOX = [-61149.622628, 6667754.851372, 28128.826409, 6744803.375884]
const handleTileRequest = asyncController(async function (req: express.Request, res: express.Response) {
try {
var tileParams = parseTileParams(req.params);
var dataParams = req.query;
} catch(err) {
console.error(err);
return res.status(400).send({error: err.message});
}
try {
const im = await mainRenderer.getTile(tileParams, dataParams);
res.writeHead(200, { 'Content-Type': 'image/png' });
res.end(im);
} catch(err) {
console.error(err);
res.status(500).send({ error: err });
}
});
// tiles router
const router = express.Router()
router.get('/highlight/:z/:x/:y.png', handleHighlightTileRequest);
router.get('/:tileset/:z/:x/:y(\\d+):scale(@\\dx)?.png', handleTileRequest);
router.get('/base_light/:z/:x/:y.png', (req, res) => {
handleTileRequest('base_light', req, res)
});
function parseTileParams(params: any): TileParams {
const { tileset, z, x, y, scale } = params;
router.get('/base_night/:z/:x/:y.png', (req, res) => {
handleTileRequest('base_night', req, res)
});
router.get('/date_year/:z/:x/:y.png', (req, res) => {
handleTileRequest('date_year', req, res)
});
router.get('/size_storeys/:z/:x/:y.png', (req, res) => {
handleTileRequest('size_storeys', req, res)
});
router.get('/location/:z/:x/:y.png', (req, res) => {
handleTileRequest('location', req, res)
});
router.get('/likes/:z/:x/:y.png', (req, res) => {
handleTileRequest('likes', req, res)
});
router.get('/conservation_area/:z/:x/:y.png', (req, res) => {
handleTileRequest('conservation_area', req, res)
});
function handleTileRequest(tileset, req, res) {
const { z, x, y } = req.params
if (!allTilesets.includes(tileset)) throw new Error('Invalid value for tileset');
const intZ = strictParseInt(z);
if (isNaN(intZ)) throw new Error('Invalid value for z');
const intX = strictParseInt(x);
if (isNaN(intX)) throw new Error('Invalid value for x');
const intY = strictParseInt(y);
if (isNaN(intY)) throw new Error('Invalid value for y');
if (isNaN(intX) || isNaN(intY) || isNaN(intZ)) {
console.error('Missing x or y or z')
return { error: 'Bad parameter' }
}
loadTile(tileset, intZ, intX, intY).then((im) => {
res.writeHead(200, { 'Content-Type': 'image/png' })
res.end(im)
}).catch((err) => {
console.error(err)
res.status(500).send({ error: err })
})
}
function loadTile(tileset, z, x, y) {
if (outsideExtent(z, x, y)) {
return emptyTile()
}
return get(tileset, z, x, y).then((im) => {
console.log(`From cache ${tileset}/${z}/${x}/${y}`)
return im
}).catch(() => {
return renderOrStitchTile(tileset, z, x, y)
})
}
function renderOrStitchTile(tileset, z, x, y) {
if (z <= STITCH_THRESHOLD) {
return StitchTile(tileset, z, x, y).then(im => {
return put(im, tileset, z, x, y).then(() => {
console.log(`Stitch ${tileset}/${z}/${x}/${y}`)
return im
}).catch((err) => {
console.error(err)
return im
})
})
let intScale: number;
if (scale === '@2x') {
intScale = 2;
} else if (scale === '@1x' || scale == undefined) {
intScale = 1;
} else {
return new Promise((resolve, reject) => {
renderTile(tileset, z, x, y, undefined, (err, im) => {
if (err) {
reject(err)
return
}
put(im, tileset, z, x, y).then(() => {
console.log(`Render ${tileset}/${z}/${x}/${y}`)
resolve(im)
}).catch((err) => {
console.error(err)
resolve(im)
})
})
})
}
}
function outsideExtent(z, x, y) {
const xy = getXYZ(EXTENT_BBOX, z);
return xy.minY > y || xy.maxY < y || xy.minX > x || xy.maxX < x;
}
function emptyTile() {
return sharp({
create: {
width: 1,
height: 1,
channels: 4,
background: { r: 0, g: 0, b: 0, alpha: 0 }
}
}).png().toBuffer()
}
function StitchTile(tileset, z, x, y) {
const bbox = getBbox(z, x, y)
const nextZ = z + 1
const nextXY = getXYZ(bbox, nextZ)
return Promise.all([
// recurse down through zoom levels, using cache if available...
loadTile(tileset, nextZ, nextXY.minX, nextXY.minY),
loadTile(tileset, nextZ, nextXY.maxX, nextXY.minY),
loadTile(tileset, nextZ, nextXY.minX, nextXY.maxY),
loadTile(tileset, nextZ, nextXY.maxX, nextXY.maxY)
]).then(([
topLeft,
topRight,
bottomLeft,
bottomRight
]) => {
// not possible to chain overlays in a single pipeline, but there may still be a better
// way to create image buffer here (four tiles resize to one at the next zoom level)
// instead of repeatedly creating `sharp` objects, to png, to buffer...
return sharp({
create: {
width: TILE_SIZE * 2,
height: TILE_SIZE * 2,
channels: 4,
background: { r: 0, g: 0, b: 0, alpha: 0 }
}
}).overlayWith(
topLeft, { gravity: sharp.gravity.northwest }
).png().toBuffer().then((buf) => {
return sharp(buf).overlayWith(
topRight, { gravity: sharp.gravity.northeast }
).png().toBuffer()
}).then((buf) => {
return sharp(buf).overlayWith(
bottomLeft, { gravity: sharp.gravity.southwest }
).png().toBuffer()
}).then((buf) => {
return sharp(buf).overlayWith(
bottomRight, { gravity: sharp.gravity.southeast }
).png().toBuffer()
}).then((buf) => {
return sharp(buf
).resize(TILE_SIZE, TILE_SIZE, { fit: 'inside' }
).png().toBuffer()
})
});
}
function handleHighlightTileRequest(req, res) {
const { z, x, y } = req.params
const intZ = strictParseInt(z);
const intX = strictParseInt(x);
const intY = strictParseInt(y);
if (isNaN(intX) || isNaN(intY) || isNaN(intZ)) {
console.error('Missing x or y or z')
return { error: 'Bad parameter' }
throw new Error('Invalid value for scale');
}
// highlight layer uses geometry_id to outline a single building
const { highlight } = req.query
const geometryId = strictParseInt(highlight);
if (isNaN(geometryId)) {
res.status(400).send({ error: 'Bad parameter' })
return
}
if (outsideExtent(z, x, y)) {
return emptyTile()
}
renderTile('highlight', intZ, intX, intY, geometryId, function (err, im) {
if (err) {throw err}
res.writeHead(200, { 'Content-Type': 'image/png' })
res.end(im)
})
return {
tileset,
z: intZ,
x: intX,
y: intY,
scale: intScale
};
}
router.use((req, res) => {
return res.status(404).send('Tile not found');
});
export default router;

43
app/src/tiles/types.ts Normal file
View File

@ -0,0 +1,43 @@
import { Image } from 'mapnik';
/**
* Bounding box in the format [w, s, e, n]
*/
type BoundingBox = [number, number, number, number];
interface TileParams {
/**
* Name of tileset to which the tile belongs
*/
tileset: string;
/**
* Zoom level
*/
z: number;
/**
* X coordinate of tile (corresponds to longitude)
*/
x: number;
/**
* Y coordinate of tile (corresponds to latitude)
*/
y: number;
/**
* Resolution scale factor for higher pixel density tiles (e.g. x2)
*/
scale: number;
}
interface TileRenderer {
getTile(tileParams: TileParams, dataParams: any): Promise<Image>
}
export {
BoundingBox,
TileParams,
TileRenderer
};

28
app/src/tiles/util.ts Normal file
View File

@ -0,0 +1,28 @@
import SphericalMercator from '@mapbox/sphericalmercator';
import { TileParams } from './types';
const TILE_SIZE = 256;
const mercator = new SphericalMercator({
size: TILE_SIZE
});
function getBbox(z, x, y) {
return mercator.bbox(x, y, z, false, '900913');
}
function getXYZ(bbox, z) {
return mercator.xyz(bbox, z, false, '900913')
}
function formatParams({ tileset, z, x, y, scale }: TileParams): string {
return `${tileset}/${z}/${x}/${y}@${scale}x`;
}
export {
TILE_SIZE,
getBbox,
getXYZ,
formatParams
};