a_adli/colouring-montreal
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use new renderers in tile server

Browse Source
This commit is contained in:
Maciej Ziarkowski 2019-09-17 18:11:42 +01:00
parent ef4d46e36b
commit 5adb8e6146
3 changed files with 47 additions and 563 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

171
app/src/tiles/cache.ts
View File

@ -1,171 +0,0 @@
/**
* Cache tiles (PNG images generated from database)
*
* Frequency of change:
* - base layer tiles change rarely - on changes to underlying geometry table
* - visualisation layer tiles change frequently - with almost any edit to the buildings table
*
* Cost of generation and storage:
* - low zoom tiles are more expensive to render, containing more features from the database
* - high zoom tiles are cheaper to rerender, and changes are more visible
* - there are many more high zoom tiles than low: 4 tiles at zoom level n+1 for each tile
* at zoom level n
*
*/
// Using node-fs package to patch fs
// for node >10 we could drop this in favour of fs.mkdir (which has recursive option)
// and then use stdlib `import fs from 'fs';`
import fs from 'node-fs';
import { getXYZ } from './tile';
// Use an environment variable to configure the cache location, somewhere we can read/write to.
const CACHE_PATH = process.env.TILECACHE_PATH
/**
* Get a tile from the cache
*
* @param {String} tileset
* @param {number} z zoom level
* @param {number} x
* @param {number} y
*/
function get(tileset, z, x, y) {
if (!shouldTryCache(tileset, z)) {
return Promise.reject(`Skip cache get ${tileset}/${z}/${x}/${y}`);
}
const location = cacheLocation(tileset, z, x, y);
return new Promise((resolve, reject) => {
fs.readFile(location.fname, (err, data) => {
if (err) {
reject(err)
} else {
resolve(data)
}
})
});
}
/**
* Put a tile in the cache
*
* @param {Buffer} im image data
* @param {String} tileset
* @param {number} z zoom level
* @param {number} x
* @param {number} y
*/
function put(im, tileset, z, x, y) {
if (!shouldTryCache(tileset, z)) {
return Promise.reject(`Skip cache put ${tileset}/${z}/${x}/${y}`);
}
const location = cacheLocation(tileset, z, x, y);
return new Promise((resolve, reject) => {
fs.writeFile(location.fname, im, 'binary', (err) => {
if (err && err.code === 'ENOENT') {
// recursively create tile directory if it didn't previously exist
fs.mkdir(location.dir, 0o755, true, (err) => {
if (err) {
reject(err);
} else {
// then write the file
fs.writeFile(location.fname, im, 'binary', (err) => {
(err)? reject(err): resolve()
});
}
});
} else {
(err)? reject(err): resolve()
}
});
})
}
/**
* Remove a single cached tile
*
* @param {String} tileset
* @param {number} z zoom level
* @param {number} x
* @param {number} y
*/
function remove(tileset, z, x, y) {
const location = cacheLocation(tileset, z, x, y)
return new Promise(resolve => {
fs.unlink(location.fname, (err) => {
if(err){
// pass
} else {
console.log('Expire cache', tileset, z, x, y)
}
resolve()
})
})
}
/**
* Remove all cached data-visualising tiles which intersect a bbox
* - initially called directly after edits; may be better on a worker process?
*
* @param {String} tileset
* @param {Array} bbox [w, s, e, n] in EPSG:3857 coordinates
*/
function removeAllAtBbox(bbox) {
// magic numbers for min/max zoom
const minZoom = 9;
const maxZoom = 18;
// magic list of tilesets - see tileserver, other cache rules
const tilesets = ['date_year', 'size_storeys', 'location', 'likes', 'conservation_area'];
let tileBounds;
const removePromises = [];
for (let ti = 0; ti < tilesets.length; ti++) {
const tileset = tilesets[ti];
for (let z = minZoom; z <= maxZoom; z++) {
tileBounds = getXYZ(bbox, z)
for (let x = tileBounds.minX; x <= tileBounds.maxX; x++){
for (let y = tileBounds.minY; y <= tileBounds.maxY; y++){
removePromises.push(remove(tileset, z, x, y))
}
}
}
}
Promise.all(removePromises)
}
/**
* Cache location for a tile
*
* @param {String} tileset
* @param {number} z zoom level
* @param {number} x
* @param {number} y
* @returns {object} { dir: <directory>, fname: <full filepath> }
*/
function cacheLocation(tileset, z, x, y) {
const dir = `${CACHE_PATH}/${tileset}/${z}/${x}`
const fname = `${dir}/${y}.png`
return {dir, fname}
}
/**
* Check rules for caching tiles
*
* @param {String} tileset
* @param {number} z zoom level
* @returns {boolean} whether to use the cache (or not)
*/
function shouldTryCache(tileset, z) {
if (tileset === 'date_year') {
// cache high zoom because of front page hits
return z <= 16
}
if (tileset === 'base_light' || tileset === 'base_night') {
// cache for higher zoom levels (unlikely to change)
return z <= 17
}
// else cache for lower zoom levels (change slowly)
return z <= 13
}
export { get, put, remove, removeAllAtBbox };

201
app/src/tiles/tile.ts
View File

@ -1,201 +0,0 @@
/**
* Render tiles
*
* Use mapnik to render map tiles from the database
*
* Styles have two sources of truth for colour ranges (could generate from single source?)
* - XML style definitions in app/map_styles/polygon.xml
* - front-end legend in app/src/frontend/legend.js
*
* Data is provided by the queries in MAP_STYLE_TABLE_DEFINITIONS below.
*
*/
import path from 'path';
import mapnik from 'mapnik';
import SphericalMercator from '@mapbox/sphericalmercator';
// connection details from environment variables
const DATASOURCE_CONFIG = {
'host': process.env.PGHOST,
'dbname': process.env.PGDATABASE,
'user': process.env.PGUSER,
'password': process.env.PGPASSWORD,
'port': process.env.PGPORT,
'geometry_field': 'geometry_geom',
'extent': '-20005048.4188,-9039211.13765,19907487.2779,17096598.5401',
'srid': 3857,
'type': 'postgis'
}
const TILE_SIZE = 256
const TILE_BUFFER_SIZE = 64
const PROJ4_STRING = '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over';
// Mapnik uses table definitions to query geometries and attributes from PostGIS.
// The queries here are eventually used as subqueries when Mapnik fetches data to render a
// tile - so given a table definition like:
// (SELECT geometry_geom FROM geometries) as def
// Mapnik will wrap it in a bbox query and PostGIS will eventually see something like:
// SELECT AsBinary("geometry") AS geom from
// (SELECT geometry_geom FROM geometries) as def
// WHERE "geometry" && SetSRID('BOX3D(0,1,2,3)'::box3d, 3857)
// see docs: https://github.com/mapnik/mapnik/wiki/OptimizeRenderingWithPostGIS
const MAP_STYLE_TABLE_DEFINITIONS = {
base_light: `(
SELECT
b.location_number as location_number,
g.geometry_geom
FROM
geometries as g,
buildings as b
WHERE
g.geometry_id = b.geometry_id
) as outline`,
base_night: `(
SELECT
b.location_number as location_number,
g.geometry_geom
FROM
geometries as g,
buildings as b
WHERE
g.geometry_id = b.geometry_id
) as outline`,
date_year: `(
SELECT
b.date_year as date_year,
g.geometry_geom
FROM
geometries as g,
buildings as b
WHERE
g.geometry_id = b.geometry_id
) as outline`,
size_storeys: `(
SELECT
(
coalesce(b.size_storeys_attic, 0) +
coalesce(b.size_storeys_core, 0)
) as size_storeys,
g.geometry_geom
FROM
geometries as g,
buildings as b
WHERE
g.geometry_id = b.geometry_id
) as outline`,
location: `(
SELECT
(
case when b.location_name is null then 0 else 1 end +
case when b.location_number is null then 0 else 1 end +
case when b.location_street is null then 0 else 1 end +
case when b.location_line_two is null then 0 else 1 end +
case when b.location_town is null then 0 else 1 end +
case when b.location_postcode is null then 0 else 1 end +
case when b.location_latitude is null then 0 else 1 end +
case when b.location_longitude is null then 0 else 1 end +
case when b.ref_toid is null then 0 else 1 end +
case when b.ref_osm_id is null then 0 else 1 end
) as location_info_count,
g.geometry_geom
FROM
geometries as g,
buildings as b
WHERE
g.geometry_id = b.geometry_id
) as location`,
likes: `(
SELECT
g.geometry_geom,
b.likes_total as likes
FROM
geometries as g,
buildings as b
WHERE
g.geometry_id = b.geometry_id
AND b.likes_total > 0
) as location`,
conservation_area: `(
SELECT
g.geometry_geom
FROM
geometries as g,
buildings as b
WHERE
g.geometry_id = b.geometry_id
AND b.planning_in_conservation_area = true
) as conservation_area`
}
// register datasource adapters for mapnik database connection
if (mapnik.register_default_input_plugins) {
mapnik.register_default_input_plugins();
}
// register fonts for text rendering
mapnik.register_default_fonts();
const mercator = new SphericalMercator({
size: TILE_SIZE
});
function getBbox(z, x, y) {
return mercator.bbox(x, y, z, false, '900913');
}
function getXYZ(bbox, z) {
return mercator.xyz(bbox, z, false, '900913')
}
function renderTile(tileset, z, x, y, geometryId, cb) {
const bbox = getBbox(z, x, y)
const map = new mapnik.Map(TILE_SIZE, TILE_SIZE, PROJ4_STRING);
map.bufferSize = TILE_BUFFER_SIZE;
const layer = new mapnik.Layer('tile', PROJ4_STRING);
const tableDefinition = (tileset === 'highlight') ?
getHighlightTableDefinition(geometryId)
: MAP_STYLE_TABLE_DEFINITIONS[tileset];
const conf = Object.assign({ table: tableDefinition }, DATASOURCE_CONFIG)
var postgis;
try {
postgis = new mapnik.Datasource(conf);
layer.datasource = postgis;
layer.styles = [tileset]
map.load(
path.join(__dirname, '..', 'map_styles', 'polygon.xml'),
{ strict: true },
function (err, map) {
if (err) {throw err}
map.add_layer(layer)
const im = new mapnik.Image(map.width, map.height)
map.extent = bbox
map.render(im, {}, (err, rendered) => {
if (err) {throw err}
rendered.encode('png', cb)
});
}
)
} catch (err) {
console.error(err);
}
}
// highlight single geometry, requires geometryId in the table query
function getHighlightTableDefinition(geometryId) {
return `(
SELECT
g.geometry_geom
FROM
geometries as g
WHERE
g.geometry_id = ${geometryId}
) as highlight`
}
export { getBbox, getXYZ, renderTile, TILE_SIZE };

236
app/src/tiles/tileserver.ts
View File

@ -1,215 +1,71 @@
/**
* Tileserver
* - routes for Express app
* - stitch tiles above a certain zoom level (compositing from sharply-rendered lower zooms)
* - render empty tile outside extent of geographical area of interest
*
* - see rendererDefinition for actual rules of rendering
*/
import express from 'express';
import sharp from 'sharp';
import { get, put } from './cache';
import { renderTile, getBbox, getXYZ, TILE_SIZE } from './tile';
import { strictParseInt } from '../parse';
import { TileParams } from './types';
import { mainRenderer } from './rendererDefinition';
import asyncController from '../api/routes/asyncController';
// zoom level when we switch from rendering direct from database to instead composing tiles
// from the zoom level below - gets similar effect, with much lower load on Postgres
const STITCH_THRESHOLD = 12
const handleTileRequest = asyncController(async function (req: express.Request, res: express.Response) {
try {
var tileParams = parseTileParams(req.params);
var dataParams = req.query;
} catch(err) {
console.error(err);
return res.status(400).send({error: err.message});
}
// Hard-code extent so we can short-circuit rendering and return empty/transparent tiles outside the area of interest
// bbox in CRS espg:3957 in form: [w, s, e, n]
const EXTENT_BBOX = [-61149.622628, 6667754.851372, 28128.826409, 6744803.375884]
try {
const im = await mainRenderer.getTile(tileParams, dataParams);
res.writeHead(200, { 'Content-Type': 'image/png' });
res.end(im);
} catch(err) {
console.error(err);
res.status(500).send({ error: err });
}
});
// tiles router
const router = express.Router()
router.get('/highlight/:z/:x/:y.png', handleHighlightTileRequest);
router.get('/:tileset/:z/:x/:y(\\d+):scale(@\\dx)?.png', handleTileRequest);
router.get('/base_light/:z/:x/:y.png', (req, res) => {
handleTileRequest('base_light', req, res)
});
function parseTileParams(params: any): TileParams {
const { tileset, z, x, y, scale } = params;
router.get('/base_night/:z/:x/:y.png', (req, res) => {
handleTileRequest('base_night', req, res)
});
router.get('/date_year/:z/:x/:y.png', (req, res) => {
handleTileRequest('date_year', req, res)
});
router.get('/size_storeys/:z/:x/:y.png', (req, res) => {
handleTileRequest('size_storeys', req, res)
});
router.get('/location/:z/:x/:y.png', (req, res) => {
handleTileRequest('location', req, res)
});
router.get('/likes/:z/:x/:y.png', (req, res) => {
handleTileRequest('likes', req, res)
});
router.get('/conservation_area/:z/:x/:y.png', (req, res) => {
handleTileRequest('conservation_area', req, res)
});
function handleTileRequest(tileset, req, res) {
const { z, x, y } = req.params
const intZ = strictParseInt(z);
if (isNaN(intZ)) throw new Error('Invalid value for z');
const intX = strictParseInt(x);
if (isNaN(intX)) throw new Error('Invalid value for x');
const intY = strictParseInt(y);
if (isNaN(intY)) throw new Error('Invalid value for y');
if (isNaN(intX) || isNaN(intY) || isNaN(intZ)) {
console.error('Missing x or y or z')
return { error: 'Bad parameter' }
}
loadTile(tileset, intZ, intX, intY).then((im) => {
res.writeHead(200, { 'Content-Type': 'image/png' })
res.end(im)
}).catch((err) => {
console.error(err)
res.status(500).send({ error: err })
})
}
function loadTile(tileset, z, x, y) {
if (outsideExtent(z, x, y)) {
return emptyTile()
}
return get(tileset, z, x, y).then((im) => {
console.log(`From cache ${tileset}/${z}/${x}/${y}`)
return im
}).catch(() => {
return renderOrStitchTile(tileset, z, x, y)
})
}
function renderOrStitchTile(tileset, z, x, y) {
if (z <= STITCH_THRESHOLD) {
return StitchTile(tileset, z, x, y).then(im => {
return put(im, tileset, z, x, y).then(() => {
console.log(`Stitch ${tileset}/${z}/${x}/${y}`)
return im
}).catch((err) => {
console.error(err)
return im
})
})
let intScale: number;
if (scale === '@2x') {
intScale = 2;
} else if (scale === '@1x' || scale == undefined) {
intScale = 1;
} else {
throw new Error('Invalid value for scale');
}
return new Promise((resolve, reject) => {
renderTile(tileset, z, x, y, undefined, (err, im) => {
if (err) {
reject(err)
return
}
put(im, tileset, z, x, y).then(() => {
console.log(`Render ${tileset}/${z}/${x}/${y}`)
resolve(im)
}).catch((err) => {
console.error(err)
resolve(im)
})
})
})
}
return {
tileset,
z: intZ,
x: intX,
y: intY,
scale: intScale
};
}
function outsideExtent(z, x, y) {
const xy = getXYZ(EXTENT_BBOX, z);
return xy.minY > y || xy.maxY < y || xy.minX > x || xy.maxX < x;
}
function emptyTile() {
return sharp({
create: {
width: 1,
height: 1,
channels: 4,
background: { r: 0, g: 0, b: 0, alpha: 0 }
}
}).png().toBuffer()
}
function StitchTile(tileset, z, x, y) {
const bbox = getBbox(z, x, y)
const nextZ = z + 1
const nextXY = getXYZ(bbox, nextZ)
return Promise.all([
// recurse down through zoom levels, using cache if available...
loadTile(tileset, nextZ, nextXY.minX, nextXY.minY),
loadTile(tileset, nextZ, nextXY.maxX, nextXY.minY),
loadTile(tileset, nextZ, nextXY.minX, nextXY.maxY),
loadTile(tileset, nextZ, nextXY.maxX, nextXY.maxY)
]).then(([
topLeft,
topRight,
bottomLeft,
bottomRight
]) => {
// not possible to chain overlays in a single pipeline, but there may still be a better
// way to create image buffer here (four tiles resize to one at the next zoom level)
// instead of repeatedly creating `sharp` objects, to png, to buffer...
return sharp({
create: {
width: TILE_SIZE * 2,
height: TILE_SIZE * 2,
channels: 4,
background: { r: 0, g: 0, b: 0, alpha: 0 }
}
}).overlayWith(
topLeft, { gravity: sharp.gravity.northwest }
).png().toBuffer().then((buf) => {
return sharp(buf).overlayWith(
topRight, { gravity: sharp.gravity.northeast }
).png().toBuffer()
}).then((buf) => {
return sharp(buf).overlayWith(
bottomLeft, { gravity: sharp.gravity.southwest }
).png().toBuffer()
}).then((buf) => {
return sharp(buf).overlayWith(
bottomRight, { gravity: sharp.gravity.southeast }
).png().toBuffer()
}).then((buf) => {
return sharp(buf
).resize(TILE_SIZE, TILE_SIZE, { fit: 'inside' }
).png().toBuffer()
})
});
}
function handleHighlightTileRequest(req, res) {
const { z, x, y } = req.params
const intZ = strictParseInt(z);
const intX = strictParseInt(x);
const intY = strictParseInt(y);
if (isNaN(intX) || isNaN(intY) || isNaN(intZ)) {
console.error('Missing x or y or z')
return { error: 'Bad parameter' }
}
// highlight layer uses geometry_id to outline a single building
const { highlight } = req.query
const geometryId = strictParseInt(highlight);
if (isNaN(geometryId)) {
res.status(400).send({ error: 'Bad parameter' })
return
}
if (outsideExtent(z, x, y)) {
return emptyTile()
}
renderTile('highlight', intZ, intX, intY, geometryId, function (err, im) {
if (err) {throw err}
res.writeHead(200, { 'Content-Type': 'image/png' })
res.end(im)
})
}
router.use((req, res) => {
return res.status(404).send('Tile not found');
});
export default router;