diff --git a/app/src/tiles/cache.ts b/app/src/tiles/cache.ts deleted file mode 100644 index f018e444..00000000 --- a/app/src/tiles/cache.ts +++ /dev/null @@ -1,171 +0,0 @@ -/** - * Cache tiles (PNG images generated from database) - * - * Frequency of change: - * - base layer tiles change rarely - on changes to underlying geometry table - * - visualisation layer tiles change frequently - with almost any edit to the buildings table - * - * Cost of generation and storage: - * - low zoom tiles are more expensive to render, containing more features from the database - * - high zoom tiles are cheaper to rerender, and changes are more visible - * - there are many more high zoom tiles than low: 4 tiles at zoom level n+1 for each tile - * at zoom level n - * - */ - -// Using node-fs package to patch fs -// for node >10 we could drop this in favour of fs.mkdir (which has recursive option) -// and then use stdlib `import fs from 'fs';` -import fs from 'node-fs'; - -import { getXYZ } from './tile'; - -// Use an environment variable to configure the cache location, somewhere we can read/write to. -const CACHE_PATH = process.env.TILECACHE_PATH - -/** - * Get a tile from the cache - * - * @param {String} tileset - * @param {number} z zoom level - * @param {number} x - * @param {number} y - */ -function get(tileset, z, x, y) { - if (!shouldTryCache(tileset, z)) { - return Promise.reject(`Skip cache get ${tileset}/${z}/${x}/${y}`); - } - const location = cacheLocation(tileset, z, x, y); - return new Promise((resolve, reject) => { - fs.readFile(location.fname, (err, data) => { - if (err) { - reject(err) - } else { - resolve(data) - } - }) - }); -} - -/** - * Put a tile in the cache - * - * @param {Buffer} im image data - * @param {String} tileset - * @param {number} z zoom level - * @param {number} x - * @param {number} y - */ -function put(im, tileset, z, x, y) { - if (!shouldTryCache(tileset, z)) { - return Promise.reject(`Skip cache put ${tileset}/${z}/${x}/${y}`); - } - const location = cacheLocation(tileset, z, x, y); - return new Promise((resolve, reject) => { - fs.writeFile(location.fname, im, 'binary', (err) => { - if (err && err.code === 'ENOENT') { - // recursively create tile directory if it didn't previously exist - fs.mkdir(location.dir, 0o755, true, (err) => { - if (err) { - reject(err); - } else { - // then write the file - fs.writeFile(location.fname, im, 'binary', (err) => { - (err)? reject(err): resolve() - }); - } - }); - } else { - (err)? reject(err): resolve() - } - }); - }) -} - -/** - * Remove a single cached tile - * - * @param {String} tileset - * @param {number} z zoom level - * @param {number} x - * @param {number} y - */ -function remove(tileset, z, x, y) { - const location = cacheLocation(tileset, z, x, y) - return new Promise(resolve => { - fs.unlink(location.fname, (err) => { - if(err){ - // pass - } else { - console.log('Expire cache', tileset, z, x, y) - } - resolve() - }) - }) -} - -/** - * Remove all cached data-visualising tiles which intersect a bbox - * - initially called directly after edits; may be better on a worker process? - * - * @param {String} tileset - * @param {Array} bbox [w, s, e, n] in EPSG:3857 coordinates - */ -function removeAllAtBbox(bbox) { - // magic numbers for min/max zoom - const minZoom = 9; - const maxZoom = 18; - // magic list of tilesets - see tileserver, other cache rules - const tilesets = ['date_year', 'size_storeys', 'location', 'likes', 'conservation_area']; - let tileBounds; - const removePromises = []; - for (let ti = 0; ti < tilesets.length; ti++) { - const tileset = tilesets[ti]; - for (let z = minZoom; z <= maxZoom; z++) { - tileBounds = getXYZ(bbox, z) - for (let x = tileBounds.minX; x <= tileBounds.maxX; x++){ - for (let y = tileBounds.minY; y <= tileBounds.maxY; y++){ - removePromises.push(remove(tileset, z, x, y)) - } - } - } - } - Promise.all(removePromises) -} - -/** - * Cache location for a tile - * - * @param {String} tileset - * @param {number} z zoom level - * @param {number} x - * @param {number} y - * @returns {object} { dir: , fname: } - */ -function cacheLocation(tileset, z, x, y) { - const dir = `${CACHE_PATH}/${tileset}/${z}/${x}` - const fname = `${dir}/${y}.png` - return {dir, fname} -} - -/** - * Check rules for caching tiles - * - * @param {String} tileset - * @param {number} z zoom level - * @returns {boolean} whether to use the cache (or not) - */ -function shouldTryCache(tileset, z) { - if (tileset === 'date_year') { - // cache high zoom because of front page hits - return z <= 16 - } - if (tileset === 'base_light' || tileset === 'base_night') { - // cache for higher zoom levels (unlikely to change) - return z <= 17 - } - // else cache for lower zoom levels (change slowly) - return z <= 13 -} - -export { get, put, remove, removeAllAtBbox }; diff --git a/app/src/tiles/tile.ts b/app/src/tiles/tile.ts deleted file mode 100644 index 56da53f2..00000000 --- a/app/src/tiles/tile.ts +++ /dev/null @@ -1,201 +0,0 @@ -/** - * Render tiles - * - * Use mapnik to render map tiles from the database - * - * Styles have two sources of truth for colour ranges (could generate from single source?) - * - XML style definitions in app/map_styles/polygon.xml - * - front-end legend in app/src/frontend/legend.js - * - * Data is provided by the queries in MAP_STYLE_TABLE_DEFINITIONS below. - * - */ -import path from 'path'; -import mapnik from 'mapnik'; -import SphericalMercator from '@mapbox/sphericalmercator'; - -// connection details from environment variables -const DATASOURCE_CONFIG = { - 'host': process.env.PGHOST, - 'dbname': process.env.PGDATABASE, - 'user': process.env.PGUSER, - 'password': process.env.PGPASSWORD, - 'port': process.env.PGPORT, - 'geometry_field': 'geometry_geom', - 'extent': '-20005048.4188,-9039211.13765,19907487.2779,17096598.5401', - 'srid': 3857, - 'type': 'postgis' -} - -const TILE_SIZE = 256 -const TILE_BUFFER_SIZE = 64 -const PROJ4_STRING = '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over'; - -// Mapnik uses table definitions to query geometries and attributes from PostGIS. -// The queries here are eventually used as subqueries when Mapnik fetches data to render a -// tile - so given a table definition like: -// (SELECT geometry_geom FROM geometries) as def -// Mapnik will wrap it in a bbox query and PostGIS will eventually see something like: -// SELECT AsBinary("geometry") AS geom from -// (SELECT geometry_geom FROM geometries) as def -// WHERE "geometry" && SetSRID('BOX3D(0,1,2,3)'::box3d, 3857) -// see docs: https://github.com/mapnik/mapnik/wiki/OptimizeRenderingWithPostGIS -const MAP_STYLE_TABLE_DEFINITIONS = { - base_light: `( - SELECT - b.location_number as location_number, - g.geometry_geom - FROM - geometries as g, - buildings as b - WHERE - g.geometry_id = b.geometry_id - ) as outline`, - base_night: `( - SELECT - b.location_number as location_number, - g.geometry_geom - FROM - geometries as g, - buildings as b - WHERE - g.geometry_id = b.geometry_id - ) as outline`, - date_year: `( - SELECT - b.date_year as date_year, - g.geometry_geom - FROM - geometries as g, - buildings as b - WHERE - g.geometry_id = b.geometry_id - ) as outline`, - size_storeys: `( - SELECT - ( - coalesce(b.size_storeys_attic, 0) + - coalesce(b.size_storeys_core, 0) - ) as size_storeys, - g.geometry_geom - FROM - geometries as g, - buildings as b - WHERE - g.geometry_id = b.geometry_id - ) as outline`, - location: `( - SELECT - ( - case when b.location_name is null then 0 else 1 end + - case when b.location_number is null then 0 else 1 end + - case when b.location_street is null then 0 else 1 end + - case when b.location_line_two is null then 0 else 1 end + - case when b.location_town is null then 0 else 1 end + - case when b.location_postcode is null then 0 else 1 end + - case when b.location_latitude is null then 0 else 1 end + - case when b.location_longitude is null then 0 else 1 end + - case when b.ref_toid is null then 0 else 1 end + - case when b.ref_osm_id is null then 0 else 1 end - ) as location_info_count, - g.geometry_geom - FROM - geometries as g, - buildings as b - WHERE - g.geometry_id = b.geometry_id - ) as location`, - likes: `( - SELECT - g.geometry_geom, - b.likes_total as likes - FROM - geometries as g, - buildings as b - WHERE - g.geometry_id = b.geometry_id - AND b.likes_total > 0 - ) as location`, - conservation_area: `( - SELECT - g.geometry_geom - FROM - geometries as g, - buildings as b - WHERE - g.geometry_id = b.geometry_id - AND b.planning_in_conservation_area = true - ) as conservation_area` -} - -// register datasource adapters for mapnik database connection -if (mapnik.register_default_input_plugins) { - mapnik.register_default_input_plugins(); -} -// register fonts for text rendering -mapnik.register_default_fonts(); - -const mercator = new SphericalMercator({ - size: TILE_SIZE -}); - -function getBbox(z, x, y) { - return mercator.bbox(x, y, z, false, '900913'); -} - -function getXYZ(bbox, z) { - return mercator.xyz(bbox, z, false, '900913') -} - -function renderTile(tileset, z, x, y, geometryId, cb) { - const bbox = getBbox(z, x, y) - - const map = new mapnik.Map(TILE_SIZE, TILE_SIZE, PROJ4_STRING); - map.bufferSize = TILE_BUFFER_SIZE; - const layer = new mapnik.Layer('tile', PROJ4_STRING); - - const tableDefinition = (tileset === 'highlight') ? - getHighlightTableDefinition(geometryId) - : MAP_STYLE_TABLE_DEFINITIONS[tileset]; - - const conf = Object.assign({ table: tableDefinition }, DATASOURCE_CONFIG) - - var postgis; - try { - postgis = new mapnik.Datasource(conf); - layer.datasource = postgis; - layer.styles = [tileset] - - map.load( - path.join(__dirname, '..', 'map_styles', 'polygon.xml'), - { strict: true }, - function (err, map) { - if (err) {throw err} - - map.add_layer(layer) - const im = new mapnik.Image(map.width, map.height) - map.extent = bbox - map.render(im, {}, (err, rendered) => { - if (err) {throw err} - rendered.encode('png', cb) - }); - } - ) - } catch (err) { - console.error(err); - } -} - -// highlight single geometry, requires geometryId in the table query -function getHighlightTableDefinition(geometryId) { - return `( - SELECT - g.geometry_geom - FROM - geometries as g - WHERE - g.geometry_id = ${geometryId} - ) as highlight` -} - -export { getBbox, getXYZ, renderTile, TILE_SIZE }; diff --git a/app/src/tiles/tileserver.ts b/app/src/tiles/tileserver.ts index 803999fd..88c4e9e1 100644 --- a/app/src/tiles/tileserver.ts +++ b/app/src/tiles/tileserver.ts @@ -1,215 +1,71 @@ /** * Tileserver * - routes for Express app - * - stitch tiles above a certain zoom level (compositing from sharply-rendered lower zooms) - * - render empty tile outside extent of geographical area of interest - * + * - see rendererDefinition for actual rules of rendering */ import express from 'express'; -import sharp from 'sharp'; -import { get, put } from './cache'; -import { renderTile, getBbox, getXYZ, TILE_SIZE } from './tile'; import { strictParseInt } from '../parse'; +import { TileParams } from './types'; +import { mainRenderer } from './rendererDefinition'; +import asyncController from '../api/routes/asyncController'; -// zoom level when we switch from rendering direct from database to instead composing tiles -// from the zoom level below - gets similar effect, with much lower load on Postgres -const STITCH_THRESHOLD = 12 - -// Hard-code extent so we can short-circuit rendering and return empty/transparent tiles outside the area of interest -// bbox in CRS espg:3957 in form: [w, s, e, n] -const EXTENT_BBOX = [-61149.622628, 6667754.851372, 28128.826409, 6744803.375884] +const handleTileRequest = asyncController(async function (req: express.Request, res: express.Response) { + try { + var tileParams = parseTileParams(req.params); + var dataParams = req.query; + } catch(err) { + console.error(err); + return res.status(400).send({error: err.message}); + } + + try { + const im = await mainRenderer.getTile(tileParams, dataParams); + res.writeHead(200, { 'Content-Type': 'image/png' }); + res.end(im); + } catch(err) { + console.error(err); + res.status(500).send({ error: err }); + } +}); // tiles router const router = express.Router() -router.get('/highlight/:z/:x/:y.png', handleHighlightTileRequest); +router.get('/:tileset/:z/:x/:y(\\d+):scale(@\\dx)?.png', handleTileRequest); -router.get('/base_light/:z/:x/:y.png', (req, res) => { - handleTileRequest('base_light', req, res) -}); +function parseTileParams(params: any): TileParams { + const { tileset, z, x, y, scale } = params; -router.get('/base_night/:z/:x/:y.png', (req, res) => { - handleTileRequest('base_night', req, res) -}); - -router.get('/date_year/:z/:x/:y.png', (req, res) => { - handleTileRequest('date_year', req, res) -}); - -router.get('/size_storeys/:z/:x/:y.png', (req, res) => { - handleTileRequest('size_storeys', req, res) -}); - -router.get('/location/:z/:x/:y.png', (req, res) => { - handleTileRequest('location', req, res) -}); - -router.get('/likes/:z/:x/:y.png', (req, res) => { - handleTileRequest('likes', req, res) -}); - -router.get('/conservation_area/:z/:x/:y.png', (req, res) => { - handleTileRequest('conservation_area', req, res) -}); - -function handleTileRequest(tileset, req, res) { - const { z, x, y } = req.params const intZ = strictParseInt(z); + if (isNaN(intZ)) throw new Error('Invalid value for z'); + const intX = strictParseInt(x); + if (isNaN(intX)) throw new Error('Invalid value for x'); + const intY = strictParseInt(y); + if (isNaN(intY)) throw new Error('Invalid value for y'); - if (isNaN(intX) || isNaN(intY) || isNaN(intZ)) { - console.error('Missing x or y or z') - return { error: 'Bad parameter' } - } - - loadTile(tileset, intZ, intX, intY).then((im) => { - res.writeHead(200, { 'Content-Type': 'image/png' }) - res.end(im) - }).catch((err) => { - console.error(err) - res.status(500).send({ error: err }) - }) -} - -function loadTile(tileset, z, x, y) { - if (outsideExtent(z, x, y)) { - return emptyTile() - } - return get(tileset, z, x, y).then((im) => { - console.log(`From cache ${tileset}/${z}/${x}/${y}`) - return im - }).catch(() => { - return renderOrStitchTile(tileset, z, x, y) - }) -} - -function renderOrStitchTile(tileset, z, x, y) { - if (z <= STITCH_THRESHOLD) { - return StitchTile(tileset, z, x, y).then(im => { - return put(im, tileset, z, x, y).then(() => { - console.log(`Stitch ${tileset}/${z}/${x}/${y}`) - return im - }).catch((err) => { - console.error(err) - return im - }) - }) + let intScale: number; + if (scale === '@2x') { + intScale = 2; + } else if (scale === '@1x' || scale == undefined) { + intScale = 1; } else { - - return new Promise((resolve, reject) => { - renderTile(tileset, z, x, y, undefined, (err, im) => { - if (err) { - reject(err) - return - } - put(im, tileset, z, x, y).then(() => { - console.log(`Render ${tileset}/${z}/${x}/${y}`) - resolve(im) - }).catch((err) => { - console.error(err) - resolve(im) - }) - }) - }) - } -} - -function outsideExtent(z, x, y) { - const xy = getXYZ(EXTENT_BBOX, z); - return xy.minY > y || xy.maxY < y || xy.minX > x || xy.maxX < x; -} - -function emptyTile() { - return sharp({ - create: { - width: 1, - height: 1, - channels: 4, - background: { r: 0, g: 0, b: 0, alpha: 0 } - } - }).png().toBuffer() -} - -function StitchTile(tileset, z, x, y) { - const bbox = getBbox(z, x, y) - const nextZ = z + 1 - const nextXY = getXYZ(bbox, nextZ) - - return Promise.all([ - // recurse down through zoom levels, using cache if available... - loadTile(tileset, nextZ, nextXY.minX, nextXY.minY), - loadTile(tileset, nextZ, nextXY.maxX, nextXY.minY), - loadTile(tileset, nextZ, nextXY.minX, nextXY.maxY), - loadTile(tileset, nextZ, nextXY.maxX, nextXY.maxY) - ]).then(([ - topLeft, - topRight, - bottomLeft, - bottomRight - ]) => { - // not possible to chain overlays in a single pipeline, but there may still be a better - // way to create image buffer here (four tiles resize to one at the next zoom level) - // instead of repeatedly creating `sharp` objects, to png, to buffer... - return sharp({ - create: { - width: TILE_SIZE * 2, - height: TILE_SIZE * 2, - channels: 4, - background: { r: 0, g: 0, b: 0, alpha: 0 } - } - }).overlayWith( - topLeft, { gravity: sharp.gravity.northwest } - ).png().toBuffer().then((buf) => { - return sharp(buf).overlayWith( - topRight, { gravity: sharp.gravity.northeast } - ).png().toBuffer() - }).then((buf) => { - return sharp(buf).overlayWith( - bottomLeft, { gravity: sharp.gravity.southwest } - ).png().toBuffer() - }).then((buf) => { - return sharp(buf).overlayWith( - bottomRight, { gravity: sharp.gravity.southeast } - ).png().toBuffer() - }).then((buf) => { - return sharp(buf - ).resize(TILE_SIZE, TILE_SIZE, { fit: 'inside' } - ).png().toBuffer() - }) - }); -} - -function handleHighlightTileRequest(req, res) { - const { z, x, y } = req.params - const intZ = strictParseInt(z); - const intX = strictParseInt(x); - const intY = strictParseInt(y); - - if (isNaN(intX) || isNaN(intY) || isNaN(intZ)) { - console.error('Missing x or y or z') - return { error: 'Bad parameter' } + throw new Error('Invalid value for scale'); } - // highlight layer uses geometry_id to outline a single building - const { highlight } = req.query - const geometryId = strictParseInt(highlight); - if (isNaN(geometryId)) { - res.status(400).send({ error: 'Bad parameter' }) - return - } - - if (outsideExtent(z, x, y)) { - return emptyTile() - } - - renderTile('highlight', intZ, intX, intY, geometryId, function (err, im) { - if (err) {throw err} - - res.writeHead(200, { 'Content-Type': 'image/png' }) - res.end(im) - }) + return { + tileset, + z: intZ, + x: intX, + y: intY, + scale: intScale + }; } +router.use((req, res) => { + return res.status(404).send('Tile not found'); +}); + export default router;