a_adli/colouring-montreal
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #463 from mz8i/feature/simplify-tile-renderer

Browse Source 
Simplify tile renderer, improve stitching and caching
This commit is contained in:
mz8i 2019-11-07 07:35:19 +00:00 committed by GitHub
commit 24b8b13ad6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 247 additions and 346 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
app/src/tiles/dataDefinition.ts
View File

@ -1,5 +1,5 @@
import { strictParseInt } from "../parse";
import { DataConfig } from "./renderers/datasourceRenderer";
import { DataConfig } from "./types";
const BUILDING_LAYER_DEFINITIONS = {
base_light: `(
@ -111,6 +111,14 @@ const BUILDING_LAYER_DEFINITIONS = {
const GEOMETRY_FIELD = 'geometry_geom';
function getBuildingLayerNames() {
return Object.keys(BUILDING_LAYER_DEFINITIONS);
}
function getAllLayerNames() {
return ['highlight', ...getBuildingLayerNames()];
}
function getBuildingsDataConfig(tileset: string, dataParams: any): DataConfig {
const table = BUILDING_LAYER_DEFINITIONS[tileset];
@ -149,7 +157,8 @@ function getHighlightDataConfig(tileset: string, dataParams: any): DataConfig {
}
export {
BUILDING_LAYER_DEFINITIONS,
getBuildingLayerNames,
getAllLayerNames,
getBuildingsDataConfig,
getHighlightDataConfig
};

109
app/src/tiles/rendererDefinition.ts
View File

@ -1,21 +1,16 @@
import { TileCache } from "./tileCache";
import { BoundingBox, TileParams } from "./types";
import { StitchRenderer } from "./renderers/stitchRenderer";
import { CachedRenderer } from "./renderers/cachedRenderer";
import { BranchingRenderer } from "./renderers/branchingRenderer";
import { WindowedRenderer } from "./renderers/windowedRenderer";
import { BlankRenderer } from "./renderers/blankRenderer";
import { DatasourceRenderer } from "./renderers/datasourceRenderer";
import { getBuildingsDataConfig, getHighlightDataConfig, BUILDING_LAYER_DEFINITIONS } from "./dataDefinition";
import { BoundingBox, TileParams, Tile } from "./types";
import { getBuildingsDataConfig, getHighlightDataConfig, getAllLayerNames, getBuildingLayerNames } from "./dataDefinition";
import { isOutsideExtent } from "./util";
import { renderDataSourceTile } from "./renderers/renderDataSourceTile";
import { getTileWithCaching } from "./renderers/getTileWithCaching";
import { stitchTile } from "./renderers/stitchTile";
import { createBlankTile } from "./renderers/createBlankTile";
/**
* A list of all tilesets handled by the tile server
*/
const allTilesets = ['highlight', ...Object.keys(BUILDING_LAYER_DEFINITIONS)];
const buildingDataRenderer = new DatasourceRenderer(getBuildingsDataConfig);
const stitchRenderer = new StitchRenderer(undefined); // depends recurrently on cache, so parameter will be set later
const allTilesets = getAllLayerNames();
/**
* Zoom level when we switch from rendering direct from database to instead composing tiles
@ -23,56 +18,60 @@ const stitchRenderer = new StitchRenderer(undefined); // depends recurrently on
*/
const STITCH_THRESHOLD = 12;
const renderOrStitchRenderer = new BranchingRenderer(
({ z }) => z <= STITCH_THRESHOLD,
stitchRenderer, // refer to the prepared stitch renderer
buildingDataRenderer
);
const tileCache = new TileCache(
process.env.TILECACHE_PATH,
{
tilesets: ['date_year', 'size_storeys', 'location', 'likes', 'conservation_area', 'sust_dec', 'building_attachment_form'],
minZoom: 9,
maxZoom: 18,
scales: [1, 2]
},
({ tileset, z }: TileParams) => (tileset === 'date_year' && z <= 16) ||
((tileset === 'base_light' || tileset === 'base_night') && z <= 17) ||
z <= 13
);
const cachedRenderer = new CachedRenderer(
tileCache,
renderOrStitchRenderer
);
// set up stitch renderer to use the data renderer with caching
stitchRenderer.tileRenderer = cachedRenderer;
const highlightRenderer = new DatasourceRenderer(getHighlightDataConfig);
const highlightOrBuildingRenderer = new BranchingRenderer(
({ tileset }) => tileset === 'highlight',
highlightRenderer,
cachedRenderer
);
const blankRenderer = new BlankRenderer();
/**
* Hard-code extent so we can short-circuit rendering and return empty/transparent tiles outside the area of interest
* bbox in CRS epsg:3857 in form: [w, s, e, n]
*/
const EXTENT_BBOX: BoundingBox = [-61149.622628, 6667754.851372, 28128.826409, 6744803.375884];
const mainRenderer = new WindowedRenderer(
EXTENT_BBOX,
highlightOrBuildingRenderer,
blankRenderer
const tileCache = new TileCache(
process.env.TILECACHE_PATH,
{
tilesets: getBuildingLayerNames(),
minZoom: 9,
maxZoom: 18,
scales: [1, 2]
},
// cache age data and base building outlines for more zoom levels than other layers
({ tileset, z }: TileParams) => (tileset === 'date_year' && z <= 16) ||
((tileset === 'base_light' || tileset === 'base_night') && z <= 17) ||
z <= 13,
// don't clear base_light and base_night on bounding box cache clear
(tileset: string) => tileset !== 'base_light' && tileset !== 'base_night'
);
const renderBuildingTile = (t: TileParams, d: any) => renderDataSourceTile(t, d, getBuildingsDataConfig);
const renderHighlightTile = (t: TileParams, d: any) => renderDataSourceTile(t, d, getHighlightDataConfig);
function cacheOrCreateBuildingTile(tileParams: TileParams, dataParams: any): Promise<Tile> {
return getTileWithCaching(tileParams, dataParams, tileCache, stitchOrRenderBuildingTile);
}
function stitchOrRenderBuildingTile(tileParams: TileParams, dataParams: any): Promise<Tile> {
if (tileParams.z <= STITCH_THRESHOLD) {
// stitch tile, using cache recursively
return stitchTile(tileParams, dataParams, cacheOrCreateBuildingTile);
} else {
return renderBuildingTile(tileParams, dataParams);
}
}
function renderTile(tileParams: TileParams, dataParams: any): Promise<Tile> {
if (isOutsideExtent(tileParams, EXTENT_BBOX)) {
return createBlankTile();
}
if (tileParams.tileset === 'highlight') {
return renderHighlightTile(tileParams, dataParams);
}
return cacheOrCreateBuildingTile(tileParams, dataParams);
}
export {
allTilesets,
mainRenderer,
renderTile,
tileCache
};

21
app/src/tiles/renderers/blankRenderer.ts
View File

@ -1,21 +0,0 @@
import { Image } from "mapnik";
import sharp from 'sharp';
import { TileParams, TileRenderer } from "../types";
class BlankRenderer implements TileRenderer {
getTile(tileParams: TileParams): Promise<Image> {
return sharp({
create: {
width: 1,
height: 1,
channels: 4,
background: { r: 0, g: 0, b: 0, alpha: 0 }
}
}).png().toBuffer();
}
}
export {
BlankRenderer
};

23
app/src/tiles/renderers/branchingRenderer.ts
View File

@ -1,23 +0,0 @@
import { Image } from "mapnik";
import { TileParams, TileRenderer } from "../types";
class BranchingRenderer {
constructor(
public branchTestFn: (tileParams: TileParams) => boolean,
public trueResultTileRenderer: TileRenderer,
public falseResultTileRenderer: TileRenderer
) {}
getTile(tileParams: TileParams, dataParams: any): Promise<Image> {
if(this.branchTestFn(tileParams)) {
return this.trueResultTileRenderer.getTile(tileParams, dataParams);
} else {
return this.falseResultTileRenderer.getTile(tileParams, dataParams);
}
}
}
export {
BranchingRenderer
};

32
app/src/tiles/renderers/cachedRenderer.ts
View File

@ -1,32 +0,0 @@
import { Image } from "mapnik";
import { TileParams, TileRenderer } from "../types";
import { TileCache } from "../tileCache";
import { formatParams } from "../util";
class CachedRenderer implements TileRenderer {
constructor(
/** Cache to use for tiles */
public tileCache: TileCache,
/** Renderer to use when tile hasn't been cached yet */
public tileRenderer: TileRenderer
) {}
async getTile(tileParams: TileParams, dataParams: any): Promise<Image> {
try {
const tile = await this.tileCache.get(tileParams);
return tile;
} catch(err) {
const im = await this.tileRenderer.getTile(tileParams, dataParams);
try {
await this.tileCache.put(im, tileParams);
} catch (err) {}
return im;
}
}
}
export {
CachedRenderer
};

18
app/src/tiles/renderers/createBlankTile.ts Normal file
View File

@ -0,0 +1,18 @@
import sharp from "sharp";
import { Tile } from "../types";
function createBlankTile(): Promise<Tile> {
return sharp({
create: {
width: 1,
height: 1,
channels: 4,
background: { r: 0, g: 0, b: 0, alpha: 0 }
}
}).png().toBuffer();
}
export {
createBlankTile
};

105
app/src/tiles/renderers/datasourceRenderer.ts
View File

@ -1,105 +0,0 @@
import path from 'path';
import mapnik from "mapnik";
import { TileParams, TileRenderer } from "../types";
import { getBbox, TILE_SIZE } from "../util";
import { promisify } from "util";
interface DataConfig {
table: string;
geometry_field: string;
}
const TILE_BUFFER_SIZE = 64;
const PROJ4_STRING = '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over';
// connection details from environment variables
const DATASOURCE_CONFIG = {
'host': process.env.PGHOST,
'dbname': process.env.PGDATABASE,
'user': process.env.PGUSER,
'password': process.env.PGPASSWORD,
'port': process.env.PGPORT,
'extent': '-20005048.4188,-9039211.13765,19907487.2779,17096598.5401',
'srid': 3857,
'type': 'postgis'
};
// register datasource adapters for mapnik database connection
if (mapnik.register_default_input_plugins) {
mapnik.register_default_input_plugins();
}
// register fonts for text rendering
mapnik.register_default_fonts();
class DatasourceRenderer implements TileRenderer {
constructor(private getTableDefinitionFn: (tileset: string, dataParams: any) => DataConfig) {}
async getTile({tileset, z, x, y, scale}: TileParams, dataParams: any): Promise<mapnik.Image> {
const bbox = getBbox(z, x, y);
const tileSize = TILE_SIZE * scale;
let map = new mapnik.Map(tileSize, tileSize, PROJ4_STRING);
map.bufferSize = TILE_BUFFER_SIZE;
const layer = new mapnik.Layer('tile', PROJ4_STRING);
const dataSourceConfig = this.getTableDefinitionFn(tileset, dataParams);
const conf = Object.assign(dataSourceConfig, DATASOURCE_CONFIG);
const postgis = new mapnik.Datasource(conf);
layer.datasource = postgis;
layer.styles = [tileset];
const stylePath = path.join(__dirname, '..', 'map_styles', 'polygon.xml');
map = await promisify(map.load.bind(map))(stylePath, {strict: true});
map.add_layer(layer);
const im = new mapnik.Image(map.width, map.height);
map.extent = bbox;
const rendered = await promisify(map.render.bind(map))(im, {});
return await promisify(rendered.encode.bind(rendered))('png');
}
}
function promiseHandler(resolve, reject) {
return function(err, result) {
if(err) reject(err);
else resolve(result);
}
}
/**
* Utility function which promisifies a method of an object and binds it to the object
* This makes it easier to use callback-based object methods in a promise-based way
* @param obj Object containing the target method
* @param methodName Method name to promisify and return
*/
function promisifyMethod<T, F>(obj: T, methodName: keyof T);
/**
* @param methodGetter accessor function to get the method from the object
*/
function promisifyMethod<T, S>(obj: T, methodGetter: (o: T) => S);
function promisifyMethod<T, S>(obj: T, paramTwo: keyof T | ((o: T) => S)) {
let method;
if (typeof paramTwo === 'string') {
method = obj[paramTwo];
} else if (typeof paramTwo === 'function') {
method = paramTwo(obj);
}
if (typeof method === 'function') {
return promisify(method.bind(obj));
} else {
throw new Error(`Cannot promisify non-function property '${paramTwo}'`);
}
}
export {
DatasourceRenderer,
DataConfig
};

21
app/src/tiles/renderers/getTileWithCaching.ts Normal file
View File

@ -0,0 +1,21 @@
import { TileParams, RendererFunction, Tile } from "../types";
import { TileCache } from "../tileCache";
async function getTileWithCaching(tileParams: TileParams, dataParams: any, tileCache: TileCache, renderTile: RendererFunction): Promise<Tile> {
try {
const tile = await tileCache.get(tileParams);
return tile;
} catch (err) {
const im = await renderTile(tileParams, dataParams);
try {
await tileCache.put(im, tileParams);
} catch (err) {}
return im;
}
}
export {
getTileWithCaching
};

64
app/src/tiles/renderers/renderDataSourceTile.ts Normal file
View File

@ -0,0 +1,64 @@
import path from 'path';
import mapnik from "mapnik";
import { TileParams, Tile, TableDefinitionFunction } from "../types";
import { getBbox, TILE_SIZE } from "../util";
import { promisify } from "util";
const TILE_BUFFER_SIZE = 64;
const PROJ4_STRING = '+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over';
// connection details from environment variables
const DATASOURCE_CONFIG = {
'host': process.env.PGHOST,
'dbname': process.env.PGDATABASE,
'user': process.env.PGUSER,
'password': process.env.PGPASSWORD,
'port': process.env.PGPORT,
'extent': '-20005048.4188,-9039211.13765,19907487.2779,17096598.5401',
'srid': 3857,
'type': 'postgis'
};
// register datasource adapters for mapnik database connection
if (mapnik.register_default_input_plugins) {
mapnik.register_default_input_plugins();
}
// register fonts for text rendering
mapnik.register_default_fonts();
async function renderDataSourceTile({ tileset, z, x, y, scale }: TileParams, dataParams: any, getTableDefinitionFn: TableDefinitionFunction): Promise<Tile> {
const bbox = getBbox(z, x, y);
const tileSize = TILE_SIZE * scale;
let map = new mapnik.Map(tileSize, tileSize, PROJ4_STRING);
map.bufferSize = TILE_BUFFER_SIZE;
const layer = new mapnik.Layer('tile', PROJ4_STRING);
const dataSourceConfig = getTableDefinitionFn(tileset, dataParams);
const conf = Object.assign(dataSourceConfig, DATASOURCE_CONFIG);
const postgis = new mapnik.Datasource(conf);
layer.datasource = postgis;
layer.styles = [tileset];
const stylePath = path.join(__dirname, '..', 'map_styles', 'polygon.xml');
map = await promisify(map.load.bind(map))(stylePath, { strict: true });
map.add_layer(layer);
const im = new mapnik.Image(map.width, map.height);
map.extent = bbox;
const rendered = await promisify(map.render.bind(map))(im, {});
return await promisify(rendered.encode.bind(rendered))('png');
}
export {
renderDataSourceTile
};

67
app/src/tiles/renderers/stitchRenderer.ts
View File

@ -1,67 +0,0 @@
import sharp from 'sharp';
import { Image } from 'mapnik';
import { TileParams, TileRenderer } from "../types";
import { getBbox, getXYZ, TILE_SIZE, formatParams } from "../util";
class StitchRenderer implements TileRenderer {
constructor(
/** Renderer to use when retrieving tiles to be stitched together */
public tileRenderer: TileRenderer
) {}
getTile(tileParams: TileParams, dataParams: any): Promise<Image> {
console.log(`Stitching tile ${formatParams(tileParams)}`);
return this.stitchTile(tileParams, dataParams, this.tileRenderer);
}
private async stitchTile({ tileset, z, x, y, scale }: TileParams, dataParams: any, tileRenderer: TileRenderer) {
const bbox = getBbox(z, x, y);
const nextZ = z + 1;
const nextXY = getXYZ(bbox, nextZ);
const tileSize = TILE_SIZE * scale;
const [topLeft, topRight, bottomLeft, bottomRight] = await Promise.all([
[nextXY.minX, nextXY.minY],
[nextXY.maxX, nextXY.minY],
[nextXY.minX, nextXY.maxY],
[nextXY.maxX, nextXY.maxY]
].map(([x, y]) => tileRenderer.getTile({ tileset, z: nextZ, x, y, scale }, dataParams)));
// not possible to chain overlays in a single pipeline, but there may still be a better
// way to create image buffer here (four tiles resize to one at the next zoom level)
// instead of repeatedly creating `sharp` objects, to png, to buffer...
return sharp({
create: {
width: tileSize * 2,
height: tileSize * 2,
channels: 4,
background: { r: 0, g: 0, b: 0, alpha: 0 }
}
}).overlayWith(
topLeft, { gravity: sharp.gravity.northwest }
).png().toBuffer().then((buf) => {
return sharp(buf).overlayWith(
topRight, { gravity: sharp.gravity.northeast }
).png().toBuffer()
}).then((buf) => {
return sharp(buf).overlayWith(
bottomLeft, { gravity: sharp.gravity.southwest }
).png().toBuffer()
}).then((buf) => {
return sharp(buf).overlayWith(
bottomRight, { gravity: sharp.gravity.southeast }
).png().toBuffer()
}).then((buf) => {
return sharp(buf
).resize(tileSize, tileSize, { fit: 'inside' }
).png().toBuffer()
})
}
}
export {
StitchRenderer
};

43
app/src/tiles/renderers/stitchTile.ts Normal file
View File

@ -0,0 +1,43 @@
import sharp from 'sharp';
import { TileParams, RendererFunction, Tile } from "../types";
import { getBbox, getXYZ, TILE_SIZE } from "../util";
async function stitchTile({ tileset, z, x, y, scale }: TileParams, dataParams: any, renderTile: RendererFunction): Promise<Tile> {
const bbox = getBbox(z, x, y);
const nextZ = z + 1;
const nextXY = getXYZ(bbox, nextZ);
const tileSize = TILE_SIZE * scale;
const [topLeft, topRight, bottomLeft, bottomRight] = await Promise.all([
[nextXY.minX, nextXY.minY],
[nextXY.maxX, nextXY.minY],
[nextXY.minX, nextXY.maxY],
[nextXY.maxX, nextXY.maxY]
].map(([x, y]) => renderTile({ tileset, z: nextZ, x, y, scale }, dataParams)));
const compositedBuffer = await sharp({
create: {
width: tileSize * 2,
height: tileSize * 2,
channels: 4,
background: { r: 0, g: 0, b: 0, alpha: 0 }
}
}).composite([
{input: topLeft, gravity: sharp.gravity.northwest},
{input: topRight, gravity: sharp.gravity.northeast},
{input: bottomLeft, gravity: sharp.gravity.southwest},
{input: bottomRight, gravity: sharp.gravity.southeast}
]).png().toBuffer();
return sharp(compositedBuffer)
.resize(tileSize, tileSize, {fit: 'inside'})
.png()
.toBuffer();
}
export {
stitchTile
};

34
app/src/tiles/renderers/windowedRenderer.ts
View File

@ -1,34 +0,0 @@
import { Image } from "mapnik";
import { BoundingBox, TileParams, TileRenderer } from "../types";
import { getXYZ } from "../util";
class WindowedRenderer implements TileRenderer {
constructor(
/** Bounding box defining the renderer window */
public bbox: BoundingBox,
/** Renderer to use for tile requests inside window */
public insideWindowRenderer: TileRenderer,
/** Renderer to use for tile requests outside window */
public outsideWindowRenderer: TileRenderer
) {}
getTile(tileParams: TileParams, dataParams: any): Promise<Image> {
if(this.isOutsideExtent(tileParams)) {
return this.outsideWindowRenderer.getTile(tileParams, dataParams);
} else {
return this.insideWindowRenderer.getTile(tileParams, dataParams);
}
}
private isOutsideExtent({x, y, z}: TileParams) {
const xy = getXYZ(this.bbox, z);
return xy.minY > y || xy.maxY < y || xy.minX > x || xy.maxX < x;
}
}
export {
WindowedRenderer
};

10
app/src/tiles/tileCache.ts
View File

@ -70,7 +70,9 @@ class TileCache {
/** Domain definition for the cache */
private cacheDomain: CacheDomain,
/** Function for defining custom caching rules (optional) */
private shouldCacheFn?: (TileParams) => boolean
private shouldCacheFn?: (TileParams) => boolean,
/** Function for defining whether the tileset should be cleared when clearing cache for bounding box */
private shouldBulkClearTilesetFn?: (tileset: string) => boolean
) {}
async get(tileParams: TileParams): Promise<Image> {
@ -108,6 +110,8 @@ class TileCache {
async removeAllAtBbox(bbox: BoundingBox): Promise<void[]> {
const removePromises: Promise<void>[] = [];
for (const tileset of this.cacheDomain.tilesets) {
if(!this.shouldBulkClearTileset(tileset)) continue;
for (let z = this.cacheDomain.minZoom; z <= this.cacheDomain.maxZoom; z++) {
let tileBounds = getXYZ(bbox, z)
for (let x = tileBounds.minX; x <= tileBounds.maxX; x++) {
@ -137,6 +141,10 @@ class TileCache {
this.cacheDomain.scales.includes(tileParams.scale) &&
(this.shouldCacheFn == undefined || this.shouldCacheFn(tileParams));
}
private shouldBulkClearTileset(tileset: string): boolean {
return this.shouldCacheFn == undefined || this.shouldBulkClearTilesetFn(tileset);
}
}
export {

4
app/src/tiles/tileserver.ts
View File

@ -7,7 +7,7 @@ import express from 'express';
import { strictParseInt } from '../parse';
import { TileParams } from './types';
import { mainRenderer, allTilesets } from './rendererDefinition';
import { renderTile, allTilesets } from './rendererDefinition';
import asyncController from '../api/routes/asyncController';
const handleTileRequest = asyncController(async function (req: express.Request, res: express.Response) {
@ -20,7 +20,7 @@ const handleTileRequest = asyncController(async function (req: express.Request,
}
try {
const im = await mainRenderer.getTile(tileParams, dataParams);
const im = await renderTile(tileParams, dataParams);
res.writeHead(200, { 'Content-Type': 'image/png' });
res.end(im);
} catch(err) {

19
app/src/tiles/types.ts
View File

@ -1,4 +1,5 @@
import { Image } from 'mapnik';
import { Sharp } from 'sharp';
/**
* Bounding box in the format [w, s, e, n]
@ -32,12 +33,26 @@ interface TileParams {
scale: number;
}
interface DataConfig {
table: string;
geometry_field: string;
}
type TableDefinitionFunction = (tileset: string, dataParams: any) => DataConfig;
type Tile = Image | Sharp;
type RendererFunction = (tileParams: TileParams, dataParams: any) => Promise<Tile>;
interface TileRenderer {
getTile(tileParams: TileParams, dataParams: any): Promise<Image>
getTile: RendererFunction
}
export {
BoundingBox,
TileParams,
TileRenderer
TileRenderer,
Tile,
RendererFunction,
DataConfig,
TableDefinitionFunction
};

10
app/src/tiles/util.ts
View File

@ -1,6 +1,6 @@
import SphericalMercator from '@mapbox/sphericalmercator';
import { TileParams } from './types';
import { TileParams, BoundingBox } from './types';
const TILE_SIZE = 256;
@ -20,9 +20,15 @@ function formatParams({ tileset, z, x, y, scale }: TileParams): string {
return `${tileset}/${z}/${x}/${y}@${scale}x`;
}
function isOutsideExtent({ x, y, z }: TileParams, bbox: BoundingBox) {
const xy = getXYZ(bbox, z);
return xy.minY > y || xy.maxY < y || xy.minX > x || xy.maxX < x;
}
export {
TILE_SIZE,
getBbox,
getXYZ,
formatParams
formatParams,
isOutsideExtent
};