Merge pull request #257 from tomalrussell/fix/clear_map_cache

Fix/clear map cache
This commit is contained in:
Tom Russell 2019-05-02 18:54:12 +01:00 committed by GitHub
commit 75ed2219cd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 226 additions and 65 deletions

View File

@ -3,6 +3,8 @@
*
*/
import db from '../db';
import { remove_all_at_bbox } from '../tiles/cache';
// data type note: PostgreSQL bigint (64-bit) is handled as string in JavaScript, because of
// JavaScript numerics are 64-bit double, giving only partial coverage.
@ -128,7 +130,7 @@ function saveBuilding(building_id, building, user_id) {
[building_id]
).then(old_building => {
const patches = compare(old_building, building, BUILDING_FIELD_WHITELIST);
console.log("Patching", patches)
console.log("Patching", building_id, patches)
const forward = patches[0];
const reverse = patches[1];
if (Object.keys(forward).length === 0) {
@ -144,7 +146,7 @@ function saveBuilding(building_id, building, user_id) {
[forward, reverse, building_id, user_id]
).then(revision => {
const sets = db.$config.pgp.helpers.sets(forward);
console.log("Setting", sets)
console.log("Setting", building_id, sets)
return t.one(
`UPDATE
buildings
@ -157,7 +159,10 @@ function saveBuilding(building_id, building, user_id) {
*
`,
[revision.log_id, sets, building_id]
)
).then((data) => {
expireBuildingTileCache(building_id)
return data
})
});
});
}).catch(function (error) {
@ -203,7 +208,10 @@ function likeBuilding(building_id, user_id) {
*
`,
[revision.log_id, building.likes, building_id]
)
).then((data) => {
expireBuildingTileCache(building_id)
return data
})
})
});
});
@ -255,7 +263,10 @@ function unlikeBuilding(building_id, user_id) {
*
`,
[revision.log_id, building.likes, building_id]
)
).then((data) => {
expireBuildingTileCache(building_id)
return data
})
})
});
});
@ -270,6 +281,33 @@ function unlikeBuilding(building_id, user_id) {
});
}
function privateQueryBuildingBBOX(building_id){
return db.one(
`SELECT
ST_XMin(envelope) as xmin,
ST_YMin(envelope) as ymin,
ST_XMax(envelope) as xmax,
ST_YMax(envelope) as ymax
FROM (
SELECT
ST_Envelope(g.geometry_geom) as envelope
FROM buildings as b, geometries as g
WHERE
b.geometry_id = g.geometry_id
AND
b.building_id = $1
) as envelope`,
[building_id]
)
}
function expireBuildingTileCache(building_id) {
privateQueryBuildingBBOX(building_id).then((bbox) => {
const building_bbox = [bbox.xmax, bbox.ymax, bbox.xmin, bbox.ymin]
remove_all_at_bbox(building_bbox);
})
}
const BUILDING_FIELD_WHITELIST = new Set([
'ref_osm_id',
// 'location_name',

View File

@ -2,9 +2,11 @@
* Load server and listen
*
*/
import app from './server';
import http from 'http';
import app from './server';
const server = http.createServer(app);
let currentApp = app;
@ -17,6 +19,7 @@ server.listen(process.env.PORT || 3000, error => {
console.log('🚀 started');
});
// In development mode, enable hot module reloading (HMR)
if (module.hot) {
console.log('✅ Server-side HMR Enabled!');

View File

@ -1,11 +1,13 @@
/**
* Utility functions for parsing
*
*/
/**
* Parse a string as positive integer or NaN
*
* @param {string} value
* @returns {number} integer or NaN
*/
function strictParseInt(value) {
if (/^([1-9][0-9]*)$/.test(value))
@ -13,7 +15,12 @@ function strictParseInt(value) {
return NaN;
}
/**
* Parse building ID from URL
*
* @param {String} url
* @returns {number|undefined}
*/
function parseBuildingURL(url) {
const re = /\/building\/([^/]+).html/;
const matches = re.exec(url);
@ -24,6 +31,12 @@ function parseBuildingURL(url) {
return undefined;
}
/**
* Parse category slug from URL
*
* @param {String} url
* @returns {String} [age]
*/
function parseCategoryURL(url) {
const default_cat = 'age';
if (url === "/") {

View File

@ -18,40 +18,143 @@
// and then use stdlib `import fs from 'fs';`
import fs from 'node-fs';
import { get_xyz } from './tile';
// Use an environment variable to configure the cache location, somewhere we can read/write to.
const CACHE_PATH = process.env.TILECACHE_PATH
function get(tileset, z, x, y, cb) {
/**
* Get a tile from the cache
*
* @param {String} tileset
* @param {number} z zoom level
* @param {number} x
* @param {number} y
*/
function get(tileset, z, x, y) {
if (!should_try_cache(tileset, z)) {
cb(`Skip cache get ${tileset}/${z}/${x}/${y}`, null)
return
return Promise.reject(`Skip cache get ${tileset}/${z}/${x}/${y}`);
}
const dir = `${CACHE_PATH}/${tileset}/${z}/${x}`
const fname = `${dir}/${y}.png`
fs.readFile(fname, cb)
}
function put(im, tileset, z, x, y, cb) {
if (!should_try_cache(tileset, z)) {
cb(`Skip cache put ${tileset}/${z}/${x}/${y}`)
return
}
const dir = `${CACHE_PATH}/${tileset}/${z}/${x}`
const fname = `${dir}/${y}.png`
fs.writeFile(fname, im, 'binary', (err) => {
if (err && err.code === 'ENOENT') {
fs.mkdir(dir, 0o755, true, (err) => {
if (err) {
cb(err);
} else {
fs.writeFile(fname, im, 'binary', cb);
}
});
} else {
cb(err)
}
const location = cache_location(tileset, z, x, y);
return new Promise((resolve, reject) => {
fs.readFile(location.fname, (err, data) => {
if (err) {
reject(err)
} else {
resolve(data)
}
})
});
}
/**
* Put a tile in the cache
*
* @param {Buffer} im image data
* @param {String} tileset
* @param {number} z zoom level
* @param {number} x
* @param {number} y
*/
function put(im, tileset, z, x, y) {
if (!should_try_cache(tileset, z)) {
return Promise.reject(`Skip cache put ${tileset}/${z}/${x}/${y}`);
}
const location = cache_location(tileset, z, x, y);
return new Promise((resolve, reject) => {
fs.writeFile(location.fname, im, 'binary', (err) => {
if (err && err.code === 'ENOENT') {
// recursively create tile directory if it didn't previously exist
fs.mkdir(location.dir, 0o755, true, (err) => {
if (err) {
reject(err);
} else {
// then write the file
fs.writeFile(location.fname, im, 'binary', (err) => {
(err)? reject(err): resolve()
});
}
});
} else {
(err)? reject(err): resolve()
}
});
})
}
/**
* Remove a single cached tile
*
* @param {String} tileset
* @param {number} z zoom level
* @param {number} x
* @param {number} y
*/
function remove(tileset, z, x, y) {
const location = cache_location(tileset, z, x, y)
return new Promise(resolve => {
fs.unlink(location.fname, (err) => {
if(err){
// pass
} else {
console.log("Expire cache", tileset, z, x, y)
}
resolve()
})
})
}
/**
* Remove all cached data-visualising tiles which intersect a bbox
* - initially called directly after edits; may be better on a worker process?
*
* @param {String} tileset
* @param {Array} bbox [w, s, e, n] in EPSG:3857 coordinates
*/
function remove_all_at_bbox(bbox) {
// magic numbers for min/max zoom
const min_zoom = 9;
const max_zoom = 18;
// magic list of tilesets - see tileserver, other cache rules
const tilesets = ['date_year', 'size_storeys', 'location', 'likes', 'conservation_area'];
let tile_bounds;
const remove_promises = [];
for (let ti = 0; ti < tilesets.length; ti++) {
const tileset = tilesets[ti];
for (let z = min_zoom; z <= max_zoom; z++) {
tile_bounds = get_xyz(bbox, z)
for (let x = tile_bounds.minX; x <= tile_bounds.maxX; x++){
for (let y = tile_bounds.minY; y <= tile_bounds.maxY; y++){
remove_promises.push(remove(tileset, z, x, y))
}
}
}
}
Promise.all(remove_promises)
}
/**
* Cache location for a tile
*
* @param {String} tileset
* @param {number} z zoom level
* @param {number} x
* @param {number} y
* @returns {object} { dir: <directory>, fname: <full filepath> }
*/
function cache_location(tileset, z, x, y) {
const dir = `${CACHE_PATH}/${tileset}/${z}/${x}`
const fname = `${dir}/${y}.png`
return {dir, fname}
}
/**
* Check rules for caching tiles
*
* @param {String} tileset
* @param {number} z zoom level
* @returns {boolean} whether to use the cache (or not)
*/
function should_try_cache(tileset, z) {
if (tileset === 'date_year') {
// cache high zoom because of front page hits
@ -65,4 +168,4 @@ function should_try_cache(tileset, z) {
return z <= 13
}
export { get, put };
export { get, put, remove, remove_all_at_bbox };

View File

@ -1,6 +1,14 @@
/**
* Render tiles
*
* Use mapnik to render map tiles from the database
*
* Styles have two sources of truth for colour ranges (could generate from single source?)
* - XML style definitions in app/map_styles/polygon.xml
* - front-end legend in app/src/frontend/legend.js
*
* Data is provided by the queries in MAP_STYLE_TABLE_DEFINITIONS below.
*
*/
import path from 'path';
import mapnik from 'mapnik';

View File

@ -1,5 +1,8 @@
/**
* Tileserver routes for Express app
* Tileserver
* - routes for Express app
* - stitch tiles above a certain zoom level (compositing from sharply-rendered lower zooms)
* - render empty tile outside extent of geographical area of interest
*
*/
import express from 'express';
@ -74,33 +77,23 @@ function load_tile(tileset, z, x, y) {
if (outside_extent(z, x, y)) {
return empty_tile()
}
return new Promise((resolve) => {
get(tileset, z, x, y, (err, im) => {
if (err) {
render_or_stitch_tile(tileset, z, x, y)
.then((im) => {
resolve(im)
})
} else {
console.log(`From cache ${tileset}/${z}/${x}/${y}`)
resolve(im)
}
})
return get(tileset, z, x, y).then((im) => {
console.log(`From cache ${tileset}/${z}/${x}/${y}`)
return im
}).catch((_) => {
return render_or_stitch_tile(tileset, z, x, y)
})
}
function render_or_stitch_tile(tileset, z, x, y) {
if (z <= STITCH_THRESHOLD) {
return stitch_tile(tileset, z, x, y).then(im => {
return new Promise((resolve, reject) => {
put(im, tileset, z, x, y, (err) => {
if (err) {
console.error(err)
} else {
console.log(`Stitch ${tileset}/${z}/${x}/${y}`)
}
resolve(im)
})
return put(im, tileset, z, x, y).then(() => {
console.log(`Stitch ${tileset}/${z}/${x}/${y}`)
return im
}).catch((err) => {
console.error(err)
return im
})
})
} else {
@ -111,12 +104,11 @@ function render_or_stitch_tile(tileset, z, x, y) {
reject(err)
return
}
put(im, tileset, z, x, y, (err) => {
if (err) {
console.error(err)
} else {
console.log(`Render ${tileset}/${z}/${x}/${y}`)
}
put(im, tileset, z, x, y).then(() => {
console.log(`Render ${tileset}/${z}/${x}/${y}`)
resolve(im)
}).catch((err) => {
console.error(err)
resolve(im)
})
})
@ -132,8 +124,8 @@ function outside_extent(z, x, y) {
function empty_tile() {
return sharp({
create: {
width: TILE_SIZE,
height: TILE_SIZE,
width: 1,
height: 1,
channels: 4,
background: { r: 0, g: 0, b: 0, alpha: 0 }
}
@ -209,6 +201,10 @@ function handle_highlight_tile_request(req, res) {
return
}
if (outside_extent(z, x, y)) {
return empty_tile()
}
render_tile('highlight', int_z, int_x, int_y, geometry_id, function (err, im) {
if (err) throw err