Merge pull request #257 from tomalrussell/fix/clear_map_cache
Fix/clear map cache
This commit is contained in:
commit
75ed2219cd
@ -3,6 +3,8 @@
|
||||
*
|
||||
*/
|
||||
import db from '../db';
|
||||
import { remove_all_at_bbox } from '../tiles/cache';
|
||||
|
||||
// data type note: PostgreSQL bigint (64-bit) is handled as string in JavaScript, because of
|
||||
// JavaScript numerics are 64-bit double, giving only partial coverage.
|
||||
|
||||
@ -128,7 +130,7 @@ function saveBuilding(building_id, building, user_id) {
|
||||
[building_id]
|
||||
).then(old_building => {
|
||||
const patches = compare(old_building, building, BUILDING_FIELD_WHITELIST);
|
||||
console.log("Patching", patches)
|
||||
console.log("Patching", building_id, patches)
|
||||
const forward = patches[0];
|
||||
const reverse = patches[1];
|
||||
if (Object.keys(forward).length === 0) {
|
||||
@ -144,7 +146,7 @@ function saveBuilding(building_id, building, user_id) {
|
||||
[forward, reverse, building_id, user_id]
|
||||
).then(revision => {
|
||||
const sets = db.$config.pgp.helpers.sets(forward);
|
||||
console.log("Setting", sets)
|
||||
console.log("Setting", building_id, sets)
|
||||
return t.one(
|
||||
`UPDATE
|
||||
buildings
|
||||
@ -157,7 +159,10 @@ function saveBuilding(building_id, building, user_id) {
|
||||
*
|
||||
`,
|
||||
[revision.log_id, sets, building_id]
|
||||
)
|
||||
).then((data) => {
|
||||
expireBuildingTileCache(building_id)
|
||||
return data
|
||||
})
|
||||
});
|
||||
});
|
||||
}).catch(function (error) {
|
||||
@ -203,7 +208,10 @@ function likeBuilding(building_id, user_id) {
|
||||
*
|
||||
`,
|
||||
[revision.log_id, building.likes, building_id]
|
||||
)
|
||||
).then((data) => {
|
||||
expireBuildingTileCache(building_id)
|
||||
return data
|
||||
})
|
||||
})
|
||||
});
|
||||
});
|
||||
@ -255,7 +263,10 @@ function unlikeBuilding(building_id, user_id) {
|
||||
*
|
||||
`,
|
||||
[revision.log_id, building.likes, building_id]
|
||||
)
|
||||
).then((data) => {
|
||||
expireBuildingTileCache(building_id)
|
||||
return data
|
||||
})
|
||||
})
|
||||
});
|
||||
});
|
||||
@ -270,6 +281,33 @@ function unlikeBuilding(building_id, user_id) {
|
||||
});
|
||||
}
|
||||
|
||||
function privateQueryBuildingBBOX(building_id){
|
||||
return db.one(
|
||||
`SELECT
|
||||
ST_XMin(envelope) as xmin,
|
||||
ST_YMin(envelope) as ymin,
|
||||
ST_XMax(envelope) as xmax,
|
||||
ST_YMax(envelope) as ymax
|
||||
FROM (
|
||||
SELECT
|
||||
ST_Envelope(g.geometry_geom) as envelope
|
||||
FROM buildings as b, geometries as g
|
||||
WHERE
|
||||
b.geometry_id = g.geometry_id
|
||||
AND
|
||||
b.building_id = $1
|
||||
) as envelope`,
|
||||
[building_id]
|
||||
)
|
||||
}
|
||||
|
||||
function expireBuildingTileCache(building_id) {
|
||||
privateQueryBuildingBBOX(building_id).then((bbox) => {
|
||||
const building_bbox = [bbox.xmax, bbox.ymax, bbox.xmin, bbox.ymin]
|
||||
remove_all_at_bbox(building_bbox);
|
||||
})
|
||||
}
|
||||
|
||||
const BUILDING_FIELD_WHITELIST = new Set([
|
||||
'ref_osm_id',
|
||||
// 'location_name',
|
||||
|
@ -2,9 +2,11 @@
|
||||
* Load server and listen
|
||||
*
|
||||
*/
|
||||
import app from './server';
|
||||
import http from 'http';
|
||||
|
||||
import app from './server';
|
||||
|
||||
|
||||
const server = http.createServer(app);
|
||||
|
||||
let currentApp = app;
|
||||
@ -17,6 +19,7 @@ server.listen(process.env.PORT || 3000, error => {
|
||||
console.log('🚀 started');
|
||||
});
|
||||
|
||||
// In development mode, enable hot module reloading (HMR)
|
||||
if (module.hot) {
|
||||
console.log('✅ Server-side HMR Enabled!');
|
||||
|
||||
|
@ -1,11 +1,13 @@
|
||||
/**
|
||||
* Utility functions for parsing
|
||||
*
|
||||
*/
|
||||
|
||||
/**
|
||||
* Parse a string as positive integer or NaN
|
||||
*
|
||||
* @param {string} value
|
||||
* @returns {number} integer or NaN
|
||||
*/
|
||||
function strictParseInt(value) {
|
||||
if (/^([1-9][0-9]*)$/.test(value))
|
||||
@ -13,7 +15,12 @@ function strictParseInt(value) {
|
||||
return NaN;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Parse building ID from URL
|
||||
*
|
||||
* @param {String} url
|
||||
* @returns {number|undefined}
|
||||
*/
|
||||
function parseBuildingURL(url) {
|
||||
const re = /\/building\/([^/]+).html/;
|
||||
const matches = re.exec(url);
|
||||
@ -24,6 +31,12 @@ function parseBuildingURL(url) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse category slug from URL
|
||||
*
|
||||
* @param {String} url
|
||||
* @returns {String} [age]
|
||||
*/
|
||||
function parseCategoryURL(url) {
|
||||
const default_cat = 'age';
|
||||
if (url === "/") {
|
||||
|
@ -18,40 +18,143 @@
|
||||
// and then use stdlib `import fs from 'fs';`
|
||||
import fs from 'node-fs';
|
||||
|
||||
import { get_xyz } from './tile';
|
||||
|
||||
// Use an environment variable to configure the cache location, somewhere we can read/write to.
|
||||
const CACHE_PATH = process.env.TILECACHE_PATH
|
||||
|
||||
function get(tileset, z, x, y, cb) {
|
||||
/**
|
||||
* Get a tile from the cache
|
||||
*
|
||||
* @param {String} tileset
|
||||
* @param {number} z zoom level
|
||||
* @param {number} x
|
||||
* @param {number} y
|
||||
*/
|
||||
function get(tileset, z, x, y) {
|
||||
if (!should_try_cache(tileset, z)) {
|
||||
cb(`Skip cache get ${tileset}/${z}/${x}/${y}`, null)
|
||||
return
|
||||
return Promise.reject(`Skip cache get ${tileset}/${z}/${x}/${y}`);
|
||||
}
|
||||
const dir = `${CACHE_PATH}/${tileset}/${z}/${x}`
|
||||
const fname = `${dir}/${y}.png`
|
||||
fs.readFile(fname, cb)
|
||||
}
|
||||
|
||||
function put(im, tileset, z, x, y, cb) {
|
||||
if (!should_try_cache(tileset, z)) {
|
||||
cb(`Skip cache put ${tileset}/${z}/${x}/${y}`)
|
||||
return
|
||||
}
|
||||
const dir = `${CACHE_PATH}/${tileset}/${z}/${x}`
|
||||
const fname = `${dir}/${y}.png`
|
||||
fs.writeFile(fname, im, 'binary', (err) => {
|
||||
if (err && err.code === 'ENOENT') {
|
||||
fs.mkdir(dir, 0o755, true, (err) => {
|
||||
if (err) {
|
||||
cb(err);
|
||||
} else {
|
||||
fs.writeFile(fname, im, 'binary', cb);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
cb(err)
|
||||
}
|
||||
const location = cache_location(tileset, z, x, y);
|
||||
return new Promise((resolve, reject) => {
|
||||
fs.readFile(location.fname, (err, data) => {
|
||||
if (err) {
|
||||
reject(err)
|
||||
} else {
|
||||
resolve(data)
|
||||
}
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Put a tile in the cache
|
||||
*
|
||||
* @param {Buffer} im image data
|
||||
* @param {String} tileset
|
||||
* @param {number} z zoom level
|
||||
* @param {number} x
|
||||
* @param {number} y
|
||||
*/
|
||||
function put(im, tileset, z, x, y) {
|
||||
if (!should_try_cache(tileset, z)) {
|
||||
return Promise.reject(`Skip cache put ${tileset}/${z}/${x}/${y}`);
|
||||
}
|
||||
const location = cache_location(tileset, z, x, y);
|
||||
return new Promise((resolve, reject) => {
|
||||
fs.writeFile(location.fname, im, 'binary', (err) => {
|
||||
if (err && err.code === 'ENOENT') {
|
||||
// recursively create tile directory if it didn't previously exist
|
||||
fs.mkdir(location.dir, 0o755, true, (err) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
// then write the file
|
||||
fs.writeFile(location.fname, im, 'binary', (err) => {
|
||||
(err)? reject(err): resolve()
|
||||
});
|
||||
}
|
||||
});
|
||||
} else {
|
||||
(err)? reject(err): resolve()
|
||||
}
|
||||
});
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a single cached tile
|
||||
*
|
||||
* @param {String} tileset
|
||||
* @param {number} z zoom level
|
||||
* @param {number} x
|
||||
* @param {number} y
|
||||
*/
|
||||
function remove(tileset, z, x, y) {
|
||||
const location = cache_location(tileset, z, x, y)
|
||||
return new Promise(resolve => {
|
||||
fs.unlink(location.fname, (err) => {
|
||||
if(err){
|
||||
// pass
|
||||
} else {
|
||||
console.log("Expire cache", tileset, z, x, y)
|
||||
}
|
||||
resolve()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove all cached data-visualising tiles which intersect a bbox
|
||||
* - initially called directly after edits; may be better on a worker process?
|
||||
*
|
||||
* @param {String} tileset
|
||||
* @param {Array} bbox [w, s, e, n] in EPSG:3857 coordinates
|
||||
*/
|
||||
function remove_all_at_bbox(bbox) {
|
||||
// magic numbers for min/max zoom
|
||||
const min_zoom = 9;
|
||||
const max_zoom = 18;
|
||||
// magic list of tilesets - see tileserver, other cache rules
|
||||
const tilesets = ['date_year', 'size_storeys', 'location', 'likes', 'conservation_area'];
|
||||
let tile_bounds;
|
||||
const remove_promises = [];
|
||||
for (let ti = 0; ti < tilesets.length; ti++) {
|
||||
const tileset = tilesets[ti];
|
||||
for (let z = min_zoom; z <= max_zoom; z++) {
|
||||
tile_bounds = get_xyz(bbox, z)
|
||||
for (let x = tile_bounds.minX; x <= tile_bounds.maxX; x++){
|
||||
for (let y = tile_bounds.minY; y <= tile_bounds.maxY; y++){
|
||||
remove_promises.push(remove(tileset, z, x, y))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Promise.all(remove_promises)
|
||||
}
|
||||
|
||||
/**
|
||||
* Cache location for a tile
|
||||
*
|
||||
* @param {String} tileset
|
||||
* @param {number} z zoom level
|
||||
* @param {number} x
|
||||
* @param {number} y
|
||||
* @returns {object} { dir: <directory>, fname: <full filepath> }
|
||||
*/
|
||||
function cache_location(tileset, z, x, y) {
|
||||
const dir = `${CACHE_PATH}/${tileset}/${z}/${x}`
|
||||
const fname = `${dir}/${y}.png`
|
||||
return {dir, fname}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check rules for caching tiles
|
||||
*
|
||||
* @param {String} tileset
|
||||
* @param {number} z zoom level
|
||||
* @returns {boolean} whether to use the cache (or not)
|
||||
*/
|
||||
function should_try_cache(tileset, z) {
|
||||
if (tileset === 'date_year') {
|
||||
// cache high zoom because of front page hits
|
||||
@ -65,4 +168,4 @@ function should_try_cache(tileset, z) {
|
||||
return z <= 13
|
||||
}
|
||||
|
||||
export { get, put };
|
||||
export { get, put, remove, remove_all_at_bbox };
|
||||
|
@ -1,6 +1,14 @@
|
||||
/**
|
||||
* Render tiles
|
||||
*
|
||||
* Use mapnik to render map tiles from the database
|
||||
*
|
||||
* Styles have two sources of truth for colour ranges (could generate from single source?)
|
||||
* - XML style definitions in app/map_styles/polygon.xml
|
||||
* - front-end legend in app/src/frontend/legend.js
|
||||
*
|
||||
* Data is provided by the queries in MAP_STYLE_TABLE_DEFINITIONS below.
|
||||
*
|
||||
*/
|
||||
import path from 'path';
|
||||
import mapnik from 'mapnik';
|
||||
|
@ -1,5 +1,8 @@
|
||||
/**
|
||||
* Tileserver routes for Express app
|
||||
* Tileserver
|
||||
* - routes for Express app
|
||||
* - stitch tiles above a certain zoom level (compositing from sharply-rendered lower zooms)
|
||||
* - render empty tile outside extent of geographical area of interest
|
||||
*
|
||||
*/
|
||||
import express from 'express';
|
||||
@ -74,33 +77,23 @@ function load_tile(tileset, z, x, y) {
|
||||
if (outside_extent(z, x, y)) {
|
||||
return empty_tile()
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
get(tileset, z, x, y, (err, im) => {
|
||||
if (err) {
|
||||
render_or_stitch_tile(tileset, z, x, y)
|
||||
.then((im) => {
|
||||
resolve(im)
|
||||
})
|
||||
} else {
|
||||
console.log(`From cache ${tileset}/${z}/${x}/${y}`)
|
||||
resolve(im)
|
||||
}
|
||||
})
|
||||
return get(tileset, z, x, y).then((im) => {
|
||||
console.log(`From cache ${tileset}/${z}/${x}/${y}`)
|
||||
return im
|
||||
}).catch((_) => {
|
||||
return render_or_stitch_tile(tileset, z, x, y)
|
||||
})
|
||||
}
|
||||
|
||||
function render_or_stitch_tile(tileset, z, x, y) {
|
||||
if (z <= STITCH_THRESHOLD) {
|
||||
return stitch_tile(tileset, z, x, y).then(im => {
|
||||
return new Promise((resolve, reject) => {
|
||||
put(im, tileset, z, x, y, (err) => {
|
||||
if (err) {
|
||||
console.error(err)
|
||||
} else {
|
||||
console.log(`Stitch ${tileset}/${z}/${x}/${y}`)
|
||||
}
|
||||
resolve(im)
|
||||
})
|
||||
return put(im, tileset, z, x, y).then(() => {
|
||||
console.log(`Stitch ${tileset}/${z}/${x}/${y}`)
|
||||
return im
|
||||
}).catch((err) => {
|
||||
console.error(err)
|
||||
return im
|
||||
})
|
||||
})
|
||||
} else {
|
||||
@ -111,12 +104,11 @@ function render_or_stitch_tile(tileset, z, x, y) {
|
||||
reject(err)
|
||||
return
|
||||
}
|
||||
put(im, tileset, z, x, y, (err) => {
|
||||
if (err) {
|
||||
console.error(err)
|
||||
} else {
|
||||
console.log(`Render ${tileset}/${z}/${x}/${y}`)
|
||||
}
|
||||
put(im, tileset, z, x, y).then(() => {
|
||||
console.log(`Render ${tileset}/${z}/${x}/${y}`)
|
||||
resolve(im)
|
||||
}).catch((err) => {
|
||||
console.error(err)
|
||||
resolve(im)
|
||||
})
|
||||
})
|
||||
@ -132,8 +124,8 @@ function outside_extent(z, x, y) {
|
||||
function empty_tile() {
|
||||
return sharp({
|
||||
create: {
|
||||
width: TILE_SIZE,
|
||||
height: TILE_SIZE,
|
||||
width: 1,
|
||||
height: 1,
|
||||
channels: 4,
|
||||
background: { r: 0, g: 0, b: 0, alpha: 0 }
|
||||
}
|
||||
@ -209,6 +201,10 @@ function handle_highlight_tile_request(req, res) {
|
||||
return
|
||||
}
|
||||
|
||||
if (outside_extent(z, x, y)) {
|
||||
return empty_tile()
|
||||
}
|
||||
|
||||
render_tile('highlight', int_z, int_x, int_y, geometry_id, function (err, im) {
|
||||
if (err) throw err
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user