Rewrite cache to allow expiry, use promises
- tileserver changes use of cache (slightly simplified from callbacks) - cache methods return promises - add 'remove' method to cache, with remove_all_at_bbox helper - from api/building.js, call remove after successful db updates
This commit is contained in:
Tom Russell
parent
cfc00f348b
commit
4362c9c947
@ -3,6 +3,8 @@
|
||||
*
|
||||
*/
|
||||
import db from '../db';
|
||||
import { remove_all_at_bbox } from '../tiles/cache';
|
||||
|
||||
// data type note: PostgreSQL bigint (64-bit) is handled as string in JavaScript, because of
|
||||
// JavaScript numerics are 64-bit double, giving only partial coverage.
|
||||
|
||||
@ -128,7 +130,7 @@ function saveBuilding(building_id, building, user_id) {
|
||||
[building_id]
|
||||
).then(old_building => {
|
||||
const patches = compare(old_building, building, BUILDING_FIELD_WHITELIST);
|
||||
console.log("Patching", patches)
|
||||
console.log("Patching", building_id, patches)
|
||||
const forward = patches[0];
|
||||
const reverse = patches[1];
|
||||
if (Object.keys(forward).length === 0) {
|
||||
@ -144,7 +146,7 @@ function saveBuilding(building_id, building, user_id) {
|
||||
[forward, reverse, building_id, user_id]
|
||||
).then(revision => {
|
||||
const sets = db.$config.pgp.helpers.sets(forward);
|
||||
console.log("Setting", sets)
|
||||
console.log("Setting", building_id, sets)
|
||||
return t.one(
|
||||
`UPDATE
|
||||
buildings
|
||||
@ -157,7 +159,10 @@ function saveBuilding(building_id, building, user_id) {
|
||||
*
|
||||
`,
|
||||
[revision.log_id, sets, building_id]
|
||||
)
|
||||
).then((data) => {
|
||||
expireBuildingTileCache(building_id)
|
||||
return data
|
||||
})
|
||||
});
|
||||
});
|
||||
}).catch(function (error) {
|
||||
@ -203,7 +208,10 @@ function likeBuilding(building_id, user_id) {
|
||||
*
|
||||
`,
|
||||
[revision.log_id, building.likes, building_id]
|
||||
)
|
||||
).then((data) => {
|
||||
expireBuildingTileCache(building_id)
|
||||
return data
|
||||
})
|
||||
})
|
||||
});
|
||||
});
|
||||
@ -255,7 +263,10 @@ function unlikeBuilding(building_id, user_id) {
|
||||
*
|
||||
`,
|
||||
[revision.log_id, building.likes, building_id]
|
||||
)
|
||||
).then((data) => {
|
||||
expireBuildingTileCache(building_id)
|
||||
return data
|
||||
})
|
||||
})
|
||||
});
|
||||
});
|
||||
@ -270,6 +281,33 @@ function unlikeBuilding(building_id, user_id) {
|
||||
});
|
||||
}
|
||||
|
||||
function privateQueryBuildingBBOX(building_id){
|
||||
return db.one(
|
||||
`SELECT
|
||||
ST_XMin(envelope) as xmin,
|
||||
ST_YMin(envelope) as ymin,
|
||||
ST_XMax(envelope) as xmax,
|
||||
ST_YMax(envelope) as ymax
|
||||
FROM (
|
||||
SELECT
|
||||
ST_Envelope(g.geometry_geom) as envelope
|
||||
FROM buildings as b, geometries as g
|
||||
WHERE
|
||||
b.geometry_id = g.geometry_id
|
||||
AND
|
||||
b.building_id = $1
|
||||
) as envelope`,
|
||||
[building_id]
|
||||
)
|
||||
}
|
||||
|
||||
function expireBuildingTileCache(building_id) {
|
||||
privateQueryBuildingBBOX(building_id).then((bbox) => {
|
||||
const building_bbox = [bbox.xmax, bbox.ymax, bbox.xmin, bbox.ymin]
|
||||
remove_all_at_bbox(building_bbox);
|
||||
})
|
||||
}
|
||||
|
||||
const BUILDING_FIELD_WHITELIST = new Set([
|
||||
'ref_osm_id',
|
||||
// 'location_name',
|
||||
|
||||
@ -18,40 +18,143 @@
|
||||
// and then use stdlib `import fs from 'fs';`
|
||||
import fs from 'node-fs';
|
||||
|
||||
import { get_xyz } from './tile';
|
||||
|
||||
// Use an environment variable to configure the cache location, somewhere we can read/write to.
|
||||
const CACHE_PATH = process.env.TILECACHE_PATH
|
||||
|
||||
function get(tileset, z, x, y, cb) {
|
||||
/**
|
||||
* Get a tile from the cache
|
||||
*
|
||||
* @param {String} tileset
|
||||
* @param {number} z zoom level
|
||||
* @param {number} x
|
||||
* @param {number} y
|
||||
*/
|
||||
function get(tileset, z, x, y) {
|
||||
if (!should_try_cache(tileset, z)) {
|
||||
cb(`Skip cache get ${tileset}/${z}/${x}/${y}`, null)
|
||||
return
|
||||
return Promise.reject(`Skip cache get ${tileset}/${z}/${x}/${y}`);
|
||||
}
|
||||
const dir = `${CACHE_PATH}/${tileset}/${z}/${x}`
|
||||
const fname = `${dir}/${y}.png`
|
||||
fs.readFile(fname, cb)
|
||||
}
|
||||
|
||||
function put(im, tileset, z, x, y, cb) {
|
||||
if (!should_try_cache(tileset, z)) {
|
||||
cb(`Skip cache put ${tileset}/${z}/${x}/${y}`)
|
||||
return
|
||||
}
|
||||
const dir = `${CACHE_PATH}/${tileset}/${z}/${x}`
|
||||
const fname = `${dir}/${y}.png`
|
||||
fs.writeFile(fname, im, 'binary', (err) => {
|
||||
if (err && err.code === 'ENOENT') {
|
||||
fs.mkdir(dir, 0o755, true, (err) => {
|
||||
if (err) {
|
||||
cb(err);
|
||||
} else {
|
||||
fs.writeFile(fname, im, 'binary', cb);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
cb(err)
|
||||
}
|
||||
const location = cache_location(tileset, z, x, y);
|
||||
return new Promise((resolve, reject) => {
|
||||
fs.readFile(location.fname, (err, data) => {
|
||||
if (err) {
|
||||
reject(err)
|
||||
} else {
|
||||
resolve(data)
|
||||
}
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Put a tile in the cache
|
||||
*
|
||||
* @param {Buffer} im image data
|
||||
* @param {String} tileset
|
||||
* @param {number} z zoom level
|
||||
* @param {number} x
|
||||
* @param {number} y
|
||||
*/
|
||||
function put(im, tileset, z, x, y) {
|
||||
if (!should_try_cache(tileset, z)) {
|
||||
return Promise.reject(`Skip cache put ${tileset}/${z}/${x}/${y}`);
|
||||
}
|
||||
const location = cache_location(tileset, z, x, y);
|
||||
return new Promise((resolve, reject) => {
|
||||
fs.writeFile(location.fname, im, 'binary', (err) => {
|
||||
if (err && err.code === 'ENOENT') {
|
||||
// recursively create tile directory if it didn't previously exist
|
||||
fs.mkdir(location.dir, 0o755, true, (err) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
} else {
|
||||
// then write the file
|
||||
fs.writeFile(location.fname, im, 'binary', (err) => {
|
||||
(err)? reject(err): resolve()
|
||||
});
|
||||
}
|
||||
});
|
||||
} else {
|
||||
(err)? reject(err): resolve()
|
||||
}
|
||||
});
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a single cached tile
|
||||
*
|
||||
* @param {String} tileset
|
||||
* @param {number} z zoom level
|
||||
* @param {number} x
|
||||
* @param {number} y
|
||||
*/
|
||||
function remove(tileset, z, x, y) {
|
||||
const location = cache_location(tileset, z, x, y)
|
||||
return new Promise(resolve => {
|
||||
fs.unlink(location.fname, (err) => {
|
||||
if(err){
|
||||
// pass
|
||||
} else {
|
||||
console.log("Expire cache", tileset, z, x, y)
|
||||
}
|
||||
resolve()
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove all cached data-visualising tiles which intersect a bbox
|
||||
* - initially called directly after edits; may be better on a worker process?
|
||||
*
|
||||
* @param {String} tileset
|
||||
* @param {Array} bbox [w, s, e, n] in EPSG:3857 coordinates
|
||||
*/
|
||||
function remove_all_at_bbox(bbox) {
|
||||
// magic numbers for min/max zoom
|
||||
const min_zoom = 9;
|
||||
const max_zoom = 18;
|
||||
// magic list of tilesets - see tileserver, other cache rules
|
||||
const tilesets = ['date_year', 'size_storeys', 'location', 'likes', 'conservation_area'];
|
||||
let tile_bounds;
|
||||
const remove_promises = [];
|
||||
for (let ti = 0; ti < tilesets.length; ti++) {
|
||||
const tileset = tilesets[ti];
|
||||
for (let z = min_zoom; z <= max_zoom; z++) {
|
||||
tile_bounds = get_xyz(bbox, z)
|
||||
for (let x = tile_bounds.minX; x <= tile_bounds.maxX; x++){
|
||||
for (let y = tile_bounds.minY; y <= tile_bounds.maxY; y++){
|
||||
remove_promises.push(remove(tileset, z, x, y))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Promise.all(remove_promises)
|
||||
}
|
||||
|
||||
/**
|
||||
* Cache location for a tile
|
||||
*
|
||||
* @param {String} tileset
|
||||
* @param {number} z zoom level
|
||||
* @param {number} x
|
||||
* @param {number} y
|
||||
* @returns {object} { dir: <directory>, fname: <full filepath> }
|
||||
*/
|
||||
function cache_location(tileset, z, x, y) {
|
||||
const dir = `${CACHE_PATH}/${tileset}/${z}/${x}`
|
||||
const fname = `${dir}/${y}.png`
|
||||
return {dir, fname}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check rules for caching tiles
|
||||
*
|
||||
* @param {String} tileset
|
||||
* @param {number} z zoom level
|
||||
* @returns {boolean} whether to use the cache (or not)
|
||||
*/
|
||||
function should_try_cache(tileset, z) {
|
||||
if (tileset === 'date_year') {
|
||||
// cache high zoom because of front page hits
|
||||
@ -65,4 +168,4 @@ function should_try_cache(tileset, z) {
|
||||
return z <= 13
|
||||
}
|
||||
|
||||
export { get, put };
|
||||
export { get, put, remove, remove_all_at_bbox };
|
||||
|
||||
@ -77,33 +77,23 @@ function load_tile(tileset, z, x, y) {
|
||||
if (outside_extent(z, x, y)) {
|
||||
return empty_tile()
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
get(tileset, z, x, y, (err, im) => {
|
||||
if (err) {
|
||||
render_or_stitch_tile(tileset, z, x, y)
|
||||
.then((im) => {
|
||||
resolve(im)
|
||||
})
|
||||
} else {
|
||||
console.log(`From cache ${tileset}/${z}/${x}/${y}`)
|
||||
resolve(im)
|
||||
}
|
||||
})
|
||||
return get(tileset, z, x, y).then((im) => {
|
||||
console.log(`From cache ${tileset}/${z}/${x}/${y}`)
|
||||
return im
|
||||
}).catch((_) => {
|
||||
return render_or_stitch_tile(tileset, z, x, y)
|
||||
})
|
||||
}
|
||||
|
||||
function render_or_stitch_tile(tileset, z, x, y) {
|
||||
if (z <= STITCH_THRESHOLD) {
|
||||
return stitch_tile(tileset, z, x, y).then(im => {
|
||||
return new Promise((resolve, reject) => {
|
||||
put(im, tileset, z, x, y, (err) => {
|
||||
if (err) {
|
||||
console.error(err)
|
||||
} else {
|
||||
console.log(`Stitch ${tileset}/${z}/${x}/${y}`)
|
||||
}
|
||||
resolve(im)
|
||||
})
|
||||
return put(im, tileset, z, x, y).then(() => {
|
||||
console.log(`Stitch ${tileset}/${z}/${x}/${y}`)
|
||||
return im
|
||||
}).catch((err) => {
|
||||
console.error(err)
|
||||
return im
|
||||
})
|
||||
})
|
||||
} else {
|
||||
@ -114,12 +104,11 @@ function render_or_stitch_tile(tileset, z, x, y) {
|
||||
reject(err)
|
||||
return
|
||||
}
|
||||
put(im, tileset, z, x, y, (err) => {
|
||||
if (err) {
|
||||
console.error(err)
|
||||
} else {
|
||||
console.log(`Render ${tileset}/${z}/${x}/${y}`)
|
||||
}
|
||||
put(im, tileset, z, x, y).then(() => {
|
||||
console.log(`Render ${tileset}/${z}/${x}/${y}`)
|
||||
resolve(im)
|
||||
}).catch((err) => {
|
||||
console.error(err)
|
||||
resolve(im)
|
||||
})
|
||||
})
|
||||
|
||||
Loading…
Reference in New Issue
Block a user