From a22bf0079ba1ab9789dc393f02042e87616e4852 Mon Sep 17 00:00:00 2001
From: Ed Chalstrey <edwardchalstrey@gmail.com>
Date: Thu, 21 Apr 2022 14:33:49 +0100
Subject: [PATCH] drop new geometries outside boundary before adding to
 geometries table

---
 etl/README.md                            | 10 ++++++++--
 etl/add_new_geometries.sh                |  4 ++++
 etl/drop_outside_limit_new_geometries.sh | 18 ++++++++++++++++++
 etl/load_new_geometries.sh               |  5 -----
 4 files changed, 30 insertions(+), 7 deletions(-)
 create mode 100644 etl/add_new_geometries.sh
 create mode 100644 etl/drop_outside_limit_new_geometries.sh

diff --git a/etl/README.md b/etl/README.md
index 6d767e0e..465fd017 100644
--- a/etl/README.md
+++ b/etl/README.md
@@ -132,7 +132,7 @@ Filter MasterMap 'building' polygons.
 sudo ./filter_transform_mastermap_for_loading.sh /path/to/mastermap_dir
 ```
 
-Load all new building outlines. This step will only add geometries that are not already present (based on the `TOID`). Note: you should ensure that `mastermap_dir` has permissions that will allow the linux `find` command to work without using sudo.
+Load all new building outlines. This step will only load geometries that are not already present (based on the `TOID`). Note: you should ensure that `mastermap_dir` has permissions that will allow the linux `find` command to work without using sudo.
 
 ```bash
 ./load_new_geometries.sh /path/to/mastermap_dir
@@ -144,7 +144,13 @@ Drop building outlines outside London boundary.
 cd ~/colouring-london/app/public/geometries
 ogr2ogr -t_srs EPSG:3857 -f "ESRI Shapefile" boundary.shp boundary-detailed.geojson
 cd ~/colouring-london/etl/
-./drop_outside_limit.sh ~/colouring-london/app/public/geometries/boundary.shp
+./drop_outside_limit_new_geometries.sh ~/colouring-london/app/public/geometries/boundary.shp
+```
+
+Add new geometries (building outlines) to existing geometries table.
+
+```bash
+./add_new_geometries.sh 
 ```
 
 Create building record to match each new geometry (TOID) that doesn't already have a linked building.
diff --git a/etl/add_new_geometries.sh b/etl/add_new_geometries.sh
new file mode 100644
index 00000000..5ac506d3
--- /dev/null
+++ b/etl/add_new_geometries.sh
@@ -0,0 +1,4 @@
+echo "Adding new geometries to geometries table..."
+psql -c "INSERT INTO geometries ( source_id, geometry_geom )
+         SELECT source_id, geometry_geom
+         FROM new_geometries;"
\ No newline at end of file
diff --git a/etl/drop_outside_limit_new_geometries.sh b/etl/drop_outside_limit_new_geometries.sh
new file mode 100644
index 00000000..d8d3a307
--- /dev/null
+++ b/etl/drop_outside_limit_new_geometries.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+#
+# Load boundary and filter geometries
+# - boundary MUST be epsg:3857
+# use: ogr2ogr -t_srs EPSG:3857 boundary.3857.shp boundary.shp
+#
+: ${1?"Usage: $0 ./path/to/boundary"}
+boundary_file=$1
+
+echo "Load boundary..."
+psql -c "DROP TABLE IF EXISTS boundary"
+shp2pgsql -s 3857 $boundary_file boundary | psql
+
+echo "Delete geometries (hence buildings, building_properties)..."
+psql -c "DELETE FROM new_geometries as g
+USING boundary as b
+WHERE b.gid = 1 AND NOT ST_ContainsProperly(b.geom, g.geometry_geom);"
diff --git a/etl/load_new_geometries.sh b/etl/load_new_geometries.sh
index 32fbb1e6..33bbc2b2 100644
--- a/etl/load_new_geometries.sh
+++ b/etl/load_new_geometries.sh
@@ -53,8 +53,3 @@ psql -c "INSERT INTO new_geometries ( source_id, geometry_geom )
          WHERE NOT EXISTS ( SELECT source_id
                             FROM geometries AS g
                             WHERE g.source_id = r.source_id);"
-         
-echo "Adding new geometries to geometries table..."
-psql -c "INSERT INTO geometries ( source_id, geometry_geom )
-         SELECT source_id, geometry_geom
-         FROM new_geometries;"