drop new geometries outside boundary before adding to geometries table

etl/README.md

@@ -132,7 +132,7 @@ Filter MasterMap 'building' polygons.
 sudo ./filter_transform_mastermap_for_loading.sh /path/to/mastermap_dir
 ```
 
-Load all new building outlines. This step will only add geometries that are not already present (based on the `TOID`). Note: you should ensure that `mastermap_dir` has permissions that will allow the linux `find` command to work without using sudo.
+Load all new building outlines. This step will only load geometries that are not already present (based on the `TOID`). Note: you should ensure that `mastermap_dir` has permissions that will allow the linux `find` command to work without using sudo.
 
 ```bash
 ./load_new_geometries.sh /path/to/mastermap_dir
@@ -144,7 +144,13 @@ Drop building outlines outside London boundary.
 cd ~/colouring-london/app/public/geometries
 ogr2ogr -t_srs EPSG:3857 -f "ESRI Shapefile" boundary.shp boundary-detailed.geojson
 cd ~/colouring-london/etl/
-./drop_outside_limit.sh ~/colouring-london/app/public/geometries/boundary.shp
+./drop_outside_limit_new_geometries.sh ~/colouring-london/app/public/geometries/boundary.shp
+```
+
+Add new geometries (building outlines) to existing geometries table.
+
+```bash
+./add_new_geometries.sh 
 ```
 
 Create building record to match each new geometry (TOID) that doesn't already have a linked building.

etl/add_new_geometries.sh

@@ -0,0 +1,4 @@
+echo "Adding new geometries to geometries table..."
+psql -c "INSERT INTO geometries ( source_id, geometry_geom )
+         SELECT source_id, geometry_geom
+         FROM new_geometries;"

etl/drop_outside_limit_new_geometries.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+#
+# Load boundary and filter geometries
+# - boundary MUST be epsg:3857
+# use: ogr2ogr -t_srs EPSG:3857 boundary.3857.shp boundary.shp
+#
+: ${1?"Usage: $0 ./path/to/boundary"}
+boundary_file=$1
+
+echo "Load boundary..."
+psql -c "DROP TABLE IF EXISTS boundary"
+shp2pgsql -s 3857 $boundary_file boundary | psql
+
+echo "Delete geometries (hence buildings, building_properties)..."
+psql -c "DELETE FROM new_geometries as g
+USING boundary as b
+WHERE b.gid = 1 AND NOT ST_ContainsProperly(b.geom, g.geometry_geom);"

etl/load_new_geometries.sh

@@ -53,8 +53,3 @@ psql -c "INSERT INTO new_geometries ( source_id, geometry_geom )
          WHERE NOT EXISTS ( SELECT source_id
                             FROM geometries AS g
                             WHERE g.source_id = r.source_id);"
-         
-echo "Adding new geometries to geometries table..."
-psql -c "INSERT INTO geometries ( source_id, geometry_geom )
-         SELECT source_id, geometry_geom
-         FROM new_geometries;"