a_adli/colouring-montreal
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add echos to filtration step

Browse Source
This commit is contained in:
Ed Chalstrey 2022-03-29 14:42:34 +01:00
parent 3f07a9b081
commit 4a8d79a54a
1 changed files with 5 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
etl/filter_transform_mastermap_for_loading.sh
View File

@ -1,22 +1,13 @@
#!/usr/bin/env bash #!/usr/bin/env bash
#
# Filter and transform for loading
#
: ${1?"Usage: $0 ./path/to/mastermap/dir"} : ${1?"Usage: $0 ./path/to/mastermap/dir"}
mastermap_dir=$1 mastermap_dir=$1
# echo "Filter WHERE descriptiveGroup = '(1:Building)'... "
# Filter
# - WHERE descriptiveGroup = '(1:Building)'
# - OR toid in addressbase_toids
#
python filter_mastermap.py $mastermap_dir python filter_mastermap.py $mastermap_dir
# echo "Transform to 3857 (web mercator)..."
# Transform to 3857 (web mercator)
#
find $mastermap_dir -type f -name '*.gml.csv' -printf "%f\n" | \ find $mastermap_dir -type f -name '*.gml.csv' -printf "%f\n" | \
parallel \ parallel \
ogr2ogr \ ogr2ogr \
@ -27,13 +18,13 @@ ogr2ogr \
$mastermap_dir/{} \ $mastermap_dir/{} \
-lco GEOMETRY=AS_WKT -lco GEOMETRY=AS_WKT
# echo "Update to EWKT (with SRID indicator for loading to Postgres)..."
# Update to EWKT (with SRID indicator for loading to Postgres) echo "Updating POLYGONs.."
#
find $mastermap_dir -type f -name '*.3857.csv' -printf "%f\n" | \ find $mastermap_dir -type f -name '*.3857.csv' -printf "%f\n" | \
parallel \ parallel \
sed -i "'s/^\"POLYGON/\"SRID=3857;POLYGON/'" $mastermap_dir/{} sed -i "'s/^\"POLYGON/\"SRID=3857;POLYGON/'" $mastermap_dir/{}
echo "Updating MULTIPOLYGONs.."
find $mastermap_dir -type f -name '*.3857.csv' -printf "%f\n" | \ find $mastermap_dir -type f -name '*.3857.csv' -printf "%f\n" | \
parallel \ parallel \
sed -i "'s/^\"MULTIPOLYGON/\"SRID=3857;MULTIPOLYGON/'" $mastermap_dir/{} sed -i "'s/^\"MULTIPOLYGON/\"SRID=3857;MULTIPOLYGON/'" $mastermap_dir/{}