a_adli/colouring-montreal
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add echos to extraction step

Browse Source
This commit is contained in:
Ed Chalstrey 2022-03-29 14:46:18 +01:00
parent 4a8d79a54a
commit 0e01971b4a
1 changed files with 6 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
etl/extract_mastermap.sh
View File

@ -1,31 +1,28 @@
#!/usr/bin/env bash #!/usr/bin/env bash
#
# Extract MasterMap
#
: ${1?"Usage: $0 ./path/to/mastermap/dir"} : ${1?"Usage: $0 ./path/to/mastermap/dir"}
data_dir=$1 data_dir=$1
#
# Extract buildings from *.gz to CSV echo "Extract buildings from *.gz..."
#
# Features where:: # Features where::
# descriptiveGroup = '(1:Building)' # descriptiveGroup = '(1:Building)'
# #
# Use `fid` as source ID, aka TOID. # Use `fid` as source ID, aka TOID.
#
find $data_dir -type f -name '*.gz' -printf "%f\n" | \ find $data_dir -type f -name '*.gz' -printf "%f\n" | \
parallel \ parallel \
gunzip $data_dir/{} -k -S gml gunzip $data_dir/{} -k -S gml
echo "Rename extracted files to .gml..."
rename 's/$/.gml/' $data_dir/*[^gzvt] rename 's/$/.gml/' $data_dir/*[^gzvt]
# Note: previously the rename cmd above resulted in some temp files being renamed to .gml # Note: previously the rename cmd above resulted in some temp files being renamed to .gml
# so I have specified the start of the filename (appears to be consistent for all OS MasterMap downloads) # so I have specified the start of the filename (appears to be consistent for all OS MasterMap downloads)
# we may need to update this below for other downloads # we may need to update this below for other downloads
echo "Covert .gml files to .csv"
find $data_dir -type f -name '*5690395*.gml' -printf "%f\n" | \ find $data_dir -type f -name '*5690395*.gml' -printf "%f\n" | \
parallel \ parallel \
ogr2ogr \ ogr2ogr \
@ -35,5 +32,6 @@ ogr2ogr \
TopographicArea \ TopographicArea \
-lco GEOMETRY=AS_WKT -lco GEOMETRY=AS_WKT
echo "Remove .gfs and .gml files from previous steps..."
rm $data_dir/*.gfs rm $data_dir/*.gfs
rm $data_dir/*.gml rm $data_dir/*.gml