First commit of the workflow module
This commit is contained in:
parent
96a09e1487
commit
adee45be00
192
handle_mtl_ds_workflow.py
Normal file
192
handle_mtl_ds_workflow.py
Normal file
@ -0,0 +1,192 @@
|
|||||||
|
from scrub_layer_class import *
|
||||||
|
from basic_functions import *
|
||||||
|
|
||||||
|
qgis_path = 'C:/Program Files/QGIS 3.34.1/apps/qgis'
|
||||||
|
|
||||||
|
# Gathering input data layers paths
|
||||||
|
input_paths = {
|
||||||
|
'NRCan':
|
||||||
|
'C:/Users/a_adli/PycharmProjects/hydroquebec_archetype_gispy/'
|
||||||
|
'data/input_data/nrcan/Autobuilding_QC_VILLE_MONTREAL.shp',
|
||||||
|
'GeoIndex':
|
||||||
|
'C:/Users/a_adli/PycharmProjects/hydroquebec_archetype_gispy/'
|
||||||
|
'data/input_data/Geoindex_81670/mamh_usage_predo_2022_s_poly.shp',
|
||||||
|
'Property Assessment':
|
||||||
|
'C:/Users/a_adli/PycharmProjects/hydroquebec_archetype_gispy/'
|
||||||
|
'data/input_data/property_assessment/uniteevaluationfonciere.shp',
|
||||||
|
'Montreal Boundary':
|
||||||
|
'C:/Users/a_adli/PycharmProjects/hydroquebec_archetype_gispy/'
|
||||||
|
'data/input_data/montreal_boundary/Montreal_boundary.shp'
|
||||||
|
}
|
||||||
|
|
||||||
|
# Defining a directory for all the output data layers
|
||||||
|
output_paths_dir = \
|
||||||
|
'C:/Users/a_adli/PycharmProjects/hydroquebec_archetype_gispy/' \
|
||||||
|
'data/gisoo_workflow_output_2'
|
||||||
|
|
||||||
|
# Preparing a bedding for output data layers paths
|
||||||
|
output_paths = {
|
||||||
|
'Fixed NRCan': '',
|
||||||
|
'Fixed GeoIndex': '',
|
||||||
|
'Clipped Fixed GeoIndex': '',
|
||||||
|
'Splitted NRCans': '',
|
||||||
|
'Pairwise Clipped Property Assessment Partitions': '',
|
||||||
|
'Pairwise Clipped Merged Property Assessment': '',
|
||||||
|
'Property Assessment and NRCan': '',
|
||||||
|
'Property Assessment and NRCan and GeoIndex': '',
|
||||||
|
'Deleted Duplicates Layer': '',
|
||||||
|
'Singled Parts Layer': ''
|
||||||
|
}
|
||||||
|
|
||||||
|
# Making folders for the output data layers
|
||||||
|
create_output_folders(output_paths, output_paths_dir)
|
||||||
|
|
||||||
|
# Initialize the input data layers
|
||||||
|
nrcan = ScrubLayer(qgis_path, input_paths['NRCan'], 'NRCan')
|
||||||
|
geo_index = ScrubLayer(qgis_path, input_paths['GeoIndex'], 'GeoIndex')
|
||||||
|
property_assessment = \
|
||||||
|
ScrubLayer(
|
||||||
|
qgis_path, input_paths['Property Assessment'], 'Property Assessment')
|
||||||
|
montreal_boundary = \
|
||||||
|
ScrubLayer(qgis_path, input_paths['Montreal Boundary'], 'Montreal Boundary')
|
||||||
|
|
||||||
|
# Processing the NRCan layer includes fixing its geometries
|
||||||
|
print('Processing the NRCan layer')
|
||||||
|
print(f'Data Count of the NRCan layer: {nrcan.data_count}')
|
||||||
|
nrcan.create_spatial_index()
|
||||||
|
nrcan.fix_geometries(output_paths['Fixed NRCan'])
|
||||||
|
|
||||||
|
# Defining a new layer for the fixed NRCan
|
||||||
|
nrcan_fixed = \
|
||||||
|
ScrubLayer(qgis_path, output_paths['Fixed NRCan'], 'Fixed NRCan')
|
||||||
|
nrcan_fixed.create_spatial_index()
|
||||||
|
print(f'Data Count of the NRCan layer after fixing geometries '
|
||||||
|
f'({nrcan_fixed.layer_name}): {nrcan_fixed.data_count}')
|
||||||
|
|
||||||
|
# Processing the GeoIndex layer includes fixing its geometries and
|
||||||
|
# clipping it based on the Montreal boundary data layer
|
||||||
|
print('Processing the GeoIndex layer')
|
||||||
|
print(f'Data Count of the GeoIndex layer: {geo_index.data_count}')
|
||||||
|
geo_index.create_spatial_index()
|
||||||
|
geo_index.fix_geometries(output_paths['Fixed GeoIndex'])
|
||||||
|
|
||||||
|
# Defining a new layer for the fixed GeoIndex
|
||||||
|
geo_index_fixed = ScrubLayer(qgis_path, output_paths['Fixed GeoIndex'],
|
||||||
|
'Fixed GeoIndex')
|
||||||
|
geo_index_fixed.create_spatial_index()
|
||||||
|
print(f'Data Count of the GeoIndex layer after fixing geometries '
|
||||||
|
f'({geo_index_fixed.layer_name}): {geo_index_fixed.data_count}')
|
||||||
|
geo_index_fixed.clip_layer(montreal_boundary.layer_path,
|
||||||
|
output_paths['Clipped Fixed GeoIndex'])
|
||||||
|
geo_index_clipped = \
|
||||||
|
ScrubLayer(qgis_path,
|
||||||
|
output_paths['Clipped Fixed GeoIndex'], 'Clipped Fixed GeoIndex')
|
||||||
|
geo_index_clipped.create_spatial_index()
|
||||||
|
print(f'Data Count of the {geo_index_fixed.layer_name} '
|
||||||
|
f' after clipping it '
|
||||||
|
f'based on the Montreal Boundary layer ({geo_index_clipped.layer_name}): '
|
||||||
|
f'{geo_index_clipped.layer.featureCount()}')
|
||||||
|
|
||||||
|
# Processing the Property Assessment layer includes a pairwise clip, and
|
||||||
|
# two spatial join with NRCan and GeoIndex layers, respectively
|
||||||
|
|
||||||
|
print(f'Data Count of the Property Assessment layer:'
|
||||||
|
f' {property_assessment.data_count}')
|
||||||
|
property_assessment.create_spatial_index()
|
||||||
|
|
||||||
|
# For the pairwise clip, number of overlaying layers can be chosen
|
||||||
|
# (meaning number of splits for NRCan layer). This improves the performance
|
||||||
|
# where may increase duplicates. This has been done because using the NRCan
|
||||||
|
# layer as a whole causes crashing the clipping process.
|
||||||
|
|
||||||
|
# First we split the overlaying layers into our desired number
|
||||||
|
nrcan_fixed.split_layer(120, output_paths['Splitted NRCans'])
|
||||||
|
|
||||||
|
# Clipping have to be done in
|
||||||
|
clipping_property_assessment = """
|
||||||
|
from input_paths_and_layers import *
|
||||||
|
|
||||||
|
property_assessment.clip_by_multiple(
|
||||||
|
120, output_paths['Splitted NRCans'],
|
||||||
|
output_paths['Pairwise Clipped Property Assessment Partitions'])"""
|
||||||
|
|
||||||
|
exec(clipping_property_assessment)
|
||||||
|
|
||||||
|
property_assessment.merge_layers(
|
||||||
|
output_paths['Pairwise Clipped Property Assessment Partitions'],
|
||||||
|
output_paths['Pairwise Clipped Merged Property Assessment'])
|
||||||
|
|
||||||
|
clipped_property_assessment = ScrubLayer(
|
||||||
|
qgis_path,
|
||||||
|
output_paths['Pairwise Clipped Merged Property Assessment'],
|
||||||
|
'Clipped Property Assessment')
|
||||||
|
|
||||||
|
print(f'Data Count of the Property Assessment layer '
|
||||||
|
f'after the pairwise clipped with NRCan layer '
|
||||||
|
f'({clipped_property_assessment.layer_name}):'
|
||||||
|
f' {clipped_property_assessment.data_count}')
|
||||||
|
clipped_property_assessment.create_spatial_index()
|
||||||
|
|
||||||
|
clipped_property_assessment.spatial_join(
|
||||||
|
nrcan_fixed.layer_path,
|
||||||
|
output_paths['Property Assessment and NRCan'])
|
||||||
|
|
||||||
|
property_assessment_nrcan = ScrubLayer(
|
||||||
|
qgis_path,
|
||||||
|
output_paths['Property Assessment and NRCan'],
|
||||||
|
'Property Assessment and NRCan')
|
||||||
|
|
||||||
|
print(f'Data Count of the Property Assessment layer '
|
||||||
|
f'after its spatial join with NRCan layer '
|
||||||
|
f'({property_assessment_nrcan.layer_name}):'
|
||||||
|
f' {property_assessment_nrcan.data_count}')
|
||||||
|
property_assessment_nrcan.create_spatial_index()
|
||||||
|
|
||||||
|
property_assessment_nrcan.spatial_join(
|
||||||
|
geo_index_clipped,
|
||||||
|
output_paths['Property Assessment and NRCan and GeoIndex'])
|
||||||
|
|
||||||
|
property_assessment_nrcan_geo = ScrubLayer(
|
||||||
|
qgis_path,
|
||||||
|
output_paths['Property Assessment and NRCan and GeoIndex'],
|
||||||
|
'Property Assessment and NRCan and GeoIndex')
|
||||||
|
|
||||||
|
print(f'Data Count of the Property Assessment-NRCan layer '
|
||||||
|
f'after its spatial join with GeoIndex layer:'
|
||||||
|
f' {property_assessment_nrcan_geo.data_count}\n'
|
||||||
|
f'Name of the new layer: '
|
||||||
|
f'{property_assessment_nrcan_geo.layer_name}')
|
||||||
|
property_assessment_nrcan_geo.create_spatial_index()
|
||||||
|
|
||||||
|
property_assessment_nrcan_geo.delete_duplicates(
|
||||||
|
output_paths['Deleted Duplicates Layer'])
|
||||||
|
|
||||||
|
deleted_dups_layer = ScrubLayer(
|
||||||
|
qgis_path,
|
||||||
|
output_paths['Deleted Duplicates Layer'],
|
||||||
|
'Deleted Duplicates Layer')
|
||||||
|
|
||||||
|
print(f'Data Count of the Property Assessment-NRCan-GeoIndex layer '
|
||||||
|
f'after deleting duplicates: {deleted_dups_layer.data_count}\n'
|
||||||
|
f'Name of the new layer: '
|
||||||
|
f'{deleted_dups_layer.layer_name}')
|
||||||
|
property_assessment_nrcan_geo.create_spatial_index()
|
||||||
|
|
||||||
|
property_assessment_nrcan_geo.multipart_to_singleparts(
|
||||||
|
output_paths['Single Parts Layer'])
|
||||||
|
|
||||||
|
single_parts_layer = ScrubLayer(
|
||||||
|
qgis_path,
|
||||||
|
output_paths['Single Parts Layer'],
|
||||||
|
'Single Parts Layer')
|
||||||
|
|
||||||
|
print(f'Data Count of the Deleted Duplicates layer '
|
||||||
|
f'after separating multiparts to clickable single parts: '
|
||||||
|
f'{single_parts_layer.data_count}\n'
|
||||||
|
f'Name of the new layer: '
|
||||||
|
f'{single_parts_layer.layer_name}')
|
||||||
|
single_parts_layer.create_spatial_index()
|
||||||
|
|
||||||
|
# The process of instantiating a new layer
|
||||||
|
# (including print and create part) can be a method
|
||||||
|
# Also experiment importing input_paths_and_layers
|
||||||
0
input_paths_and_layers.py
Normal file
0
input_paths_and_layers.py
Normal file
Loading…
Reference in New Issue
Block a user