From adee45be008215fcbf24c7dfdd74fbf6f6ed95ca Mon Sep 17 00:00:00 2001 From: Alireza Adli Date: Fri, 12 Apr 2024 11:16:30 -0400 Subject: [PATCH] First commit of the workflow module --- handle_mtl_ds_workflow.py | 192 ++++++++++++++++++++++++++++++++++++++ input_paths_and_layers.py | 0 2 files changed, 192 insertions(+) create mode 100644 handle_mtl_ds_workflow.py create mode 100644 input_paths_and_layers.py diff --git a/handle_mtl_ds_workflow.py b/handle_mtl_ds_workflow.py new file mode 100644 index 0000000..d81398f --- /dev/null +++ b/handle_mtl_ds_workflow.py @@ -0,0 +1,192 @@ +from scrub_layer_class import * +from basic_functions import * + +qgis_path = 'C:/Program Files/QGIS 3.34.1/apps/qgis' + +# Gathering input data layers paths +input_paths = { + 'NRCan': + 'C:/Users/a_adli/PycharmProjects/hydroquebec_archetype_gispy/' + 'data/input_data/nrcan/Autobuilding_QC_VILLE_MONTREAL.shp', + 'GeoIndex': + 'C:/Users/a_adli/PycharmProjects/hydroquebec_archetype_gispy/' + 'data/input_data/Geoindex_81670/mamh_usage_predo_2022_s_poly.shp', + 'Property Assessment': + 'C:/Users/a_adli/PycharmProjects/hydroquebec_archetype_gispy/' + 'data/input_data/property_assessment/uniteevaluationfonciere.shp', + 'Montreal Boundary': + 'C:/Users/a_adli/PycharmProjects/hydroquebec_archetype_gispy/' + 'data/input_data/montreal_boundary/Montreal_boundary.shp' +} + +# Defining a directory for all the output data layers +output_paths_dir = \ + 'C:/Users/a_adli/PycharmProjects/hydroquebec_archetype_gispy/' \ + 'data/gisoo_workflow_output_2' + +# Preparing a bedding for output data layers paths +output_paths = { + 'Fixed NRCan': '', + 'Fixed GeoIndex': '', + 'Clipped Fixed GeoIndex': '', + 'Splitted NRCans': '', + 'Pairwise Clipped Property Assessment Partitions': '', + 'Pairwise Clipped Merged Property Assessment': '', + 'Property Assessment and NRCan': '', + 'Property Assessment and NRCan and GeoIndex': '', + 'Deleted Duplicates Layer': '', + 'Singled Parts Layer': '' +} + +# Making folders for the output data layers +create_output_folders(output_paths, output_paths_dir) + +# Initialize the input data layers +nrcan = ScrubLayer(qgis_path, input_paths['NRCan'], 'NRCan') +geo_index = ScrubLayer(qgis_path, input_paths['GeoIndex'], 'GeoIndex') +property_assessment = \ + ScrubLayer( + qgis_path, input_paths['Property Assessment'], 'Property Assessment') +montreal_boundary = \ + ScrubLayer(qgis_path, input_paths['Montreal Boundary'], 'Montreal Boundary') + +# Processing the NRCan layer includes fixing its geometries +print('Processing the NRCan layer') +print(f'Data Count of the NRCan layer: {nrcan.data_count}') +nrcan.create_spatial_index() +nrcan.fix_geometries(output_paths['Fixed NRCan']) + +# Defining a new layer for the fixed NRCan +nrcan_fixed = \ + ScrubLayer(qgis_path, output_paths['Fixed NRCan'], 'Fixed NRCan') +nrcan_fixed.create_spatial_index() +print(f'Data Count of the NRCan layer after fixing geometries ' + f'({nrcan_fixed.layer_name}): {nrcan_fixed.data_count}') + +# Processing the GeoIndex layer includes fixing its geometries and +# clipping it based on the Montreal boundary data layer +print('Processing the GeoIndex layer') +print(f'Data Count of the GeoIndex layer: {geo_index.data_count}') +geo_index.create_spatial_index() +geo_index.fix_geometries(output_paths['Fixed GeoIndex']) + +# Defining a new layer for the fixed GeoIndex +geo_index_fixed = ScrubLayer(qgis_path, output_paths['Fixed GeoIndex'], + 'Fixed GeoIndex') +geo_index_fixed.create_spatial_index() +print(f'Data Count of the GeoIndex layer after fixing geometries ' + f'({geo_index_fixed.layer_name}): {geo_index_fixed.data_count}') +geo_index_fixed.clip_layer(montreal_boundary.layer_path, + output_paths['Clipped Fixed GeoIndex']) +geo_index_clipped = \ + ScrubLayer(qgis_path, + output_paths['Clipped Fixed GeoIndex'], 'Clipped Fixed GeoIndex') +geo_index_clipped.create_spatial_index() +print(f'Data Count of the {geo_index_fixed.layer_name} ' + f' after clipping it ' + f'based on the Montreal Boundary layer ({geo_index_clipped.layer_name}): ' + f'{geo_index_clipped.layer.featureCount()}') + +# Processing the Property Assessment layer includes a pairwise clip, and +# two spatial join with NRCan and GeoIndex layers, respectively + +print(f'Data Count of the Property Assessment layer:' + f' {property_assessment.data_count}') +property_assessment.create_spatial_index() + +# For the pairwise clip, number of overlaying layers can be chosen +# (meaning number of splits for NRCan layer). This improves the performance +# where may increase duplicates. This has been done because using the NRCan +# layer as a whole causes crashing the clipping process. + +# First we split the overlaying layers into our desired number +nrcan_fixed.split_layer(120, output_paths['Splitted NRCans']) + +# Clipping have to be done in +clipping_property_assessment = """ +from input_paths_and_layers import * + +property_assessment.clip_by_multiple( + 120, output_paths['Splitted NRCans'], + output_paths['Pairwise Clipped Property Assessment Partitions'])""" + +exec(clipping_property_assessment) + +property_assessment.merge_layers( + output_paths['Pairwise Clipped Property Assessment Partitions'], + output_paths['Pairwise Clipped Merged Property Assessment']) + +clipped_property_assessment = ScrubLayer( + qgis_path, + output_paths['Pairwise Clipped Merged Property Assessment'], + 'Clipped Property Assessment') + +print(f'Data Count of the Property Assessment layer ' + f'after the pairwise clipped with NRCan layer ' + f'({clipped_property_assessment.layer_name}):' + f' {clipped_property_assessment.data_count}') +clipped_property_assessment.create_spatial_index() + +clipped_property_assessment.spatial_join( + nrcan_fixed.layer_path, + output_paths['Property Assessment and NRCan']) + +property_assessment_nrcan = ScrubLayer( + qgis_path, + output_paths['Property Assessment and NRCan'], + 'Property Assessment and NRCan') + +print(f'Data Count of the Property Assessment layer ' + f'after its spatial join with NRCan layer ' + f'({property_assessment_nrcan.layer_name}):' + f' {property_assessment_nrcan.data_count}') +property_assessment_nrcan.create_spatial_index() + +property_assessment_nrcan.spatial_join( + geo_index_clipped, + output_paths['Property Assessment and NRCan and GeoIndex']) + +property_assessment_nrcan_geo = ScrubLayer( + qgis_path, + output_paths['Property Assessment and NRCan and GeoIndex'], + 'Property Assessment and NRCan and GeoIndex') + +print(f'Data Count of the Property Assessment-NRCan layer ' + f'after its spatial join with GeoIndex layer:' + f' {property_assessment_nrcan_geo.data_count}\n' + f'Name of the new layer: ' + f'{property_assessment_nrcan_geo.layer_name}') +property_assessment_nrcan_geo.create_spatial_index() + +property_assessment_nrcan_geo.delete_duplicates( + output_paths['Deleted Duplicates Layer']) + +deleted_dups_layer = ScrubLayer( + qgis_path, + output_paths['Deleted Duplicates Layer'], + 'Deleted Duplicates Layer') + +print(f'Data Count of the Property Assessment-NRCan-GeoIndex layer ' + f'after deleting duplicates: {deleted_dups_layer.data_count}\n' + f'Name of the new layer: ' + f'{deleted_dups_layer.layer_name}') +property_assessment_nrcan_geo.create_spatial_index() + +property_assessment_nrcan_geo.multipart_to_singleparts( + output_paths['Single Parts Layer']) + +single_parts_layer = ScrubLayer( + qgis_path, + output_paths['Single Parts Layer'], + 'Single Parts Layer') + +print(f'Data Count of the Deleted Duplicates layer ' + f'after separating multiparts to clickable single parts: ' + f'{single_parts_layer.data_count}\n' + f'Name of the new layer: ' + f'{single_parts_layer.layer_name}') +single_parts_layer.create_spatial_index() + +# The process of instantiating a new layer +# (including print and create part) can be a method +# Also experiment importing input_paths_and_layers diff --git a/input_paths_and_layers.py b/input_paths_and_layers.py new file mode 100644 index 0000000..e69de29