Add splitting the clipped nrcan process

This commit is contained in:
Alireza Adli 2024-02-26 10:51:24 -05:00
parent c86f4dc966
commit 321b9f4ff5

View File

@ -1,6 +1,7 @@
from qgis.core import QgsApplication, QgsVectorLayer, QgsProject, QgsProcessingFeedback
from qgis.analysis import QgsNativeAlgorithms
import processing
import processing, os
from services_scripts.basic_functions import create_folders
# This function loads a layer. It's used to count the data records
@ -51,7 +52,8 @@ print(f'Clipping of {fixed_nrcan_name} is completed.')
# After each step, data records are counted
clipped_nrcan, clipped_nrcan_name = load_layer(clipped_nrcan_layer, 'Clipped NRCan')
print(f'{clipped_nrcan_name} data count: {clipped_nrcan.featureCount()}')
clipped_nrcan_length = clipped_nrcan.featureCount()
print(f'{clipped_nrcan_name} data count: {clipped_nrcan_length}')
# Creating spatial index is needed for most of the newly built layers.
@ -90,8 +92,8 @@ params_create_index_geoindex = {'INPUT': clipped_geoindex_layer, 'OUTPUT': 'Outp
processing.run("native:createspatialindex", params_create_index_geoindex)
print(f'Creating spatial index for {clipped_geoindex_name} is completed.')
# Reading the Property Assessment (uniteevaluationfonciere) dataset
property_assessment_layer = 'C:/Users/a_adli/PycharmProjects/hydroquebec_archetype_gispy/data/input_data/uniteevaluationfonciere/uniteevaluationfonciere.shp'
# Reading the Property Assessment (AKA uniteevaluationfonciere) dataset
property_assessment_layer = 'C:/Users/a_adli/PycharmProjects/hydroquebec_archetype_gispy/data/input_data/property_assessment/uniteevaluationfonciere.shp'
property_assessment_read, property_assessment_layer_name = load_layer(property_assessment_layer, 'Property Assesment')
print(f'{property_assessment_layer_name} data count: {property_assessment_read.featureCount()}')
@ -101,6 +103,38 @@ params_create_index_property_assessment = {'INPUT': property_assessment_layer, '
processing.run("native:createspatialindex", params_create_index_property_assessment)
print(f'Creating spatial index for {property_assessment_layer_name} is completed.')
intersection_nrcan_property_layer = 'C:/Users/a_adli/PycharmProjects/hydroquebec_archetype_gispy/data/tests/intersection_nrcan_property/intersection_nrcan_property.shp'
# For carrying out pairwise clip (clip in QGIS) on property assessment data with
# Clipped and fixed NRCan data as the overlay, we need to split the latter.
# This is being done to improve the performance of the clipping task. Otherwise, the task crashes the program.
# One can decide on the number of this divisions by assigning a number to num_layers. I assume, a number way bigger than
# 20 can help the performance even more significantly (Will be tested).
output_layers_folder = 'C:/Users/a_adli/PycharmProjects/hydroquebec_archetype_gispy/data/output_data/splitted_clipped_nrcan'
num_layers = 20
create_folders(output_layers_folder, num_layers)
splitting_intervals = clipped_nrcan_length // num_layers
for each in range(num_layers):
output_layer_path = f'C:/Users/a_adli/PycharmProjects/hydroquebec_archetype_gispy/data/output_data/splitted_clipped_nrcan/layer_{each}/layer_{each}.shp'
params = {'INPUT': clipped_nrcan_layer,
'EXPRESSION': f'$id >= {each * splitting_intervals} AND $id < {(each + 1) * splitting_intervals}\r\n',
'OUTPUT': output_layer_path}
processing.run("native:extractbyexpression", params)
# Creating spatian index for each new layer
create_spatial_indext_params = {'INPUT': output_layer_path, 'OUTPUT': 'Output'}
processing.run("native:createspatialindex", create_spatial_indext_params)
# Putting the remaining features in one last layer (So, overall we're going to have [num_layers + 1] layers).
remaining_features = num_layers
os.makedirs(f'C:/Users/a_adli/PycharmProjects/hydroquebec_archetype_gispy/data/aEndeavor/divided_all/layer_{remaining_features}')
output_layer_path = f'C:/Users/a_adli/PycharmProjects/hydroquebec_archetype_gispy/data/aEndeavor/divided_all/layer_{remaining_features}/layer_{remaining_features}.shp'
params = {'INPUT': clipped_nrcan_layer,
'EXPRESSION': f'$id >= {num_layers * splitting_intervals}\r\n',
'OUTPUT': output_layer_path}
processing.run("native:extractbyexpression", params)
qgs.exitQgis()