k_nejadshamsi/zele-utils
1
0
Fork 0
mirror of https://github.com/knejadshamsi/zele-utils.git synced 2024-11-14 17:40:28 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

metro and bus functions fixed

Browse Source
This commit is contained in:
Kian 2024-09-16 08:38:38 -04:00
parent 6d479d8c07
commit df3b49a94a
6 changed files with 90 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
functions/__Init__.py
View File

@ -2,7 +2,7 @@ from .population import process_buffer_population, process_travels
from .network import process_nodes, process_links, process_links_attr from .network import process_nodes, process_links, process_links_attr
from .metro import metro_processing from .metro import metro_processing
from .bus import bus_processing from .bus import bus_processing
from .helpers import file_validator,buffer_creator,push_to_db_coords,write_to_csv, push_to_db_linestring from .helpers import file_validator,buffer_creator,push_to_db_coords,write_to_csv, push_to_db_linestring, push_to_db
from .printers import error_printer,success_printer,info_printer,notice_printer from .printers import error_printer,success_printer,info_printer,notice_printer
__all__ = [ __all__ = [
@ -10,6 +10,6 @@ __all__ = [
'process_nodes', 'process_links', 'process_links_attr', 'process_nodes', 'process_links', 'process_links_attr',
'metro_processing', 'metro_processing',
'bus_processing', 'bus_processing',
'file_validator','buffer_creator','push_to_db_coords', 'write_to_csv', 'push_to_db_linestring' 'file_validator','buffer_creator','push_to_db_coords', 'write_to_csv', 'push_to_db_linestring', 'push_to_db',
'error_printer','success_printer','info_printer', 'notice_printer', 'error_printer','success_printer','info_printer', 'notice_printer',
] ]

27
functions/bus.py
View File

@ -1,32 +1,33 @@
from .printers import error_printer, success_printer from .printers import error_printer, success_printer
import geopandas, typer import geopandas, typer
def bus_processing(city, files): def bus_processing(city, file):
df_stm_arrets_sig = None
df_stm_lignes_sig = None df = None
file_name = file.stem
if city == "mtl": if city == "mtl":
required_files = ["stm_arrets_sig.shp", "stm_lignes_sig.shp"] required_files = ["stm_arrets_sig", "stm_lignes_sig"]
common_files = [file for file in files if file.name in required_files] if file_name not in required_files:
if len(common_files) == 0:
error_printer("Incorrect file input") error_printer("Incorrect file input")
raise typer.Exit() return None
for file in common_files: if file_name == required_files[0]:
if file.name == required_files[0]:
try: try:
df_arrets = geopandas.read_file(file) df_arrets = geopandas.read_file(file)
df_arrets_filtered = df_arrets[~df_arrets['stop_url'].str.contains('metro', case=False, na=False)] df_arrets_filtered = df_arrets[~df_arrets['stop_url'].str.contains('metro', case=False, na=False)]
df_arrets_filtered = df_arrets_filtered.rename(columns={'geometry': 'coordinates'}) df_arrets_filtered = df_arrets_filtered.rename(columns={'geometry': 'coordinates'})
df_stm_arrets_sig = df_arrets_filtered df = df_arrets_filtered
except Exception as e: except Exception as e:
error_printer(f"Failed to process stm_arrets_sig.shp: {e}") error_printer(f"Failed to process stm_arrets_sig.shp: {e}")
elif file.name == required_files[1]: raise typer.Exit()
elif file_name == required_files[1]:
try: try:
df_lignes = geopandas.read_file(file) df_lignes = geopandas.read_file(file)
df_lignes_filtered = df_lignes[~df_lignes['route_name'].str.contains('Ligne', na=False)] df_lignes_filtered = df_lignes[~df_lignes['route_name'].str.contains('Ligne', na=False)]
df_stm_lignes_sig = df_lignes_filtered df = df_lignes_filtered
except Exception as e: except Exception as e:
error_printer(f"Failed to process stm_lignes_sig.shp: {e}") error_printer(f"Failed to process stm_lignes_sig.shp: {e}")
raise typer.Exit()
return df_stm_arrets_sig, df_stm_lignes_sig return df

6
functions/helpers.py
View File

@ -75,6 +75,7 @@ def push_to_db_linestring(name, data, mode):
) )
def write_to_csv(name,data, file): def write_to_csv(name,data, file):
print(file)
directory = file.parent directory = file.parent
id = datetime.datetime.now().strftime("%Y%m%d") id = datetime.datetime.now().strftime("%Y%m%d")
csv = directory / (file.stem + f"-{name}-{id}.csv") csv = directory / (file.stem + f"-{name}-{id}.csv")
@ -82,3 +83,8 @@ def write_to_csv(name,data, file):
data.to_csv(csv, mode='a',index=False) data.to_csv(csv, mode='a',index=False)
else: else:
data.to_csv(csv,index=False) data.to_csv(csv,index=False)
def push_to_db(name,df,pushmode):
name_type = name.split("-")
if "stations" in name_type: push_to_db_coords(name,df,pushmode)
if "lines" in name_type: push_to_db_linestring(name,df,pushmode)

27
functions/metro.py
View File

@ -1,34 +1,33 @@
from .printers import error_printer, success_printer from .printers import error_printer, success_printer
import geopandas, typer import geopandas, typer
def metro_processing(city, files): def metro_processing(city, file):
df_stm_arrets_sig = None df = None
df_stm_lignes_sig = None file_name = file.stem
if city == "mtl": if city == "mtl":
required_files = ["stm_arrets_sig.shp", "stm_lignes_sig.shp"] required_files = ["stm_arrets_sig", "stm_lignes_sig"]
common_files = [file for file in files if file.stem in required_files] if file_name not in required_files:
if len(common_files) == 0:
error_printer("Incorrect file input") error_printer("Incorrect file input")
raise typer.Exit() return None
for file in common_files: if file_name == required_files[0]:
if file == required_files[0]:
try: try:
df_arrets = geopandas.read_file(file) df_arrets = geopandas.read_file(file)
df_arrets_filtered = df_arrets[df_arrets['stop_url'].str.contains('metro', case=False, na=False)] df_arrets_filtered = df_arrets[df_arrets['stop_url'].str.contains('metro', case=False, na=False)]
df_arrets_filtered = df_arrets_filtered.rename(columns={'geometry': 'coordinates'}) df_arrets_filtered = df_arrets_filtered.rename(columns={'geometry': 'coordinates'})
df_stm_arrets_sig = df_arrets_filtered df = df_arrets_filtered
except Exception as e: except Exception as e:
error_printer(f"Failed to process stm_arrets_sig.shp: {e}") error_printer(f"Failed to process stm_arrets_sig.shp: {e}")
elif file == required_files[1]: raise typer.Exit()
elif file_name == required_files[1]:
try: try:
df_lignes = geopandas.read_file(file) df_lignes = geopandas.read_file(file)
df_lignes_filtered = df_lignes[df_lignes['route_name'].str.contains('Ligne', na=False)] df_lignes_filtered = df_lignes[df_lignes['route_name'].str.contains('Ligne', na=False)]
df_stm_lignes_sig = df_lignes_filtered df = df_lignes_filtered
except Exception as e: except Exception as e:
error_printer(f"Failed to process stm_lignes_sig.shp: {e}") error_printer(f"Failed to process stm_lignes_sig.shp: {e}")
raise typer.Exit()
return df_stm_arrets_sig, df_stm_lignes_sig return df

72
main.py
View File

@ -11,7 +11,7 @@ from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskPr
import time import time
from classes import City, DBMode, RTMode from classes import City, DBMode, RTMode
from functions import file_validator,buffer_creator, process_buffer_population,process_travels, push_to_db_coords, write_to_csv, push_to_db_linestring from functions import file_validator,buffer_creator, process_buffer_population,process_travels, push_to_db_coords, write_to_csv, push_to_db
from functions import process_nodes,process_links,process_links_attr from functions import process_nodes,process_links,process_links_attr
from functions import error_printer,success_printer,info_printer,notice_printer from functions import error_printer,success_printer,info_printer,notice_printer
from functions import metro_processing, bus_processing from functions import metro_processing, bus_processing
@ -219,71 +219,43 @@ def network(
@app.command() @app.command()
def metro( def metro(
city: Annotated[City, typer.Argument(..., help="Choose a city", show_default=False)], city: Annotated[City, typer.Argument(..., help="Choose a city", show_default=False)],
files: list[Path] = typer.Option(None, "--files", "-f", help="Provide the relative path to [yellow bold underline]shape files[/yellow bold underline].", show_default=False), files: list[Path] = typer.Argument(..., help="Provide the relative path to [yellow bold underline]shape files[/yellow bold underline].", show_default=False),
cleandata: bool = typer.Option(False, "--cleandata", "-cd", help="Drop the rows that have missing values."), cleandata: bool = typer.Option(False, "--cleandata", "-cd", help="Drop the rows that have missing values."),
push: bool = typer.Option(False, "--push", "-p", help="Save the output directly to the database When mentioned. Otherwise, saves as a [green bold]CSV file[/green bold] in the input directory"), push: bool = typer.Option(False, "--push", "-p", help="Save the output directly to the database When mentioned. Otherwise, saves as a [green bold]CSV file[/green bold] in the input directory"),
pushmode: Optional[DBMode] = typer.Option(None, help="Specify either [underline]'append'[/underline] or [underline]'drop'[/underline] when pushing data", show_default=False), pushmode: Optional[DBMode] = typer.Option(None, help="Specify either [underline]'append'[/underline] or [underline]'drop'[/underline] when pushing data", show_default=False),
): ):
required_files = ["stm_arrets_sig", "stm_lignes_sig"]
for file in files: for file in files:
if not file.exists(): table_name = "metro-stations" if file.stem == required_files[0] else "metro-lines"
error_printer(f"Shapefile {file} does not exist.")
raise typer.Exit()
if file.suffix != '.shp': if file.suffix != '.shp':
error_printer(f"File {file} is not a .shp file.") error_printer(f"File [bold]{file}[/bold] is not a .shp file.")
raise typer.Exit() raise typer.Exit()
success_printer("Shapefiles validated successfully.") success_printer(f"Validated: [bold]{file}[/bold]")
metro_stations_df, metro_lines_df = metro_processing(city, files) df = metro_processing(city, file)
if not metro_stations_df or not metro_lines_df: if df.empty:error_printer(f"processing [bold]{file}[/bold] failed"); raise typer.Exit()
error_printer("dataframes were processed successfully") if cleandata: df = df.dropna()
raise typer.Exit() if push: push_to_db(table_name,df,pushmode); continue
if cleandata: write_to_csv(table_name,df,file)
if metro_stations_df: metro_stations_df = metro_stations_df.dropna()
if metro_lines_df: metro_lines_df = metro_lines_df.dropna()
if push:
if metro_stations_df: push_to_db_coords("metro-stations",metro_stations_df,pushmode)
if metro_lines_df: push_to_db_linestring("metro-lines",metro_lines_df, pushmode)
else:
if metro_stations_df: write_to_csv("metro-stations",metro_stations_df,file)
if metro_lines_df: write_to_csv("metro-lines",metro_lines_df,file)
success_printer("Processing complete.")
@app.command() @app.command()
def bus( def bus(
city: Annotated[City, typer.Argument(..., help="Choose a city", show_default=False)], city: Annotated[City, typer.Argument(..., help="Choose a city", show_default=False)],
files: list[Path] = typer.Option(None, "--files", "-f", help="Provide the relative path to [yellow bold underline]shape files[/yellow bold underline].", show_default=False), files: list[Path] = typer.Argument(..., help="Provide the relative path to [yellow bold underline]shape files[/yellow bold underline].", show_default=False),
cleandata: bool = typer.Option(False, "--cleandata", "-cd", help="Drop the rows that have missing values."), cleandata: bool = typer.Option(False, "--cleandata", "-cd", help="Drop the rows that have missing values."),
push: bool = typer.Option(False, "--push", "-p", help="Save the output directly to the database when mentioned. Otherwise, saves as a [green bold]CSV file[/green bold] in the input directory"), push: bool = typer.Option(False, "--push", "-p", help="Save the output directly to the database when mentioned. Otherwise, saves as a [green bold]CSV file[/green bold] in the input directory"),
pushmode: Optional[DBMode] = typer.Option(None, help="Specify either [underline]'append'[/underline] or [underline]'drop'[/underline] when pushing data", show_default=False), pushmode: Optional[DBMode] = typer.Option(None, help="Specify either [underline]'append'[/underline] or [underline]'drop'[/underline] when pushing data", show_default=False),
): ):
required_files = ["stm_arrets_sig", "stm_lignes_sig"]
for file in files: for file in files:
if not file.exists(): table_name = "bus-stations" if file.stem == required_files[0] else "bus-lines"
error_printer(f"Shapefile {file} does not exist.") if file.suffix != '.shp':error_printer(f"File {file} is not a .shp file"); raise typer.Exit()
raise typer.Exit() success_printer("Shapefiles validated successfully")
if file.suffix != '.shp': df = bus_processing(city, file)
error_printer(f"File {file} is not a .shp file.") if df.empty: error_printer(f"processing [bold]{file}[/bold] failed"); raise typer.Exit()
raise typer.Exit() if cleandata: df.dropna()
success_printer("Shapefiles validated successfully.") if push: push_to_db(table_name,df,pushmode); continue
bus_stations_df, bus_lines_df = bus_processing(city, files) write_to_csv(table_name,df,file)
if not bus_stations_df and not bus_lines_df: success_printer(f"Processing {file} complete.")
error_printer("No dataframes were processed successfully.")
raise typer.Exit()
if cleandata:
if bus_stations_df is not None:
bus_stations_df = bus_stations_df.dropna()
if bus_lines_df is not None:
bus_lines_df = bus_lines_df.dropna()
if push:
if bus_stations_df is not None:
push_to_db_coords("bus-stations", bus_stations_df, pushmode)
if bus_lines_df is not None:
push_to_db_linestring("bus-lines", bus_lines_df, pushmode)
else:
if bus_stations_df is not None:
write_to_csv("bus-stations", bus_stations_df, file)
if bus_lines_df is not None:
write_to_csv("bus-lines", bus_lines_df, file)
success_printer("Processing complete.")
if __name__ == "__main__": if __name__ == "__main__":
app() app()

19
styles/help.py
View File

@ -84,7 +84,6 @@ def network_help():
) )
line3 = "The expected format of the input MATSim network XML file is as follows:" line3 = "The expected format of the input MATSim network XML file is as follows:"
# Code snippet with syntax highlighting
code = '''<network> code = '''<network>
<!-- ====================================================================== --> <!-- ====================================================================== -->
@ -189,24 +188,25 @@ def metro_help():
"[bright_cyan bold link=https://www.stm.info/en/about/developers?utm_campaign=menubas&utm_source=developpeurs]" "[bright_cyan bold link=https://www.stm.info/en/about/developers?utm_campaign=menubas&utm_source=developpeurs]"
"https://www.stm.info/en/about/developers[/bright_cyan bold link]\n" "https://www.stm.info/en/about/developers[/bright_cyan bold link]\n"
"Scroll down and find [bold]\"Download the Shapefile\"[/bold] button and download it.\n" "Scroll down and find [bold]\"Download the Shapefile\"[/bold] button and download it.\n"
"Extract the downloaded archive to obtain the two [bold].shp[/bold] files." "Extract the downloaded archive to obtain the [bold].shp[/bold] and other files.\n"
"Use the relative path to [bold].shp[/bold] as your [bold]files[/bold] argument."
) )
line4 = "The resulting table structures include columns such as the following:\n" line4 = "The resulting table structures include columns such as the following:\n"
# Example table for 'metro-stations' # Example table for 'metro-stations'
table_stations = Table( table_stations = Table(
"route_id", "route_name", "headsign", "shape_id", "ct", "service_id", "geometry" "stop_code","stop_id","stop_name","stop_url","wheelchair","route_id","loc_type","shelter","service_id","coordinates"
) )
table_stations.add_row( table_stations.add_row(
"1", "Ligne 1 - Verte", "Station Angrignon", "11072", "0", "24S", "LINESTRING (302040.087 5050741.446, 301735.98 ...)" "10118","43","Station Angrignon","http://www.stm.info/fr/infos/reseaux/metro/angrignon","1","1","0","NaN","24S","POINT (296733.6694496935 5034064.601946615)"
) )
# Example table for 'metro-lines' # Example table for 'metro-lines'
table_lines = Table( table_lines = Table(
"id", "name", "geom", "stop_url", "wheelchair", "route_id", "loc_type", "shelter", "service_id" "route_id","route_name","headsign","shape_id","ct","service_id","geometry"
) )
table_lines.add_row( table_lines.add_row(
"10118", "43", "Station Angrignon", "http://www.stm.info/fr/infos/reseaux/metro/ang...", "1", "1", "0", "NaN", "24S" "1","Ligne 1 - Verte","Station Angrignon","11072","0","24S","LINESTRING (302040.086988879 5050741.446280612...)"
) )
line5 = ( line5 = (
@ -245,16 +245,17 @@ def bus_help():
"[bright_cyan bold link=https://www.stm.info/en/about/developers?utm_campaign=menubas&utm_source=developpeurs]" "[bright_cyan bold link=https://www.stm.info/en/about/developers?utm_campaign=menubas&utm_source=developpeurs]"
"https://www.stm.info/en/about/developers[/bright_cyan bold link]\n" "https://www.stm.info/en/about/developers[/bright_cyan bold link]\n"
"Scroll down and find [bold]\"Download the Shapefile\"[/bold] button and download it.\n" "Scroll down and find [bold]\"Download the Shapefile\"[/bold] button and download it.\n"
"Extract the downloaded archive to obtain the two [bold].shp[/bold] files." "Extract the downloaded archive to obtain the [bold].shp[/bold] and related files.\n"
"Use the relative path to [bold].shp[/bold] as your [bold]files[/bold] argument."
) )
line4 = "The resulting table structures include columns such as the following:\n" line4 = "The resulting table structures include columns such as the following:\n"
# Example table for 'bus-stations' # Example table for 'bus-stations'
table_stations = Table( table_stations = Table(
"stop_code","stop_id","stop_name","stop_url","wheelchair","route_id","loc_type","shelter","service_id","geometry" "stop_code","stop_id","stop_name","stop_url","wheelchair","route_id","loc_type","shelter","service_id","coordinates"
) )
table_stations.add_row( table_stations.add_row(
"10282","66-01","Station Cartier - Terminus Cartier (A)","","1","","2","","24S","POINT (290599.75509576086 5046736.8438161155)" "10282","66-01","Station Cartier - Terminus Cartier (A)","NaN","1","NaN","2","NaN","24S","POINT (290599.75509576086 5046736.8438161155)"
) )
# Example table for 'bus-lines' # Example table for 'bus-lines'