diff --git a/functions/__Init__.py b/functions/__Init__.py index 2aafaf5..a969d7d 100644 --- a/functions/__Init__.py +++ b/functions/__Init__.py @@ -2,7 +2,7 @@ from .population import process_buffer_population, process_travels from .network import process_nodes, process_links, process_links_attr from .metro import metro_processing from .bus import bus_processing -from .helpers import file_validator,buffer_creator,push_to_db_coords,write_to_csv, push_to_db_linestring +from .helpers import file_validator,buffer_creator,push_to_db_coords,write_to_csv, push_to_db_linestring, push_to_db from .printers import error_printer,success_printer,info_printer,notice_printer __all__ = [ @@ -10,6 +10,6 @@ __all__ = [ 'process_nodes', 'process_links', 'process_links_attr', 'metro_processing', 'bus_processing', - 'file_validator','buffer_creator','push_to_db_coords', 'write_to_csv', 'push_to_db_linestring' + 'file_validator','buffer_creator','push_to_db_coords', 'write_to_csv', 'push_to_db_linestring', 'push_to_db', 'error_printer','success_printer','info_printer', 'notice_printer', ] diff --git a/functions/bus.py b/functions/bus.py index 7955e85..31f41a0 100644 --- a/functions/bus.py +++ b/functions/bus.py @@ -1,32 +1,33 @@ from .printers import error_printer, success_printer import geopandas, typer -def bus_processing(city, files): - df_stm_arrets_sig = None - df_stm_lignes_sig = None +def bus_processing(city, file): + + df = None + file_name = file.stem if city == "mtl": - required_files = ["stm_arrets_sig.shp", "stm_lignes_sig.shp"] - common_files = [file for file in files if file.name in required_files] - if len(common_files) == 0: + required_files = ["stm_arrets_sig", "stm_lignes_sig"] + if file_name not in required_files: error_printer("Incorrect file input") - raise typer.Exit() + return None - for file in common_files: - if file.name == required_files[0]: - try: - df_arrets = geopandas.read_file(file) - df_arrets_filtered = df_arrets[~df_arrets['stop_url'].str.contains('metro', case=False, na=False)] - df_arrets_filtered = df_arrets_filtered.rename(columns={'geometry': 'coordinates'}) - df_stm_arrets_sig = df_arrets_filtered - except Exception as e: - error_printer(f"Failed to process stm_arrets_sig.shp: {e}") - elif file.name == required_files[1]: - try: - df_lignes = geopandas.read_file(file) - df_lignes_filtered = df_lignes[~df_lignes['route_name'].str.contains('Ligne', na=False)] - df_stm_lignes_sig = df_lignes_filtered - except Exception as e: - error_printer(f"Failed to process stm_lignes_sig.shp: {e}") + if file_name == required_files[0]: + try: + df_arrets = geopandas.read_file(file) + df_arrets_filtered = df_arrets[~df_arrets['stop_url'].str.contains('metro', case=False, na=False)] + df_arrets_filtered = df_arrets_filtered.rename(columns={'geometry': 'coordinates'}) + df = df_arrets_filtered + except Exception as e: + error_printer(f"Failed to process stm_arrets_sig.shp: {e}") + raise typer.Exit() + elif file_name == required_files[1]: + try: + df_lignes = geopandas.read_file(file) + df_lignes_filtered = df_lignes[~df_lignes['route_name'].str.contains('Ligne', na=False)] + df = df_lignes_filtered + except Exception as e: + error_printer(f"Failed to process stm_lignes_sig.shp: {e}") + raise typer.Exit() - return df_stm_arrets_sig, df_stm_lignes_sig \ No newline at end of file + return df \ No newline at end of file diff --git a/functions/helpers.py b/functions/helpers.py index 8d50cd0..b180506 100644 --- a/functions/helpers.py +++ b/functions/helpers.py @@ -75,10 +75,16 @@ def push_to_db_linestring(name, data, mode): ) def write_to_csv(name,data, file): + print(file) directory = file.parent id = datetime.datetime.now().strftime("%Y%m%d") csv = directory / (file.stem + f"-{name}-{id}.csv") if csv.exists(): data.to_csv(csv, mode='a',index=False) else: - data.to_csv(csv,index=False) \ No newline at end of file + data.to_csv(csv,index=False) + +def push_to_db(name,df,pushmode): + name_type = name.split("-") + if "stations" in name_type: push_to_db_coords(name,df,pushmode) + if "lines" in name_type: push_to_db_linestring(name,df,pushmode) \ No newline at end of file diff --git a/functions/metro.py b/functions/metro.py index 2aa46ff..672e7d2 100644 --- a/functions/metro.py +++ b/functions/metro.py @@ -1,34 +1,33 @@ from .printers import error_printer, success_printer import geopandas, typer -def metro_processing(city, files): +def metro_processing(city, file): - df_stm_arrets_sig = None - df_stm_lignes_sig = None + df = None + file_name = file.stem if city == "mtl": - required_files = ["stm_arrets_sig.shp", "stm_lignes_sig.shp"] - common_files = [file for file in files if file.stem in required_files] - - if len(common_files) == 0: + required_files = ["stm_arrets_sig", "stm_lignes_sig"] + if file_name not in required_files: error_printer("Incorrect file input") - raise typer.Exit() + return None - for file in common_files: - if file == required_files[0]: - try: - df_arrets = geopandas.read_file(file) - df_arrets_filtered = df_arrets[df_arrets['stop_url'].str.contains('metro', case=False, na=False)] - df_arrets_filtered = df_arrets_filtered.rename(columns={'geometry': 'coordinates'}) - df_stm_arrets_sig = df_arrets_filtered - except Exception as e: - error_printer(f"Failed to process stm_arrets_sig.shp: {e}") - elif file == required_files[1]: - try: - df_lignes = geopandas.read_file(file) - df_lignes_filtered = df_lignes[df_lignes['route_name'].str.contains('Ligne', na=False)] - df_stm_lignes_sig = df_lignes_filtered - except Exception as e: - error_printer(f"Failed to process stm_lignes_sig.shp: {e}") + if file_name == required_files[0]: + try: + df_arrets = geopandas.read_file(file) + df_arrets_filtered = df_arrets[df_arrets['stop_url'].str.contains('metro', case=False, na=False)] + df_arrets_filtered = df_arrets_filtered.rename(columns={'geometry': 'coordinates'}) + df = df_arrets_filtered + except Exception as e: + error_printer(f"Failed to process stm_arrets_sig.shp: {e}") + raise typer.Exit() + elif file_name == required_files[1]: + try: + df_lignes = geopandas.read_file(file) + df_lignes_filtered = df_lignes[df_lignes['route_name'].str.contains('Ligne', na=False)] + df = df_lignes_filtered + except Exception as e: + error_printer(f"Failed to process stm_lignes_sig.shp: {e}") + raise typer.Exit() - return df_stm_arrets_sig, df_stm_lignes_sig \ No newline at end of file + return df \ No newline at end of file diff --git a/main.py b/main.py index 786263b..061d280 100644 --- a/main.py +++ b/main.py @@ -11,7 +11,7 @@ from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskPr import time from classes import City, DBMode, RTMode -from functions import file_validator,buffer_creator, process_buffer_population,process_travels, push_to_db_coords, write_to_csv, push_to_db_linestring +from functions import file_validator,buffer_creator, process_buffer_population,process_travels, push_to_db_coords, write_to_csv, push_to_db from functions import process_nodes,process_links,process_links_attr from functions import error_printer,success_printer,info_printer,notice_printer from functions import metro_processing, bus_processing @@ -219,71 +219,43 @@ def network( @app.command() def metro( city: Annotated[City, typer.Argument(..., help="Choose a city", show_default=False)], - files: list[Path] = typer.Option(None, "--files", "-f", help="Provide the relative path to [yellow bold underline]shape files[/yellow bold underline].", show_default=False), + files: list[Path] = typer.Argument(..., help="Provide the relative path to [yellow bold underline]shape files[/yellow bold underline].", show_default=False), cleandata: bool = typer.Option(False, "--cleandata", "-cd", help="Drop the rows that have missing values."), push: bool = typer.Option(False, "--push", "-p", help="Save the output directly to the database When mentioned. Otherwise, saves as a [green bold]CSV file[/green bold] in the input directory"), pushmode: Optional[DBMode] = typer.Option(None, help="Specify either [underline]'append'[/underline] or [underline]'drop'[/underline] when pushing data", show_default=False), ): + required_files = ["stm_arrets_sig", "stm_lignes_sig"] for file in files: - if not file.exists(): - error_printer(f"Shapefile {file} does not exist.") - raise typer.Exit() + table_name = "metro-stations" if file.stem == required_files[0] else "metro-lines" if file.suffix != '.shp': - error_printer(f"File {file} is not a .shp file.") + error_printer(f"File [bold]{file}[/bold] is not a .shp file.") raise typer.Exit() - success_printer("Shapefiles validated successfully.") - metro_stations_df, metro_lines_df = metro_processing(city, files) - if not metro_stations_df or not metro_lines_df: - error_printer("dataframes were processed successfully") - raise typer.Exit() - if cleandata: - if metro_stations_df: metro_stations_df = metro_stations_df.dropna() - if metro_lines_df: metro_lines_df = metro_lines_df.dropna() - if push: - if metro_stations_df: push_to_db_coords("metro-stations",metro_stations_df,pushmode) - if metro_lines_df: push_to_db_linestring("metro-lines",metro_lines_df, pushmode) - else: - if metro_stations_df: write_to_csv("metro-stations",metro_stations_df,file) - if metro_lines_df: write_to_csv("metro-lines",metro_lines_df,file) - success_printer("Processing complete.") + success_printer(f"Validated: [bold]{file}[/bold]") + df = metro_processing(city, file) + if df.empty:error_printer(f"processing [bold]{file}[/bold] failed"); raise typer.Exit() + if cleandata: df = df.dropna() + if push: push_to_db(table_name,df,pushmode); continue + write_to_csv(table_name,df,file) @app.command() def bus( city: Annotated[City, typer.Argument(..., help="Choose a city", show_default=False)], - files: list[Path] = typer.Option(None, "--files", "-f", help="Provide the relative path to [yellow bold underline]shape files[/yellow bold underline].", show_default=False), + files: list[Path] = typer.Argument(..., help="Provide the relative path to [yellow bold underline]shape files[/yellow bold underline].", show_default=False), cleandata: bool = typer.Option(False, "--cleandata", "-cd", help="Drop the rows that have missing values."), push: bool = typer.Option(False, "--push", "-p", help="Save the output directly to the database when mentioned. Otherwise, saves as a [green bold]CSV file[/green bold] in the input directory"), pushmode: Optional[DBMode] = typer.Option(None, help="Specify either [underline]'append'[/underline] or [underline]'drop'[/underline] when pushing data", show_default=False), ): + required_files = ["stm_arrets_sig", "stm_lignes_sig"] for file in files: - if not file.exists(): - error_printer(f"Shapefile {file} does not exist.") - raise typer.Exit() - if file.suffix != '.shp': - error_printer(f"File {file} is not a .shp file.") - raise typer.Exit() - success_printer("Shapefiles validated successfully.") - bus_stations_df, bus_lines_df = bus_processing(city, files) - if not bus_stations_df and not bus_lines_df: - error_printer("No dataframes were processed successfully.") - raise typer.Exit() - if cleandata: - if bus_stations_df is not None: - bus_stations_df = bus_stations_df.dropna() - if bus_lines_df is not None: - bus_lines_df = bus_lines_df.dropna() - if push: - if bus_stations_df is not None: - push_to_db_coords("bus-stations", bus_stations_df, pushmode) - if bus_lines_df is not None: - push_to_db_linestring("bus-lines", bus_lines_df, pushmode) - else: - if bus_stations_df is not None: - write_to_csv("bus-stations", bus_stations_df, file) - if bus_lines_df is not None: - write_to_csv("bus-lines", bus_lines_df, file) - success_printer("Processing complete.") - + table_name = "bus-stations" if file.stem == required_files[0] else "bus-lines" + if file.suffix != '.shp':error_printer(f"File {file} is not a .shp file"); raise typer.Exit() + success_printer("Shapefiles validated successfully") + df = bus_processing(city, file) + if df.empty: error_printer(f"processing [bold]{file}[/bold] failed"); raise typer.Exit() + if cleandata: df.dropna() + if push: push_to_db(table_name,df,pushmode); continue + write_to_csv(table_name,df,file) + success_printer(f"Processing {file} complete.") if __name__ == "__main__": app() \ No newline at end of file diff --git a/styles/help.py b/styles/help.py index 08db1b3..2167c56 100644 --- a/styles/help.py +++ b/styles/help.py @@ -84,7 +84,6 @@ def network_help(): ) line3 = "The expected format of the input MATSim network XML file is as follows:" - # Code snippet with syntax highlighting code = ''' @@ -189,24 +188,25 @@ def metro_help(): "[bright_cyan bold link=https://www.stm.info/en/about/developers?utm_campaign=menubas&utm_source=developpeurs]" "https://www.stm.info/en/about/developers[/bright_cyan bold link]\n" "Scroll down and find [bold]\"Download the Shapefile\"[/bold] button and download it.\n" - "Extract the downloaded archive to obtain the two [bold].shp[/bold] files." + "Extract the downloaded archive to obtain the [bold].shp[/bold] and other files.\n" + "Use the relative path to [bold].shp[/bold] as your [bold]files[/bold] argument." ) line4 = "The resulting table structures include columns such as the following:\n" # Example table for 'metro-stations' table_stations = Table( - "route_id", "route_name", "headsign", "shape_id", "ct", "service_id", "geometry" + "stop_code","stop_id","stop_name","stop_url","wheelchair","route_id","loc_type","shelter","service_id","coordinates" ) table_stations.add_row( - "1", "Ligne 1 - Verte", "Station Angrignon", "11072", "0", "24S", "LINESTRING (302040.087 5050741.446, 301735.98 ...)" + "10118","43","Station Angrignon","http://www.stm.info/fr/infos/reseaux/metro/angrignon","1","1","0","NaN","24S","POINT (296733.6694496935 5034064.601946615)" ) # Example table for 'metro-lines' table_lines = Table( - "id", "name", "geom", "stop_url", "wheelchair", "route_id", "loc_type", "shelter", "service_id" + "route_id","route_name","headsign","shape_id","ct","service_id","geometry" ) table_lines.add_row( - "10118", "43", "Station Angrignon", "http://www.stm.info/fr/infos/reseaux/metro/ang...", "1", "1", "0", "NaN", "24S" + "1","Ligne 1 - Verte","Station Angrignon","11072","0","24S","LINESTRING (302040.086988879 5050741.446280612...)" ) line5 = ( @@ -245,16 +245,17 @@ def bus_help(): "[bright_cyan bold link=https://www.stm.info/en/about/developers?utm_campaign=menubas&utm_source=developpeurs]" "https://www.stm.info/en/about/developers[/bright_cyan bold link]\n" "Scroll down and find [bold]\"Download the Shapefile\"[/bold] button and download it.\n" - "Extract the downloaded archive to obtain the two [bold].shp[/bold] files." + "Extract the downloaded archive to obtain the [bold].shp[/bold] and related files.\n" + "Use the relative path to [bold].shp[/bold] as your [bold]files[/bold] argument." ) line4 = "The resulting table structures include columns such as the following:\n" # Example table for 'bus-stations' table_stations = Table( - "stop_code","stop_id","stop_name","stop_url","wheelchair","route_id","loc_type","shelter","service_id","geometry" + "stop_code","stop_id","stop_name","stop_url","wheelchair","route_id","loc_type","shelter","service_id","coordinates" ) table_stations.add_row( - "10282","66-01","Station Cartier - Terminus Cartier (A)","","1","","2","","24S","POINT (290599.75509576086 5046736.8438161155)" + "10282","66-01","Station Cartier - Terminus Cartier (A)","NaN","1","NaN","2","NaN","24S","POINT (290599.75509576086 5046736.8438161155)" ) # Example table for 'bus-lines'