From d9797385d9b037969b8fa70d03c42c166ec1a73f Mon Sep 17 00:00:00 2001 From: Tom Russell Date: Tue, 25 Sep 2018 21:47:29 +0100 Subject: [PATCH] Use default pool size (CPU count) in ETL --- etl/check_ab_mm_match.py | 2 +- etl/create_building_records.sh | 0 etl/filter_mastermap.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) mode change 100644 => 100755 etl/create_building_records.sh diff --git a/etl/check_ab_mm_match.py b/etl/check_ab_mm_match.py index 76c59686..a43dc509 100644 --- a/etl/check_ab_mm_match.py +++ b/etl/check_ab_mm_match.py @@ -22,7 +22,7 @@ def main(ab_path, mm_path): zipped_paths = zip(ab_paths, mm_paths) # parallel map over tiles - with Pool(4) as p: + with Pool() as p: p.starmap(check, zipped_paths) def check(ab_path, mm_path): diff --git a/etl/create_building_records.sh b/etl/create_building_records.sh old mode 100644 new mode 100755 diff --git a/etl/filter_mastermap.py b/etl/filter_mastermap.py index a32bde15..e35b3ea4 100644 --- a/etl/filter_mastermap.py +++ b/etl/filter_mastermap.py @@ -25,7 +25,7 @@ def main(ab_path, mm_path): zipped_paths = zip(mm_paths, toid_paths) # parallel map over tiles - with Pool(4) as p: + with Pool() as p: p.starmap(filter, zipped_paths) def filter(mm_path, toid_path):