progress bar

Makefile

@@ -2,7 +2,7 @@ all: install data train eval
 
 city_distances_full.csv: check generate_data.py
 	@echo "Generating distance data..."
-	@bash -c 'time python generate_data.py --country US --workers 8 --chunk-size 4200'
+	@bash -c 'time python generate_data.py --country US --workers 8 --chunk-size 8000'
 
 data: city_distances_full.csv
 

generate_data.py

@@ -34,11 +34,6 @@ parser.add_argument(
     type=str,
     default="city_distances_full.csv",
 )
-parser.add_argument(
-    "--shuffle",
-    action="store_true",
-    help="Option to shuffle combinations list before iterating over it",
-)
 args = parser.parse_args()
 
 
@@ -121,8 +116,6 @@ def main():
     cities = list(us_cities.values())
     print(f"Num cities: {len(cities)}")
     city_combinations = list(itertools.combinations(cities, 2))
-    if args.shuffle:
-        np.random.shuffle(city_combinations)
     chunk_size = args.chunk_size
     num_chunks = len(city_combinations) // chunk_size + 1
     output_file = args.output_file