citybert/generate_data.py

import argparse
import concurrent.futures
import csv
import itertools
from concurrent.futures import as_completed
from functools import lru_cache

import geonamescache
import numpy as np
from geopy.distance import geodesic
from tqdm import tqdm

MAX_DISTANCE = 20_037.5

# Add argparse
parser = argparse.ArgumentParser()
parser.add_argument(
    "-c", "--country", help="Specify the country code", type=str, default="US"
)
parser.add_argument(
    "-w", "--workers", help="Specify the number of workers", type=int, default=1
)
parser.add_argument(
    "-s",
    "--chunk-size",
    help="Specify chunk size for batching calculations",
    type=int,
    default=1000,
)
parser.add_argument(
    "-o",
    "--output-file",
    help="Specify the name of the output file (file.csv)",
    type=str,
    default="city_distances_full.csv",
)
args = parser.parse_args()


gc = geonamescache.GeonamesCache()
cities = gc.get_cities()
us_cities = {
    k: c
    for k, c in cities.items()
    if (c.get("countrycode") == args.country)  # & (c.get("population", 0) > 5e4)
}


@lru_cache(maxsize=None)
def get_coordinates(city_name, country_code="US"):
    """
    Get the coordinates of a city.

    Parameters
    ----------
    city_name : str
        The name of the city.
    country_code : str, optional
        The country code of the city, by default 'US'.

    Returns
    -------
    tuple
        A tuple containing the latitude and longitude of the city,
        or None if the city is not found.
    """
    search_results = gc.search_cities(city_name, case_sensitive=True)
    search_results = [
        d for d in search_results if (d.get("countrycode") == country_code)
    ]

    if not search_results:
        return None
    populations = [city.get("population") for city in search_results]
    city = search_results[np.argmax(populations)]
    return city.get("latitude"), city.get("longitude")


def get_distance(city1, city2, country1="US", country2="US"):
    """
    Get the distance between two cities in kilometers.

    Parameters
    ----------
    city1 : str
        The name of the first city.
    city2 : str
        The name of the second city.
    country1 : str, optional
        The country code of the first city, by default 'US'.
    country2 : str, optional
        The country code of the second city, by default 'US'.

    Returns
    -------
    float
        The distance between the two cities in kilometers,
        or None if one or both city names were not found.
    """
    city1_coords = get_coordinates(city1, country1)
    city2_coords = get_coordinates(city2, country2)

    if (city1_coords is None) or (city2_coords is None):
        return None

    return geodesic(city1_coords, city2_coords).km


def calculate_distance(pair):
    city1, city2 = pair
    distance = get_distance(city1["name"], city2["name"])
    return city1["name"], city2["name"], distance


def main():
    cities = list(us_cities.values())
    print(f"Num cities: {len(cities)}")
    city_combinations = list(itertools.combinations(cities, 2))
    # np.random.shuffle(city_combinations)  # will this help or hurt caching? 1.03it/s
    chunk_size = args.chunk_size
    num_chunks = len(city_combinations) // chunk_size + 1
    output_file = args.output_file

    with open(output_file, "w", newline="") as csvfile:
        fieldnames = ["city_from", "city_to", "distance"]
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
        writer.writeheader()

        try:
            executor = concurrent.futures.ProcessPoolExecutor(max_workers=args.workers)
            for i in tqdm(
                range(num_chunks),
                total=num_chunks,
                desc="Processing chunks",
                ncols=100,
                bar_format="{l_bar}{bar}{r_bar}",
            ):
                chunk = city_combinations[(i * chunk_size) : (i + 1) * chunk_size]
                futures = {
                    executor.submit(calculate_distance, pair): pair for pair in chunk
                }
                for future in as_completed(futures):
                    city_from, city_to, distance = future.result()
                    if distance is not None:
                        writer.writerow(
                            {
                                "city_from": city_from,
                                "city_to": city_to,
                                "distance": distance,
                            }
                        )
                        csvfile.flush()  # write to disk immediately
        except KeyboardInterrupt:
            print("Interrupted. Terminating processes...")
            executor.shutdown(wait=False)
            raise SystemExit("Execution terminated by user.")


if __name__ == "__main__":
    main()