<reponame>edesz/chicago-bikeshare
#!/usr/bin/env python3
# -*- coding: utf-8 -*-


import os
import re
import shutil
from io import BytesIO
from multiprocessing import cpu_count
from urllib.request import urlopen
from zipfile import ZipFile

from joblib import Parallel, delayed


def get_bikeshare_file(
    fname, raw_data_path, divvy_trips_base_url, verbose=False
):
    files_to_move = []
    if verbose:
        print(f"Downloading {fname} to {raw_data_path}...", end="")
    try:
        url = os.path.join(divvy_trips_base_url, fname)
        with urlopen(url) as zipresp:
            # print(raw_data_path)
            with ZipFile(BytesIO(zipresp.read())) as zfile:
                zfile.extractall(raw_data_path)
    except Exception as e:
        print(f"{url} gave error - {str(e)}")
    else:
        if verbose:
            print("done.\n")
    fname_no_ext = os.path.splitext(fname)[0]
    if (
        "2014_Q3Q4" in fname
        or "2016_Q1Q2" in fname
        or "Divvy_Stations_Trips_2013" in fname
    ):
        files_to_move.append(fname_no_ext)
    return files_to_move


def get_bikeshare_files(
    raw_data_path,
    bikeshare_trips_fnames,
    divvy_trips_base_url,
    use_parallel=False,
):
    if use_parallel:
        executor = Parallel(n_jobs=cpu_count(), backend="multiprocessing")
        tasks = (
            delayed(get_bikeshare_file)(
                fname, raw_data_path, divvy_trips_base_url, False
            )
            for fname in bikeshare_trips_fnames
        )
        files_to_move = executor(tasks)
    else:
        files_to_move = [
            get_bikeshare_file(
                fname, raw_data_path, divvy_trips_base_url, verbose=True
            )
            for fname in bikeshare_trips_fnames
        ]
    files_to_move = [x for x in files_to_move if x != []]
    return files_to_move


def move_files_by_year(f, raw_data_path, year, verbose=False):
    if os.path.isfile(f):
        shutil.move(f, os.path.join(raw_data_path, str(year)))
        if verbose:
            print(f"Moved {f}")


def move_2020_2021_files(files_to_move, raw_data_path, verbose=False):
    executor = Parallel(n_jobs=cpu_count(), backend="multiprocessing")
    tasks = (
        delayed(move_files_by_year)(
            f,
            raw_data_path,
            re.search(r"(\d{4})", os.path.basename(f)).group(1),
            verbose,
        )
        for f in files_to_move
    )
    _ = executor(tasks)

#!/usr/bin/env python3
# -*- coding: utf-8 -*-


import os
import re
import shutil
from io import BytesIO
from multiprocessing import cpu_count
from urllib.request import urlopen
from zipfile import ZipFile

from joblib import Parallel, delayed


def get_bikeshare_file(
    fname, raw_data_path, divvy_trips_base_url, verbose=False
):
    files_to_move = []
    if verbose:
        print(f"Downloading {fname} to {raw_data_path}...", end="")
    try:
        url = os.path.join(divvy_trips_base_url, fname)
        with urlopen(url) as zipresp:
            # print(raw_data_path)
            with ZipFile(BytesIO(zipresp.read())) as zfile:
                zfile.extractall(raw_data_path)
    except Exception as e:
        print(f"{url} gave error - {str(e)}")
    else:
        if verbose:
            print("done.\n")
    fname_no_ext = os.path.splitext(fname)[0]
    if (
        "2014_Q3Q4" in fname
        or "2016_Q1Q2" in fname
        or "Divvy_Stations_Trips_2013" in fname
    ):
        files_to_move.append(fname_no_ext)
    return files_to_move


def get_bikeshare_files(
    raw_data_path,
    bikeshare_trips_fnames,
    divvy_trips_base_url,
    use_parallel=False,
):
    if use_parallel:
        executor = Parallel(n_jobs=cpu_count(), backend="multiprocessing")
        tasks = (
            delayed(get_bikeshare_file)(
                fname, raw_data_path, divvy_trips_base_url, False
            )
            for fname in bikeshare_trips_fnames
        )
        files_to_move = executor(tasks)
    else:
        files_to_move = [
            get_bikeshare_file(
                fname, raw_data_path, divvy_trips_base_url, verbose=True
            )
            for fname in bikeshare_trips_fnames
        ]
    files_to_move = [x for x in files_to_move if x != []]
    return files_to_move


def move_files_by_year(f, raw_data_path, year, verbose=False):
    if os.path.isfile(f):
        shutil.move(f, os.path.join(raw_data_path, str(year)))
        if verbose:
            print(f"Moved {f}")


def move_2020_2021_files(files_to_move, raw_data_path, verbose=False):
    executor = Parallel(n_jobs=cpu_count(), backend="multiprocessing")
    tasks = (
        delayed(move_files_by_year)(
            f,
            raw_data_path,
            re.search(r"(\d{4})", os.path.basename(f)).group(1),
            verbose,
        )
        for f in files_to_move
    )
    _ = executor(tasks)
#!/usr/bin/env python3 # -*- coding: utf-8 -*- # print(raw_data_path)