<filename>DSAI_proj/extract.py<gh_stars>0
# AUTOGENERATED! DO NOT EDIT! File to edit: 00_extract.ipynb (unless otherwise specified).

__all__ = ['extract_dataset_threaded']

# Internal Cell

from pandas import DataFrame
import os
import concurrent
import requests
import pandas as pd

def extract_dataset(api_key: str, n: int):
    req_url = f"https://api.themoviedb.org/3/movie/{n}?api_key={api_key}&language=en-US"
    response = requests.get(req_url)
    if response.status_code == 200:
        array = response.json()
        return array
    return

def save_to_csv(df: DataFrame,
                save_path: str,
                fname: str):
    df.to_csv(os.path.join(save_path, fname))
    return

# Cell

def extract_dataset_threaded(api_key: str,
                             max_ds_size: int,
                             max_threads: int,
                             save_path: str,
                             fname: str) -> DataFrame:

    max_threads = max_threads if max_threads < max_ds_size else max_ds_size
    with concurrent.futures.ThreadPoolExecutor(max_workers=max_threads) as executor:
        movies = [ex for n in range(max_ds_size) if (ex := executor.submit(extract_dataset, api_key, n).result()) is not None]
    movies = pd.DataFrame.from_records(movies)
    save_to_csv(df=movies, save_path=save_path, fname=fname)
    return movies
<filename>DSAI_proj/extract.py<gh_stars>0
# AUTOGENERATED! DO NOT EDIT! File to edit: 00_extract.ipynb (unless otherwise specified).

__all__ = ['extract_dataset_threaded']

# Internal Cell

from pandas import DataFrame
import os
import concurrent
import requests
import pandas as pd

def extract_dataset(api_key: str, n: int):
    req_url = f"https://api.themoviedb.org/3/movie/{n}?api_key={api_key}&language=en-US"
    response = requests.get(req_url)
    if response.status_code == 200:
        array = response.json()
        return array
    return

def save_to_csv(df: DataFrame,
                save_path: str,
                fname: str):
    df.to_csv(os.path.join(save_path, fname))
    return

# Cell

def extract_dataset_threaded(api_key: str,
                             max_ds_size: int,
                             max_threads: int,
                             save_path: str,
                             fname: str) -> DataFrame:

    max_threads = max_threads if max_threads < max_ds_size else max_ds_size
    with concurrent.futures.ThreadPoolExecutor(max_workers=max_threads) as executor:
        movies = [ex for n in range(max_ds_size) if (ex := executor.submit(extract_dataset, api_key, n).result()) is not None]
    movies = pd.DataFrame.from_records(movies)
    save_to_csv(df=movies, save_path=save_path, fname=fname)
    return movies
# AUTOGENERATED! DO NOT EDIT! File to edit: 00_extract.ipynb (unless otherwise specified). # Internal Cell # Cell