# -*- coding: utf-8 -*-
import click
import logging
import os
from pathlib import Path
#from dotenv import find_dotenv, load_dotenv
import requests
from tqdm import tqdm
import zipfile


URL = 'http://databank.worldbank.org/data/download/Edstats_csv.zip'
Geo_URL = 'https://raw.githubusercontent.com/AshKyd/geojson-regions/master/countries/110m/all.geojson'


def download_dataset(url, output_filepath):
    """Download the dataset."""
    logging.info("Start download, this can take a while...")
    response = requests.get(url, stream=True,
                            headers={"User-Agent": "Mozilla/5.0",
                                     "Accept-Encoding": None})
    content_length = int(response.headers.get('content-length'))
    with open(output_filepath, 'wb') as file:
        chunk_size = 1024 * 1024
        for data in tqdm(response.iter_content(chunk_size=chunk_size),
                         total=content_length // chunk_size,
                         desc='Download data',
                         unit='Mo'):
            file.write(data)

def unzip(filepath, purge=True):
    """Unzip the zip archive."""
    logging.info("Unzip the dataset. %s", filepath)
    output_path = '/'.join(filepath.split('/')[:-1])
    logging.info("output_path : %s", output_path)
    with zipfile.ZipFile(filepath, 'r') as zip_r:
        zip_r.extractall(output_path)
    csv_name = [x for x in os.listdir(output_path)]
    if purge:
        os.remove(filepath)
    logging.info("downloaded files : %s ", csv_name)

def init_data_dir(filepath):
    """Create empty dir if they not exists"""
    logging.info("Init data directories")
    root_dir = '/'.join(filepath.split('/')[:-1])
    data_dirs = ['external',
                 'interim',
                 'processed',
                 'raw']
    if not os.path.exists(root_dir):
        os.mkdir(root_dir)
        for dir in data_dirs:
            os.mkdir(os.path.join(root_dir, dir))
    else:
        print('already exists')

def download_assets(url, output_filepath):
    logging.info("Start download, this can take a while...")
    response = requests.get(url, stream=True,
                            headers={"User-Agent": "Mozilla/5.0",
                                     "Accept-Encoding": None})
    content_length = int(response.headers.get('content-length'))
    with open(output_filepath, 'wb') as file:
        chunk_size = 1024
        for data in tqdm(response.iter_content(chunk_size=chunk_size),
                         total=content_length // chunk_size,
                         desc='Download data',
                         unit='Ko'):
            file.write(data)

@click.command()
@click.argument('input_filepath', type=click.Path())
@click.argument('output_filepath', type=click.Path())
def main(input_filepath, output_filepath):
    """ Runs data processing scripts to turn raw data from (../raw) into
        cleaned data ready to be analyzed (saved in ../processed).
    """
    logger = logging.getLogger(__name__)
    logger.info('making data set.')
    if not os.path.exists(os.path.join(input_filepath, 'data.zip')):
        logger.info("Dataset not already downloaded.")
        init_data_dir(input_filepath)
        download_dataset(URL, os.path.join(input_filepath, 'data.zip'))
        external_dir = os.path.join(*input_filepath.split('/')[:-1], 'external')
        download_assets(Geo_URL, os.path.join(external_dir, 'world.json'))
        unzip(os.path.join(input_filepath, 'data.zip'))
    else:
        logger.info('Dataset is already in raw data dir')

if __name__ == '__main__':
    log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    logging.basicConfig(level=logging.INFO, format=log_fmt)

    # not used in this stub but often useful for finding various files
    project_dir = Path(__file__).resolve().parents[2]

    # find .env automagically by walking up directories until it's found, then
    # load up the .env entries as environment variables
    #load_dotenv(find_dotenv())

    main()

# -*- coding: utf-8 -*-
import click
import logging
import os
from pathlib import Path
#from dotenv import find_dotenv, load_dotenv
import requests
from tqdm import tqdm
import zipfile


URL = 'http://databank.worldbank.org/data/download/Edstats_csv.zip'
Geo_URL = 'https://raw.githubusercontent.com/AshKyd/geojson-regions/master/countries/110m/all.geojson'


def download_dataset(url, output_filepath):
    """Download the dataset."""
    logging.info("Start download, this can take a while...")
    response = requests.get(url, stream=True,
                            headers={"User-Agent": "Mozilla/5.0",
                                     "Accept-Encoding": None})
    content_length = int(response.headers.get('content-length'))
    with open(output_filepath, 'wb') as file:
        chunk_size = 1024 * 1024
        for data in tqdm(response.iter_content(chunk_size=chunk_size),
                         total=content_length // chunk_size,
                         desc='Download data',
                         unit='Mo'):
            file.write(data)

def unzip(filepath, purge=True):
    """Unzip the zip archive."""
    logging.info("Unzip the dataset. %s", filepath)
    output_path = '/'.join(filepath.split('/')[:-1])
    logging.info("output_path : %s", output_path)
    with zipfile.ZipFile(filepath, 'r') as zip_r:
        zip_r.extractall(output_path)
    csv_name = [x for x in os.listdir(output_path)]
    if purge:
        os.remove(filepath)
    logging.info("downloaded files : %s ", csv_name)

def init_data_dir(filepath):
    """Create empty dir if they not exists"""
    logging.info("Init data directories")
    root_dir = '/'.join(filepath.split('/')[:-1])
    data_dirs = ['external',
                 'interim',
                 'processed',
                 'raw']
    if not os.path.exists(root_dir):
        os.mkdir(root_dir)
        for dir in data_dirs:
            os.mkdir(os.path.join(root_dir, dir))
    else:
        print('already exists')

def download_assets(url, output_filepath):
    logging.info("Start download, this can take a while...")
    response = requests.get(url, stream=True,
                            headers={"User-Agent": "Mozilla/5.0",
                                     "Accept-Encoding": None})
    content_length = int(response.headers.get('content-length'))
    with open(output_filepath, 'wb') as file:
        chunk_size = 1024
        for data in tqdm(response.iter_content(chunk_size=chunk_size),
                         total=content_length // chunk_size,
                         desc='Download data',
                         unit='Ko'):
            file.write(data)

@click.command()
@click.argument('input_filepath', type=click.Path())
@click.argument('output_filepath', type=click.Path())
def main(input_filepath, output_filepath):
    """ Runs data processing scripts to turn raw data from (../raw) into
        cleaned data ready to be analyzed (saved in ../processed).
    """
    logger = logging.getLogger(__name__)
    logger.info('making data set.')
    if not os.path.exists(os.path.join(input_filepath, 'data.zip')):
        logger.info("Dataset not already downloaded.")
        init_data_dir(input_filepath)
        download_dataset(URL, os.path.join(input_filepath, 'data.zip'))
        external_dir = os.path.join(*input_filepath.split('/')[:-1], 'external')
        download_assets(Geo_URL, os.path.join(external_dir, 'world.json'))
        unzip(os.path.join(input_filepath, 'data.zip'))
    else:
        logger.info('Dataset is already in raw data dir')

if __name__ == '__main__':
    log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
    logging.basicConfig(level=logging.INFO, format=log_fmt)

    # not used in this stub but often useful for finding various files
    project_dir = Path(__file__).resolve().parents[2]

    # find .env automagically by walking up directories until it's found, then
    # load up the .env entries as environment variables
    #load_dotenv(find_dotenv())

    main()

# -*- coding: utf-8 -*- #from dotenv import find_dotenv, load_dotenv Download the dataset. Unzip the zip archive. Create empty dir if they not exists Runs data processing scripts to turn raw data from (../raw) into         cleaned data ready to be analyzed (saved in ../processed). # not used in this stub but often useful for finding various files # find .env automagically by walking up directories until it's found, then # load up the .env entries as environment variables #load_dotenv(find_dotenv())