import os
import requests
from lxml import html
import pandas as pd


class DatasetDownloader:
    """
    Class containing methods and attributes for pulling and preprocessing data.
    """

    VALID_NAMES = ["annotation", "cell", "event", "location", "mac", "marker", "sensor"]
    URL = None
    USERNAME = None
    PASSWORD = None

    @staticmethod
    def setup_directory( dir_name):
        """Setup directory in case it does not exist
        """
        if not os.path.exists(dir_name):
            try:
                os.makedirs(dir_name)
                print("Created Directory: {}".format(dir_name) )
            except:
                print("Could not create directory: {}".format(dir_name))

    @staticmethod
    def get_data_dir():
        """ Returns the data dir relative from this file
        """
        project_dir = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
        data_dir = os.path.join(project_dir, "data")

        return data_dir

    @staticmethod
    def get_parsed_page(token):
        """
        Return the content of the website on the given url in
        a parsed lxml format that is easy to query.

        Parameters
        ----------
        token : token to access the data
        """
        url_with_token = DatasetDownloader.URL + str(token)
        response = requests.get(url_with_token, auth=(DatasetDownloader.USERNAME, DatasetDownloader.PASSWORD))
        parsed_page = None
        if response.status_code == 200:
            parsed_page = html.fromstring(response.content)
        return parsed_page

    @staticmethod
    def download_file(url, token):
        """
        This method downloads the tar.gz files and extracts them into
        pandas DataFrames.

        """

        response = requests.get(url, auth=(DatasetDownloader.USERNAME, DatasetDownloader.PASSWORD))
        if response.status_code == 200:
            extracted_file_name = url.replace(DatasetDownloader.URL,"").partition("/")[2]
            file_path = os.path.join(DatasetDownloader.get_data_dir(),"raw",str(token))
            DatasetDownloader.setup_directory(file_path)
            print("Downloaded ",extracted_file_name)
            file_path = os.path.join(file_path, extracted_file_name)
            with open(file_path, 'wb') as fd:
                for chunk in response.iter_content():
                    fd.write(chunk)

    @staticmethod
    def download_data_sets(token, file_ending=".gz", file_names=None):
        """
        This method downloads all recorded trips for one user in one token directory
        and saves it into the data/raw/token directory

        Parameters
        ----------
        token : token to access the data
        file_ending: string, optional, default=.gz
                    file ending of the downloaded file
        file_names: list of file names, optional, default=None
                    In order to reduce the requests on the homepage, the file_names
                    can also be passed if they are already known

        """
        if file_names == None:
            recorded_data = DatasetDownloader.list_recorded_data(
                token,
                file_ending=file_ending,
                remove_file_ending=False
            )
            file_names = recorded_data["full_name"]

        url_with_token = DatasetDownloader.URL + str(token)
        for csv_file_name in file_names:
            full_url = url_with_token + "/" + str(csv_file_name)
            DatasetDownloader.download_file(full_url, token)

    @staticmethod
    def get_file_names_for(dir_name, token, file_ending=".gz"):
        """ Returns the file names for one specific token directory
        """
        file_names = list()
        dir_name = os.path.join(dir_name,token)
        dir_content = os.listdir(dir_name)
        for file in dir_content:
            if file.endswith(file_ending):
                file_names.append(os.path.join(str(token),file))
        return file_names

    @staticmethod
    def list_recorded_data(token, file_ending=".gz", remove_file_ending=True):
        """
        This method lists all recorded trips in one token directory.

        Parameters
        ----------
        token : token to access the data
        file_ending: string, optional, default=".gz"
                    file ending of the downloaded file
        remove_file_ending: bool, optional, default=True
                            Indicate wether the file endings should be removed
                            or not.

        Returns
        -------
        recorded_data : pandas DataFrame
            the recorded data with information on last modified and size
        """
        parsed_page = DatasetDownloader.get_parsed_page(token)
        table_content = parsed_page.xpath('//table//text()')
        track_records = []
        last_modified = []
        size = []
        for i, name in enumerate(table_content):
            if name.endswith(file_ending):
                if remove_file_ending:
                    name_without_file_ending = name.partition(".")[0]
                    track_records.append(name_without_file_ending)
                else:
                    track_records.append(name)
                last_modified.append(table_content[i+1])
                size.append(table_content[i+2])

        recorded_data = pd.DataFrame({"full_name":track_records,
                                      "last_modified":last_modified,
                                      "size":size})
        return recorded_data

    @staticmethod
    def get_file_names(dir_name, token=None, file_ending=".gz"):
        """ Returns all tar file names from all token folder if token is
            not specified
        """
        file_names = list()
        if os.path.exists(dir_name):
            if token != None:
                file_names = DatasetDownloader.get_file_names_for(dir_name, token)
            else:
                for root, dirs, files in os.walk(dir_name, topdown=False):
                    for directory_i in dirs:
                        directory_i_absolute_path = os.path.join(root, directory_i)
                        dir_content = os.listdir(directory_i_absolute_path)
                        for file in dir_content:
                            if file.endswith(file_ending):
                                file_names.append(os.path.join(directory_i, file))

        return file_names

    @staticmethod
    def download_all():
        """This method downloads all recorded trips for our team and saves
            it into the data/raw directory
        """
        tokens = [os.environ.get("KEY_RAPHAEL"), os.environ.get("KEY_MORITZ"),
                  os.environ.get("KEY_LUKAS")]
        for token in tokens:
            DatasetDownloader.download_data_sets(token)

import os
import requests
from lxml import html
import pandas as pd


class DatasetDownloader:
    """
    Class containing methods and attributes for pulling and preprocessing data.
    """

    VALID_NAMES = ["annotation", "cell", "event", "location", "mac", "marker", "sensor"]
    URL = None
    USERNAME = None
    PASSWORD = None

    @staticmethod
    def setup_directory( dir_name):
        """Setup directory in case it does not exist
        """
        if not os.path.exists(dir_name):
            try:
                os.makedirs(dir_name)
                print("Created Directory: {}".format(dir_name) )
            except:
                print("Could not create directory: {}".format(dir_name))

    @staticmethod
    def get_data_dir():
        """ Returns the data dir relative from this file
        """
        project_dir = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
        data_dir = os.path.join(project_dir, "data")

        return data_dir

    @staticmethod
    def get_parsed_page(token):
        """
        Return the content of the website on the given url in
        a parsed lxml format that is easy to query.

        Parameters
        ----------
        token : token to access the data
        """
        url_with_token = DatasetDownloader.URL + str(token)
        response = requests.get(url_with_token, auth=(DatasetDownloader.USERNAME, DatasetDownloader.PASSWORD))
        parsed_page = None
        if response.status_code == 200:
            parsed_page = html.fromstring(response.content)
        return parsed_page

    @staticmethod
    def download_file(url, token):
        """
        This method downloads the tar.gz files and extracts them into
        pandas DataFrames.

        """

        response = requests.get(url, auth=(DatasetDownloader.USERNAME, DatasetDownloader.PASSWORD))
        if response.status_code == 200:
            extracted_file_name = url.replace(DatasetDownloader.URL,"").partition("/")[2]
            file_path = os.path.join(DatasetDownloader.get_data_dir(),"raw",str(token))
            DatasetDownloader.setup_directory(file_path)
            print("Downloaded ",extracted_file_name)
            file_path = os.path.join(file_path, extracted_file_name)
            with open(file_path, 'wb') as fd:
                for chunk in response.iter_content():
                    fd.write(chunk)

    @staticmethod
    def download_data_sets(token, file_ending=".gz", file_names=None):
        """
        This method downloads all recorded trips for one user in one token directory
        and saves it into the data/raw/token directory

        Parameters
        ----------
        token : token to access the data
        file_ending: string, optional, default=.gz
                    file ending of the downloaded file
        file_names: list of file names, optional, default=None
                    In order to reduce the requests on the homepage, the file_names
                    can also be passed if they are already known

        """
        if file_names == None:
            recorded_data = DatasetDownloader.list_recorded_data(
                token,
                file_ending=file_ending,
                remove_file_ending=False
            )
            file_names = recorded_data["full_name"]

        url_with_token = DatasetDownloader.URL + str(token)
        for csv_file_name in file_names:
            full_url = url_with_token + "/" + str(csv_file_name)
            DatasetDownloader.download_file(full_url, token)

    @staticmethod
    def get_file_names_for(dir_name, token, file_ending=".gz"):
        """ Returns the file names for one specific token directory
        """
        file_names = list()
        dir_name = os.path.join(dir_name,token)
        dir_content = os.listdir(dir_name)
        for file in dir_content:
            if file.endswith(file_ending):
                file_names.append(os.path.join(str(token),file))
        return file_names

    @staticmethod
    def list_recorded_data(token, file_ending=".gz", remove_file_ending=True):
        """
        This method lists all recorded trips in one token directory.

        Parameters
        ----------
        token : token to access the data
        file_ending: string, optional, default=".gz"
                    file ending of the downloaded file
        remove_file_ending: bool, optional, default=True
                            Indicate wether the file endings should be removed
                            or not.

        Returns
        -------
        recorded_data : pandas DataFrame
            the recorded data with information on last modified and size
        """
        parsed_page = DatasetDownloader.get_parsed_page(token)
        table_content = parsed_page.xpath('//table//text()')
        track_records = []
        last_modified = []
        size = []
        for i, name in enumerate(table_content):
            if name.endswith(file_ending):
                if remove_file_ending:
                    name_without_file_ending = name.partition(".")[0]
                    track_records.append(name_without_file_ending)
                else:
                    track_records.append(name)
                last_modified.append(table_content[i+1])
                size.append(table_content[i+2])

        recorded_data = pd.DataFrame({"full_name":track_records,
                                      "last_modified":last_modified,
                                      "size":size})
        return recorded_data

    @staticmethod
    def get_file_names(dir_name, token=None, file_ending=".gz"):
        """ Returns all tar file names from all token folder if token is
            not specified
        """
        file_names = list()
        if os.path.exists(dir_name):
            if token != None:
                file_names = DatasetDownloader.get_file_names_for(dir_name, token)
            else:
                for root, dirs, files in os.walk(dir_name, topdown=False):
                    for directory_i in dirs:
                        directory_i_absolute_path = os.path.join(root, directory_i)
                        dir_content = os.listdir(directory_i_absolute_path)
                        for file in dir_content:
                            if file.endswith(file_ending):
                                file_names.append(os.path.join(directory_i, file))

        return file_names

    @staticmethod
    def download_all():
        """This method downloads all recorded trips for our team and saves
            it into the data/raw directory
        """
        tokens = [os.environ.get("KEY_RAPHAEL"), os.environ.get("KEY_MORITZ"),
                  os.environ.get("KEY_LUKAS")]
        for token in tokens:
            DatasetDownloader.download_data_sets(token)

Class containing methods and attributes for pulling and preprocessing data. Setup directory in case it does not exist Returns the data dir relative from this file Return the content of the website on the given url in         a parsed lxml format that is easy to query.          Parameters         ----------         token : token to access the data This method downloads the tar.gz files and extracts them into         pandas DataFrames. This method downloads all recorded trips for one user in one token directory         and saves it into the data/raw/token directory          Parameters         ----------         token : token to access the data         file_ending: string, optional, default=.gz                     file ending of the downloaded file         file_names: list of file names, optional, default=None                     In order to reduce the requests on the homepage, the file_names                     can also be passed if they are already known Returns the file names for one specific token directory This method lists all recorded trips in one token directory.          Parameters         ----------         token : token to access the data         file_ending: string, optional, default=".gz"                     file ending of the downloaded file         remove_file_ending: bool, optional, default=True                             Indicate wether the file endings should be removed                             or not.          Returns         -------         recorded_data : pandas DataFrame             the recorded data with information on last modified and size Returns all tar file names from all token folder if token is             not specified This method downloads all recorded trips for our team and saves             it into the data/raw directory