import os
import gevent 
from pennsieve import Pennsieve
from os.path import isdir, isfile, join, splitext, getmtime, basename, normpath, exists, expanduser, split, dirname, getsize, abspath
import requests
import shutil 
from shutil import copy2
import pathlib
import dateutil.parser

userpath = expanduser("~")
configpath = join(userpath, '.pennsieve', 'config.ini')
metadatapath = join(userpath, 'SODA', 'SODA_metadata')
sodavalidatorpath = join(userpath, 'SODA', 'SODA_Validator_Dataset')



def bf_get_current_user_permission(bf, myds):

    """
    Function to get the permission of currently logged in user for a selected dataset

    Args:
        bf: logged Pennsieve acccount (dict)
        myds: selected Pennsieve dataset (dict)
    Output:
        permission of current user (string)
    """

    try:
        selected_dataset_id = myds.id
        return bf._api._get(f"/datasets/{str(selected_dataset_id)}/role")["role"]

    except Exception as e:
        raise e

def create_local_dataset(soda_json_obj):
    
    def get_relative_path(bfpath):
        current_folder_path = sodavalidatorpath
        for relative_path in bfpath:
            current_folder_path = join(current_folder_path, relative_path)
        return current_folder_path
    
    def recursive_folder_file_create(dataset_folder):
        if "folders" in dataset_folder:
            for folder in dataset_folder["folders"]:
                if "bfpath" in dataset_folder["folders"][folder]:
                    current_folder_path = get_relative_path(dataset_folder["folders"][folder]["bfpath"])
                    if not os.path.exists(current_folder_path):
                        pathlib.Path(current_folder_path).mkdir(parents = True, exist_ok = True)
                    recursive_folder_file_create(dataset_folder["folders"][folder])

        if "files" in dataset_folder:
            for file in dataset_folder["files"]:
                if "bfpath" in dataset_folder["files"][file]:
                    current_folder_path = get_relative_path(dataset_folder["files"][file]["bfpath"])
                    current_file_path = join(current_folder_path, file)
                    if "url" in dataset_folder["files"][file]:
                        url = dataset_folder["files"][file]["url"]
                        with open(current_file_path, 'wb') as new_file:
                            if url != "":
                                gevent.sleep(0)
                                req = requests.get(url)
                                url_content = req.content
                                new_file.write(url_content)
                            else:
                                new_file.write("Mock file generated by SODA".encode())
                        temp_time = dateutil.parser.parse(dataset_folder["files"][file]["timestamp"])
                        temp_time = temp_time.timestamp()
                        os.utime(current_file_path, (temp_time, temp_time))
                    else:
                        raise Exception
                    
    def metadata_file_create(metadata_files):
        for file in metadata_files:
            current_folder_path = get_relative_path([])
            current_file_path = join(current_folder_path, file)
            if "url" in metadata_files[file]:
                url = metadata_files[file]["url"]
                with open(current_file_path, 'wb') as new_file:
                    if url != "":
                        gevent.sleep(0)
                        req = requests.get(url)
                        url_content = req.content
                        new_file.write(url_content)
                    else:
                        new_file.write("Mock file generated by SODA".encode())
                temp_time = dateutil.parser.parse(metadata_files["files"][file]["timestamp"])
                temp_time = temp_time.timestamp()
                os.utime(current_file_path, (temp_time, temp_time))
            else:
                raise Exception
    
    dir_path = pathlib.Path(sodavalidatorpath)
    try:
        shutil.rmtree(dir_path)
    except OSError as e:
        # no folder present
        print("Error: %s : %s" % (dir_path, e.strerror))
    
    if not os.path.exists(sodavalidatorpath):
        pathlib.Path(sodavalidatorpath).mkdir(parents = True, exist_ok = True)
    
    if "dataset-structure" in soda_json_obj:
        try:
            recursive_folder_file_create(soda_json_obj["dataset-structure"])
        except e:
            raise e
            
    if "metadata-files" in soda_json_obj:
        try:
            metadata_file_create(soda_json_obj["metadata-files"])
        except e:
            raise e
    else:
        raise Exception("Empty dataset")



def ps_retrieve_dataset(soda_json_structure):
    """
    Function for importing Pennsieve data files info into the "dataset-structure" key of the soda json structure,
    including metadata from any existing manifest files in the high-level folders
    (name, id, timestamp, description, additional metadata)
    Args:
        soda_json_structure: soda structure with bf account and dataset info available
    Output:
        same soda structure with Pennsieve data file info included under the "dataset-structure" key
    """

    double_extensions = ['.ome.tiff','.ome.tif','.ome.tf2,','.ome.tf8','.ome.btf','.ome.xml','.brukertiff.gz','.mefd.gz','.moberg.gz','.nii.gz','.mgh.gz','.tar.gz','.bcl.gz']
    
    download_extensions = [".xlsx", ".csv", ".xlsm", ".xlsb", ".xltx", ".xltm", ".xls", ".xlt", ".xls", ".xml", ".xlam", ".xla", ".xlw", ".xlr", ".json", ".txt"]
    #f = open("dataset_contents.soda", "a")

    def verify_file_name(file_name, extension):
        if extension == "":
            return (file_name, extension)

        double_ext = any(file_name.find(ext) != -1 for ext in double_extensions)
        extension_from_name = ""

        extension_from_name = (
            os.path.splitext(os.path.splitext(file_name)[0])[1]
            + os.path.splitext(file_name)[1]
            if double_ext
            else os.path.splitext(file_name)[1]
        )

        if extension_from_name == f'.{extension}':
            return (file_name, extension_from_name)
        else:
            return f'{file_name}.{extension}', f'.{extension}'

    # Add a new key containing the path to all the files and folders on the
    # local data structure..
    def recursive_item_path_create(folder, path):
        if "files" in folder.keys():
            for item in list(folder["files"]):
                if "bfpath" not in folder["files"][item]:
                    folder["files"][item]['bfpath'] = path[:]

        if "folders" in folder.keys():
            for item in list(folder["folders"]):
                if "bfpath" not in folder["folders"][item]:
                    folder["folders"][item]['bfpath'] = path[:]
                    folder["folders"][item]['bfpath'].append(item)
                recursive_item_path_create(folder["folders"][item], folder["folders"][item]['bfpath'][:])
        return

    level = 0

    def recursive_dataset_import(my_item, metadata_files, dataset_folder, my_level):
        level = 0
        col_count = 0
        file_count = 0

        for item in my_item:
            if "folders" not in dataset_folder:
                dataset_folder["folders"] = {}
            if "files" not in dataset_folder:
                dataset_folder["files"] = {}

            if item.type == "Collection":
                col_count += 1
                folder_name = item.name

                if col_count == 1:
                    level = my_level + 1
                dataset_folder["folders"][folder_name] = {
                    "type": "bf", "action": ["existing"], "path": item.id}
                sub_folder = dataset_folder["folders"][folder_name]

                if "folders" not in sub_folder:
                    sub_folder["folders"] = {}
                if "files" not in sub_folder:
                    sub_folder["files"] = {}

                recursive_dataset_import(item,metadata_files, sub_folder, level)
            else:
                package_id = item.id
                gevent.sleep(0)
                package_details = bf._api._get(f'/packages/{str(package_id)}')

                if ("extension" not in package_details):
                    (file_name, ext) = verify_file_name(package_details["content"]["name"], "")
                else:
                    (file_name, ext) = verify_file_name(package_details["content"]["name"], package_details["extension"])

                if my_level == 0:
                    if ext in download_extensions:
                        gevent.sleep(0)
                        file_details = bf._api._get(f'/packages/{str(package_id)}/view')
                        print(file_details)
                        file_id = file_details[0]["content"]["id"]
                        gevent.sleep(0)
                        file_url = bf._api._get(f'/packages/{str(package_id)}/files/{str(file_id)}')
                        timestamp = (package_details["content"]["updatedAt"])
                        metadata_files[file_name] = {
                            "type": "bf", 
                            "action": ["existing"], 
                            "path": item.id, 
                            "timestamp": timestamp, 
                            "extension": ext, 
                            "url": file_url["url"],  
                            "size": file_details[0]["content"]["size"]
                        }
                    else:
                        timestamp = (package_details["content"]["updatedAt"])
                        metadata_files[file_name] = {
                            "type": "bf", "action": ["existing"], "path": item.id, "timestamp": timestamp, "extension": ext, "url": "", "size": 1}
                else:
                    file_count += 1
                    if ext in download_extensions:
                        gevent.sleep(0)
                        file_details = bf._api._get(f'/packages/{str(package_id)}/view')
                        file_id = file_details[0]["content"]["id"]
                        gevent.sleep(0)
                        file_url = bf._api._get(f'/packages/{str(package_id)}/files/{str(file_id)}')
                        timestamp = (package_details["content"]["updatedAt"])
                        dataset_folder["files"][file_name] = {
                            "type": "bf","action": ["existing"], 
                            "path": item.id, 
                            "timestamp": timestamp, 
                            "extension": ext, 
                            "url": file_url["url"], 
                            "size": file_details[0]["content"]["size"]
                        }
                    else:
                        timestamp = (package_details["content"]["updatedAt"])
                        dataset_folder["files"][file_name] = {
                            "type": "bf","action": ["existing"], "path": item.id, "timestamp": timestamp, "extension": ext, "url": "", "size": 1}

    error = []

    # check that the Pennsieve account is valid
    try:
        bf_account_name = soda_json_structure["bf-account-selected"]["account-name"]
    except Exception as e:
        raise e

    try:
        bf = Pennsieve(bf_account_name)
    except Exception as e:
        error.append('Error: Please select a valid Pennsieve account')
        raise Exception(error)

    # check that the Pennsieve dataset is valid
    try:
        bf_dataset_name = soda_json_structure["bf-dataset-selected"]["dataset-name"]
    except Exception as e:
        raise e
    try:
        myds = bf.get_dataset(bf_dataset_name)
    except Exception as e:
        error.append('Error: Please select a valid Pennsieve dataset')
        raise Exception(error)

    # check that the user has permission to edit this dataset
    try:
        role = bf_get_current_user_permission(bf, myds)
        if role not in ['owner', 'manager']:
            curatestatus = 'Done'
            error.append("Error: You don't have permissions for uploading to this Pennsieve dataset")
            raise Exception(error)
    except Exception as e:
        raise e

    try:
        # import files and folders in the soda json structure
        soda_json_structure["dataset-structure"] = {}
        soda_json_structure["metadata-files"] = {}
        dataset_folder = soda_json_structure["dataset-structure"]
        metadata_files = soda_json_structure["metadata-files"]
        recursive_dataset_import(myds, metadata_files, dataset_folder, level)

        #remove metadata files keys if empty
        metadata_files = soda_json_structure["metadata-files"]
        if not metadata_files:
            del soda_json_structure['metadata-files']

        dataset_folder = soda_json_structure["dataset-structure"]

        recursive_item_path_create(soda_json_structure["dataset-structure"], [])
        success_message = "Data files under a valid high-level SPARC folders have been imported"

        create_local_dataset(soda_json_structure)
        return "created"

    except Exception as e:
        raise e
import os
import gevent 
from pennsieve import Pennsieve
from os.path import isdir, isfile, join, splitext, getmtime, basename, normpath, exists, expanduser, split, dirname, getsize, abspath
import requests
import shutil 
from shutil import copy2
import pathlib
import dateutil.parser

userpath = expanduser("~")
configpath = join(userpath, '.pennsieve', 'config.ini')
metadatapath = join(userpath, 'SODA', 'SODA_metadata')
sodavalidatorpath = join(userpath, 'SODA', 'SODA_Validator_Dataset')



def bf_get_current_user_permission(bf, myds):

    """
    Function to get the permission of currently logged in user for a selected dataset

    Args:
        bf: logged Pennsieve acccount (dict)
        myds: selected Pennsieve dataset (dict)
    Output:
        permission of current user (string)
    """

    try:
        selected_dataset_id = myds.id
        return bf._api._get(f"/datasets/{str(selected_dataset_id)}/role")["role"]

    except Exception as e:
        raise e

def create_local_dataset(soda_json_obj):
    
    def get_relative_path(bfpath):
        current_folder_path = sodavalidatorpath
        for relative_path in bfpath:
            current_folder_path = join(current_folder_path, relative_path)
        return current_folder_path
    
    def recursive_folder_file_create(dataset_folder):
        if "folders" in dataset_folder:
            for folder in dataset_folder["folders"]:
                if "bfpath" in dataset_folder["folders"][folder]:
                    current_folder_path = get_relative_path(dataset_folder["folders"][folder]["bfpath"])
                    if not os.path.exists(current_folder_path):
                        pathlib.Path(current_folder_path).mkdir(parents = True, exist_ok = True)
                    recursive_folder_file_create(dataset_folder["folders"][folder])

        if "files" in dataset_folder:
            for file in dataset_folder["files"]:
                if "bfpath" in dataset_folder["files"][file]:
                    current_folder_path = get_relative_path(dataset_folder["files"][file]["bfpath"])
                    current_file_path = join(current_folder_path, file)
                    if "url" in dataset_folder["files"][file]:
                        url = dataset_folder["files"][file]["url"]
                        with open(current_file_path, 'wb') as new_file:
                            if url != "":
                                gevent.sleep(0)
                                req = requests.get(url)
                                url_content = req.content
                                new_file.write(url_content)
                            else:
                                new_file.write("Mock file generated by SODA".encode())
                        temp_time = dateutil.parser.parse(dataset_folder["files"][file]["timestamp"])
                        temp_time = temp_time.timestamp()
                        os.utime(current_file_path, (temp_time, temp_time))
                    else:
                        raise Exception
                    
    def metadata_file_create(metadata_files):
        for file in metadata_files:
            current_folder_path = get_relative_path([])
            current_file_path = join(current_folder_path, file)
            if "url" in metadata_files[file]:
                url = metadata_files[file]["url"]
                with open(current_file_path, 'wb') as new_file:
                    if url != "":
                        gevent.sleep(0)
                        req = requests.get(url)
                        url_content = req.content
                        new_file.write(url_content)
                    else:
                        new_file.write("Mock file generated by SODA".encode())
                temp_time = dateutil.parser.parse(metadata_files["files"][file]["timestamp"])
                temp_time = temp_time.timestamp()
                os.utime(current_file_path, (temp_time, temp_time))
            else:
                raise Exception
    
    dir_path = pathlib.Path(sodavalidatorpath)
    try:
        shutil.rmtree(dir_path)
    except OSError as e:
        # no folder present
        print("Error: %s : %s" % (dir_path, e.strerror))
    
    if not os.path.exists(sodavalidatorpath):
        pathlib.Path(sodavalidatorpath).mkdir(parents = True, exist_ok = True)
    
    if "dataset-structure" in soda_json_obj:
        try:
            recursive_folder_file_create(soda_json_obj["dataset-structure"])
        except e:
            raise e
            
    if "metadata-files" in soda_json_obj:
        try:
            metadata_file_create(soda_json_obj["metadata-files"])
        except e:
            raise e
    else:
        raise Exception("Empty dataset")



def ps_retrieve_dataset(soda_json_structure):
    """
    Function for importing Pennsieve data files info into the "dataset-structure" key of the soda json structure,
    including metadata from any existing manifest files in the high-level folders
    (name, id, timestamp, description, additional metadata)
    Args:
        soda_json_structure: soda structure with bf account and dataset info available
    Output:
        same soda structure with Pennsieve data file info included under the "dataset-structure" key
    """

    double_extensions = ['.ome.tiff','.ome.tif','.ome.tf2,','.ome.tf8','.ome.btf','.ome.xml','.brukertiff.gz','.mefd.gz','.moberg.gz','.nii.gz','.mgh.gz','.tar.gz','.bcl.gz']
    
    download_extensions = [".xlsx", ".csv", ".xlsm", ".xlsb", ".xltx", ".xltm", ".xls", ".xlt", ".xls", ".xml", ".xlam", ".xla", ".xlw", ".xlr", ".json", ".txt"]
    #f = open("dataset_contents.soda", "a")

    def verify_file_name(file_name, extension):
        if extension == "":
            return (file_name, extension)

        double_ext = any(file_name.find(ext) != -1 for ext in double_extensions)
        extension_from_name = ""

        extension_from_name = (
            os.path.splitext(os.path.splitext(file_name)[0])[1]
            + os.path.splitext(file_name)[1]
            if double_ext
            else os.path.splitext(file_name)[1]
        )

        if extension_from_name == f'.{extension}':
            return (file_name, extension_from_name)
        else:
            return f'{file_name}.{extension}', f'.{extension}'

    # Add a new key containing the path to all the files and folders on the
    # local data structure..
    def recursive_item_path_create(folder, path):
        if "files" in folder.keys():
            for item in list(folder["files"]):
                if "bfpath" not in folder["files"][item]:
                    folder["files"][item]['bfpath'] = path[:]

        if "folders" in folder.keys():
            for item in list(folder["folders"]):
                if "bfpath" not in folder["folders"][item]:
                    folder["folders"][item]['bfpath'] = path[:]
                    folder["folders"][item]['bfpath'].append(item)
                recursive_item_path_create(folder["folders"][item], folder["folders"][item]['bfpath'][:])
        return

    level = 0

    def recursive_dataset_import(my_item, metadata_files, dataset_folder, my_level):
        level = 0
        col_count = 0
        file_count = 0

        for item in my_item:
            if "folders" not in dataset_folder:
                dataset_folder["folders"] = {}
            if "files" not in dataset_folder:
                dataset_folder["files"] = {}

            if item.type == "Collection":
                col_count += 1
                folder_name = item.name

                if col_count == 1:
                    level = my_level + 1
                dataset_folder["folders"][folder_name] = {
                    "type": "bf", "action": ["existing"], "path": item.id}
                sub_folder = dataset_folder["folders"][folder_name]

                if "folders" not in sub_folder:
                    sub_folder["folders"] = {}
                if "files" not in sub_folder:
                    sub_folder["files"] = {}

                recursive_dataset_import(item,metadata_files, sub_folder, level)
            else:
                package_id = item.id
                gevent.sleep(0)
                package_details = bf._api._get(f'/packages/{str(package_id)}')

                if ("extension" not in package_details):
                    (file_name, ext) = verify_file_name(package_details["content"]["name"], "")
                else:
                    (file_name, ext) = verify_file_name(package_details["content"]["name"], package_details["extension"])

                if my_level == 0:
                    if ext in download_extensions:
                        gevent.sleep(0)
                        file_details = bf._api._get(f'/packages/{str(package_id)}/view')
                        print(file_details)
                        file_id = file_details[0]["content"]["id"]
                        gevent.sleep(0)
                        file_url = bf._api._get(f'/packages/{str(package_id)}/files/{str(file_id)}')
                        timestamp = (package_details["content"]["updatedAt"])
                        metadata_files[file_name] = {
                            "type": "bf", 
                            "action": ["existing"], 
                            "path": item.id, 
                            "timestamp": timestamp, 
                            "extension": ext, 
                            "url": file_url["url"],  
                            "size": file_details[0]["content"]["size"]
                        }
                    else:
                        timestamp = (package_details["content"]["updatedAt"])
                        metadata_files[file_name] = {
                            "type": "bf", "action": ["existing"], "path": item.id, "timestamp": timestamp, "extension": ext, "url": "", "size": 1}
                else:
                    file_count += 1
                    if ext in download_extensions:
                        gevent.sleep(0)
                        file_details = bf._api._get(f'/packages/{str(package_id)}/view')
                        file_id = file_details[0]["content"]["id"]
                        gevent.sleep(0)
                        file_url = bf._api._get(f'/packages/{str(package_id)}/files/{str(file_id)}')
                        timestamp = (package_details["content"]["updatedAt"])
                        dataset_folder["files"][file_name] = {
                            "type": "bf","action": ["existing"], 
                            "path": item.id, 
                            "timestamp": timestamp, 
                            "extension": ext, 
                            "url": file_url["url"], 
                            "size": file_details[0]["content"]["size"]
                        }
                    else:
                        timestamp = (package_details["content"]["updatedAt"])
                        dataset_folder["files"][file_name] = {
                            "type": "bf","action": ["existing"], "path": item.id, "timestamp": timestamp, "extension": ext, "url": "", "size": 1}

    error = []

    # check that the Pennsieve account is valid
    try:
        bf_account_name = soda_json_structure["bf-account-selected"]["account-name"]
    except Exception as e:
        raise e

    try:
        bf = Pennsieve(bf_account_name)
    except Exception as e:
        error.append('Error: Please select a valid Pennsieve account')
        raise Exception(error)

    # check that the Pennsieve dataset is valid
    try:
        bf_dataset_name = soda_json_structure["bf-dataset-selected"]["dataset-name"]
    except Exception as e:
        raise e
    try:
        myds = bf.get_dataset(bf_dataset_name)
    except Exception as e:
        error.append('Error: Please select a valid Pennsieve dataset')
        raise Exception(error)

    # check that the user has permission to edit this dataset
    try:
        role = bf_get_current_user_permission(bf, myds)
        if role not in ['owner', 'manager']:
            curatestatus = 'Done'
            error.append("Error: You don't have permissions for uploading to this Pennsieve dataset")
            raise Exception(error)
    except Exception as e:
        raise e

    try:
        # import files and folders in the soda json structure
        soda_json_structure["dataset-structure"] = {}
        soda_json_structure["metadata-files"] = {}
        dataset_folder = soda_json_structure["dataset-structure"]
        metadata_files = soda_json_structure["metadata-files"]
        recursive_dataset_import(myds, metadata_files, dataset_folder, level)

        #remove metadata files keys if empty
        metadata_files = soda_json_structure["metadata-files"]
        if not metadata_files:
            del soda_json_structure['metadata-files']

        dataset_folder = soda_json_structure["dataset-structure"]

        recursive_item_path_create(soda_json_structure["dataset-structure"], [])
        success_message = "Data files under a valid high-level SPARC folders have been imported"

        create_local_dataset(soda_json_structure)
        return "created"

    except Exception as e:
        raise e
Function to get the permission of currently logged in user for a selected dataset      Args:         bf: logged Pennsieve acccount (dict)         myds: selected Pennsieve dataset (dict)     Output:         permission of current user (string) # no folder present Function for importing Pennsieve data files info into the "dataset-structure" key of the soda json structure,     including metadata from any existing manifest files in the high-level folders     (name, id, timestamp, description, additional metadata)     Args:         soda_json_structure: soda structure with bf account and dataset info available     Output:         same soda structure with Pennsieve data file info included under the "dataset-structure" key #f = open("dataset_contents.soda", "a") # Add a new key containing the path to all the files and folders on the # local data structure.. # check that the Pennsieve account is valid # check that the Pennsieve dataset is valid # check that the user has permission to edit this dataset # import files and folders in the soda json structure #remove metadata files keys if empty