import csv
import os
import sys
import warnings

from openpyxl import load_workbook

from baroque import defaults


class BaroqueProject(object):
    """
    Stores details about the current project.

    It takes a path to a source directory and a path to a destination directory:
    - the source directory can be either a shipment-level, collection-level, or item-level directory.
    - the destination directory stores reports generated by baroque.

    It characterizes the source directory level by analyzing what is inside the source directory.
    Then, it parses the source directory recursively down to the item-level.

    An initial instantiation of this object might look like:
    {
        "source": source_directory,
        "destination": destination_directory,
        "errors": {},
        "shipment" : [
            {
            "id": "",
            "path": "",
            }
        ],
        "collections" : [
            {
            "id": "",
            "path": "",
            }
        ],
        "items": [
            {
            "id": "",
            "path": "",
            "files": {
                "wav": [],
                "mp3": [],
                "jpg": [],
                "xml": [],
                "md5": [],
                "txt": [],
                "other": []
                },
            }
        ],
    }
    """

    def __init__(self, source_directory, destination_directory, metadata_export):
        if not os.path.exists(source_directory):
            print("SYSTEM ERROR: source_directory does not exist: {}".format(source_directory))
            sys.exit()
        if not os.path.exists(destination_directory):
            print("SYSTEM WARNING: destination_directory {} does not exist. Using default directory {}".format(destination_directory, defaults.REPORTS_DIR))
            if not os.path.exists(defaults.REPORTS_DIR):
                os.mkdir(defaults.REPORTS_DIR)
            destination_directory = defaults.REPORTS_DIR
        if not os.path.exists(metadata_export):
            print("SYSTEM ERROR: metadata export does not exist: {}".format(metadata_export))
            sys.exit()

        self.source_directory = source_directory
        self.destination_directory = destination_directory
        self.metadata_export = metadata_export

        self.shipment = []
        self.collections = []
        self.items = []
        self.errors = {}

        self.source_type = self.characterize_source_directory()
        print("SYSTEM REPORT: source_directory is {}".format(self.source_type))
        if self.source_type == "shipment":
            self.parse_shipment(source_directory)
        elif self.source_type == "collection":
            self.parse_collection(source_directory)
        elif self.source_type == "item":
            self.parse_item(source_directory)

        self.metadata = self.parse_metadata_export()

    def characterize_source_directory(self):
        """
        Characterize the source directory level by analyzing what is inside the source directory.
        """
        character_directory_name = os.path.basename(self.source_directory)
        character_directory_dirs = []
        character_directory_files = []

        # Loop the source directory to get files and sub-directories and append their name into respective lists
        for dir_entry in os.scandir(self.source_directory):
            if dir_entry.is_file():
                character_directory_files.append(str(dir_entry.name))
            elif dir_entry.is_dir():
                character_directory_dirs.append(str(dir_entry.name))

        # Return the source directory level as "item", if the lists show the source directory
        # has files and does not have sub-directories
        if len(character_directory_files) > 0 and len(character_directory_dirs) == 0:
            return "item"

        elif len(character_directory_dirs) > 0:
            # Return the source directory level as "collection", if the lists show the source directory
            # has sub-directories and sub-directory names start with the source directory name
            if any([directory.startswith(character_directory_name) for directory in character_directory_dirs]):
                return "collection"

            # Return the source directory level as "shipment", if the lists show the source directory
            # has sub-directories and sub-directory names do not start with the source directory name
            else:
                return "shipment"

        # Exit when the source directory does not have files nor sub-directories
        else:
            print("SYSTEM ERROR: source_directory is empty")
            sys.exit()

    def parse_shipment(self, shipment_directory):
        """
        Take a shipment-level directory path, parse its id and directory path,
        and add a dictionary to the "shipment" attribute.
        Then, loop the shipment-level directory for collection-level directories,
        and run "parse_collection" method on each collection-level directory.
        """
        self.shipment.append({
            "id": os.path.basename(shipment_directory),
            "path": shipment_directory
        })

        for dir_entry in os.scandir(shipment_directory):
            if dir_entry.is_dir():
                self.parse_collection(dir_entry.path)

    def parse_collection(self, collection_directory):
        """
        Take a collection-level directory path, parse its id and directory path,
        and add a dictionary to the "collections" attribute.
        Then, loop the directory for collection-level directories,
        and run "parse_item" method on each item-level directory.
        """
        self.collections.append({
            "id": os.path.basename(collection_directory),
            "path": collection_directory
        })

        for dir_entry in os.scandir(collection_directory):
            if dir_entry.is_dir():
                self.parse_item(dir_entry.path)

    def parse_item(self, item_directory):
        """
        Take a item-level directory path, parse id, directory path, files by their file formats,
        and add a dictionary to the "items" attribute.
        """
        files = {"wav": [], "mp3": [], "jpg": [], "xml": [], "md5": [], "txt": [], "other": []}
        file_formats = {
            "wav": ["wav", "wave"],
            "mp3": ["mp3"],
            "jpg": ["jpg", "jpeg", "jpe", "jif", "jfif", "jfi"],
            "xml": ["xml"],
            "md5": ["md5"],
            "txt": ["txt"]
        }

        for file in os.listdir(item_directory):
            other = True
            extension = file.lower().split(".")[-1]

            for format, extensions in file_formats.items():
                if extension in extensions:
                    files[format].append(file)
                    other = False
                    break

            if other is True:
                files["other"].append(file)

        if len(files["wav"]) > 0 or len(files["mp3"]) > 0:
            self.items.append({
                "id": os.path.basename(item_directory),
                "path": item_directory,
                "files": files
            })
        else:
            self.errors["baroque_project"] = []
            self.add_errors("baroque_project", "warning", item_directory, os.path.basename(item_directory), "item directory does not appear to be an audio recording")

    def _parse_collection_id(self, item_id):
        return item_id.split("-")[0] # NOTE: Collection IDs are parsed from item IDs

    def _read_export(self, keys, rows, export_type):
        for row in rows:
            if export_type == ".csv":
                values = row
            elif export_type == ".xlsx":
                values = [c.value for c in row]
            yield dict(zip(keys, values))

    def check_values_exist(self, value_name, value_var, row_counter):
        if not value_var:
            print("SYSTEM ERROR: {} missing in row {} of metadata export".format(value_name, row_counter))
            sys.exit()

    def parse_metadata_export(self):
        """
        This function parses the metadata export supplied by BHL to the vendor (either a CSV or xlsx file)
        It stores the values of the DigFile Calc, CollectionTitle, ItemTitle, and ItemDate columns
        """
        metadata_export = self.metadata_export

        metadata = {"collections_ids": [], "items_ids": [], "item_metadata": {}}

        column_headers = {
            "item_id_column" : "DigFile Calc",
            "collection_title_column" : "COLLECTIONS::CollectionTitle",
            "item_title_column" : "ItemTitle",
            "item_date_column" : "ItemDate"
            }

        export_type = os.path.splitext(metadata_export)[1]
        if export_type in [".csv", ".xlsx"]:
            if export_type == ".csv":
                with open(metadata_export, "r", newline="") as f:
                    reader = csv.reader(f)
                    keys = next(reader)
                    rows = [row for row in reader]

            elif export_type == ".xlsx":
                # momentarily set warnings to ignore to hide openpyxl's "UserWarning: Workbook contains no default style" message
                warnings.simplefilter("ignore")
                workbook = load_workbook(metadata_export)
                warnings.simplefilter("default")
                sheet = workbook.active
                reader = sheet.rows
                keys = [c.value for c in next(reader)]
                rows = [row for row in reader]
                workbook.close()

            for header in column_headers.keys():
                if column_headers[header] not in keys:
                    print("SYSTEM WARNING: '{}' column not found in metadata export".format(column_headers[header]))
                    column_headers[header] = input("Type the column name for '{}' values and press Enter: ".format(column_headers[header]))
                    if column_headers[header] in keys:
                        print("SYSTEM REPORT: '{}' column found, proceeding as usual".format(column_headers[header]))
                    else:
                        print("SYSTEM REPORT: '{}' column not found, will not compare".format(column_headers[header]))

            item_id_column = column_headers["item_id_column"]
            collection_title_column = column_headers["collection_title_column"]
            item_title_column = column_headers["item_title_column"]
            item_date_column = column_headers["item_date_column"]
            row_counter = 1

            for row in self._read_export(keys, rows, export_type):
                row_counter += 1
                item_id = row.get(item_id_column)
                self.check_values_exist("item id (DigFile Calc)", item_id, row_counter)
                if self.source_type == "item" and item_id not in [item["id"] for item in self.items]:
                    continue
                collection_id = self._parse_collection_id(item_id)
                if self.source_type == "collection" and collection_id not in [collection["id"] for collection in self.collections]:
                    continue
                collection_title = row.get(collection_title_column)
                item_title = row.get(item_title_column)
                item_date = row.get(item_date_column)
                metadata["items_ids"].append(item_id)
                if collection_id not in metadata["collections_ids"]:
                    metadata["collections_ids"].append(collection_id)
                metadata["item_metadata"][item_id] = {
                    "collection_title": collection_title,
                    "item_title": item_title,
                    "item_date": item_date
                }

        # File type: neither csv nor xlsx
        else:
            print("SYSTEM ERROR: metadata export is an unexpected file type: {}".format(export_type))
            sys.exit()

        return metadata

    def add_errors(self, validation, error_type, path, id, error):
        """
        Add errors, organized by validation, to the BaroqueProject error attribute.
        Each validation (e.g., structure) has its own list of errors.
        Each error message is a dictionary with five key, value pairs:
        - validation : the validation step where the error was found
        - error_type : classification of the errror (either "requirement" or "warning")
        - path : where the error occurs (e.g., "C:\\Users\\person\\Desktop\\2019103\\12345")
        - id : what the error pertains to (e.g., "85429-SR-7")
        - error : a message describing the error (e.g., "empty directory", "empty file")
        """

        self.errors[validation].append({
                "validation": validation,
                "error_type": error_type, # NOTE: Needs to be "requirement" or "warning".
                "path": path,
                "id": id,
                "error": error
            })

import csv
import os
import sys
import warnings

from openpyxl import load_workbook

from baroque import defaults


class BaroqueProject(object):
    """
    Stores details about the current project.

    It takes a path to a source directory and a path to a destination directory:
    - the source directory can be either a shipment-level, collection-level, or item-level directory.
    - the destination directory stores reports generated by baroque.

    It characterizes the source directory level by analyzing what is inside the source directory.
    Then, it parses the source directory recursively down to the item-level.

    An initial instantiation of this object might look like:
    {
        "source": source_directory,
        "destination": destination_directory,
        "errors": {},
        "shipment" : [
            {
            "id": "",
            "path": "",
            }
        ],
        "collections" : [
            {
            "id": "",
            "path": "",
            }
        ],
        "items": [
            {
            "id": "",
            "path": "",
            "files": {
                "wav": [],
                "mp3": [],
                "jpg": [],
                "xml": [],
                "md5": [],
                "txt": [],
                "other": []
                },
            }
        ],
    }
    """

    def __init__(self, source_directory, destination_directory, metadata_export):
        if not os.path.exists(source_directory):
            print("SYSTEM ERROR: source_directory does not exist: {}".format(source_directory))
            sys.exit()
        if not os.path.exists(destination_directory):
            print("SYSTEM WARNING: destination_directory {} does not exist. Using default directory {}".format(destination_directory, defaults.REPORTS_DIR))
            if not os.path.exists(defaults.REPORTS_DIR):
                os.mkdir(defaults.REPORTS_DIR)
            destination_directory = defaults.REPORTS_DIR
        if not os.path.exists(metadata_export):
            print("SYSTEM ERROR: metadata export does not exist: {}".format(metadata_export))
            sys.exit()

        self.source_directory = source_directory
        self.destination_directory = destination_directory
        self.metadata_export = metadata_export

        self.shipment = []
        self.collections = []
        self.items = []
        self.errors = {}

        self.source_type = self.characterize_source_directory()
        print("SYSTEM REPORT: source_directory is {}".format(self.source_type))
        if self.source_type == "shipment":
            self.parse_shipment(source_directory)
        elif self.source_type == "collection":
            self.parse_collection(source_directory)
        elif self.source_type == "item":
            self.parse_item(source_directory)

        self.metadata = self.parse_metadata_export()

    def characterize_source_directory(self):
        """
        Characterize the source directory level by analyzing what is inside the source directory.
        """
        character_directory_name = os.path.basename(self.source_directory)
        character_directory_dirs = []
        character_directory_files = []

        # Loop the source directory to get files and sub-directories and append their name into respective lists
        for dir_entry in os.scandir(self.source_directory):
            if dir_entry.is_file():
                character_directory_files.append(str(dir_entry.name))
            elif dir_entry.is_dir():
                character_directory_dirs.append(str(dir_entry.name))

        # Return the source directory level as "item", if the lists show the source directory
        # has files and does not have sub-directories
        if len(character_directory_files) > 0 and len(character_directory_dirs) == 0:
            return "item"

        elif len(character_directory_dirs) > 0:
            # Return the source directory level as "collection", if the lists show the source directory
            # has sub-directories and sub-directory names start with the source directory name
            if any([directory.startswith(character_directory_name) for directory in character_directory_dirs]):
                return "collection"

            # Return the source directory level as "shipment", if the lists show the source directory
            # has sub-directories and sub-directory names do not start with the source directory name
            else:
                return "shipment"

        # Exit when the source directory does not have files nor sub-directories
        else:
            print("SYSTEM ERROR: source_directory is empty")
            sys.exit()

    def parse_shipment(self, shipment_directory):
        """
        Take a shipment-level directory path, parse its id and directory path,
        and add a dictionary to the "shipment" attribute.
        Then, loop the shipment-level directory for collection-level directories,
        and run "parse_collection" method on each collection-level directory.
        """
        self.shipment.append({
            "id": os.path.basename(shipment_directory),
            "path": shipment_directory
        })

        for dir_entry in os.scandir(shipment_directory):
            if dir_entry.is_dir():
                self.parse_collection(dir_entry.path)

    def parse_collection(self, collection_directory):
        """
        Take a collection-level directory path, parse its id and directory path,
        and add a dictionary to the "collections" attribute.
        Then, loop the directory for collection-level directories,
        and run "parse_item" method on each item-level directory.
        """
        self.collections.append({
            "id": os.path.basename(collection_directory),
            "path": collection_directory
        })

        for dir_entry in os.scandir(collection_directory):
            if dir_entry.is_dir():
                self.parse_item(dir_entry.path)

    def parse_item(self, item_directory):
        """
        Take a item-level directory path, parse id, directory path, files by their file formats,
        and add a dictionary to the "items" attribute.
        """
        files = {"wav": [], "mp3": [], "jpg": [], "xml": [], "md5": [], "txt": [], "other": []}
        file_formats = {
            "wav": ["wav", "wave"],
            "mp3": ["mp3"],
            "jpg": ["jpg", "jpeg", "jpe", "jif", "jfif", "jfi"],
            "xml": ["xml"],
            "md5": ["md5"],
            "txt": ["txt"]
        }

        for file in os.listdir(item_directory):
            other = True
            extension = file.lower().split(".")[-1]

            for format, extensions in file_formats.items():
                if extension in extensions:
                    files[format].append(file)
                    other = False
                    break

            if other is True:
                files["other"].append(file)

        if len(files["wav"]) > 0 or len(files["mp3"]) > 0:
            self.items.append({
                "id": os.path.basename(item_directory),
                "path": item_directory,
                "files": files
            })
        else:
            self.errors["baroque_project"] = []
            self.add_errors("baroque_project", "warning", item_directory, os.path.basename(item_directory), "item directory does not appear to be an audio recording")

    def _parse_collection_id(self, item_id):
        return item_id.split("-")[0] # NOTE: Collection IDs are parsed from item IDs

    def _read_export(self, keys, rows, export_type):
        for row in rows:
            if export_type == ".csv":
                values = row
            elif export_type == ".xlsx":
                values = [c.value for c in row]
            yield dict(zip(keys, values))

    def check_values_exist(self, value_name, value_var, row_counter):
        if not value_var:
            print("SYSTEM ERROR: {} missing in row {} of metadata export".format(value_name, row_counter))
            sys.exit()

    def parse_metadata_export(self):
        """
        This function parses the metadata export supplied by BHL to the vendor (either a CSV or xlsx file)
        It stores the values of the DigFile Calc, CollectionTitle, ItemTitle, and ItemDate columns
        """
        metadata_export = self.metadata_export

        metadata = {"collections_ids": [], "items_ids": [], "item_metadata": {}}

        column_headers = {
            "item_id_column" : "DigFile Calc",
            "collection_title_column" : "COLLECTIONS::CollectionTitle",
            "item_title_column" : "ItemTitle",
            "item_date_column" : "ItemDate"
            }

        export_type = os.path.splitext(metadata_export)[1]
        if export_type in [".csv", ".xlsx"]:
            if export_type == ".csv":
                with open(metadata_export, "r", newline="") as f:
                    reader = csv.reader(f)
                    keys = next(reader)
                    rows = [row for row in reader]

            elif export_type == ".xlsx":
                # momentarily set warnings to ignore to hide openpyxl's "UserWarning: Workbook contains no default style" message
                warnings.simplefilter("ignore")
                workbook = load_workbook(metadata_export)
                warnings.simplefilter("default")
                sheet = workbook.active
                reader = sheet.rows
                keys = [c.value for c in next(reader)]
                rows = [row for row in reader]
                workbook.close()

            for header in column_headers.keys():
                if column_headers[header] not in keys:
                    print("SYSTEM WARNING: '{}' column not found in metadata export".format(column_headers[header]))
                    column_headers[header] = input("Type the column name for '{}' values and press Enter: ".format(column_headers[header]))
                    if column_headers[header] in keys:
                        print("SYSTEM REPORT: '{}' column found, proceeding as usual".format(column_headers[header]))
                    else:
                        print("SYSTEM REPORT: '{}' column not found, will not compare".format(column_headers[header]))

            item_id_column = column_headers["item_id_column"]
            collection_title_column = column_headers["collection_title_column"]
            item_title_column = column_headers["item_title_column"]
            item_date_column = column_headers["item_date_column"]
            row_counter = 1

            for row in self._read_export(keys, rows, export_type):
                row_counter += 1
                item_id = row.get(item_id_column)
                self.check_values_exist("item id (DigFile Calc)", item_id, row_counter)
                if self.source_type == "item" and item_id not in [item["id"] for item in self.items]:
                    continue
                collection_id = self._parse_collection_id(item_id)
                if self.source_type == "collection" and collection_id not in [collection["id"] for collection in self.collections]:
                    continue
                collection_title = row.get(collection_title_column)
                item_title = row.get(item_title_column)
                item_date = row.get(item_date_column)
                metadata["items_ids"].append(item_id)
                if collection_id not in metadata["collections_ids"]:
                    metadata["collections_ids"].append(collection_id)
                metadata["item_metadata"][item_id] = {
                    "collection_title": collection_title,
                    "item_title": item_title,
                    "item_date": item_date
                }

        # File type: neither csv nor xlsx
        else:
            print("SYSTEM ERROR: metadata export is an unexpected file type: {}".format(export_type))
            sys.exit()

        return metadata

    def add_errors(self, validation, error_type, path, id, error):
        """
        Add errors, organized by validation, to the BaroqueProject error attribute.
        Each validation (e.g., structure) has its own list of errors.
        Each error message is a dictionary with five key, value pairs:
        - validation : the validation step where the error was found
        - error_type : classification of the errror (either "requirement" or "warning")
        - path : where the error occurs (e.g., "C:\\Users\\person\\Desktop\\2019103\\12345")
        - id : what the error pertains to (e.g., "85429-SR-7")
        - error : a message describing the error (e.g., "empty directory", "empty file")
        """

        self.errors[validation].append({
                "validation": validation,
                "error_type": error_type, # NOTE: Needs to be "requirement" or "warning".
                "path": path,
                "id": id,
                "error": error
            })

Stores details about the current project.      It takes a path to a source directory and a path to a destination directory:     - the source directory can be either a shipment-level, collection-level, or item-level directory.     - the destination directory stores reports generated by baroque.      It characterizes the source directory level by analyzing what is inside the source directory.     Then, it parses the source directory recursively down to the item-level.      An initial instantiation of this object might look like:     {         "source": source_directory,         "destination": destination_directory,         "errors": {},         "shipment" : [             {             "id": "",             "path": "",             }         ],         "collections" : [             {             "id": "",             "path": "",             }         ],         "items": [             {             "id": "",             "path": "",             "files": {                 "wav": [],                 "mp3": [],                 "jpg": [],                 "xml": [],                 "md5": [],                 "txt": [],                 "other": []                 },             }         ],     } Characterize the source directory level by analyzing what is inside the source directory. # Loop the source directory to get files and sub-directories and append their name into respective lists # Return the source directory level as "item", if the lists show the source directory # has files and does not have sub-directories # Return the source directory level as "collection", if the lists show the source directory # has sub-directories and sub-directory names start with the source directory name # Return the source directory level as "shipment", if the lists show the source directory # has sub-directories and sub-directory names do not start with the source directory name # Exit when the source directory does not have files nor sub-directories Take a shipment-level directory path, parse its id and directory path,         and add a dictionary to the "shipment" attribute.         Then, loop the shipment-level directory for collection-level directories,         and run "parse_collection" method on each collection-level directory. Take a collection-level directory path, parse its id and directory path,         and add a dictionary to the "collections" attribute.         Then, loop the directory for collection-level directories,         and run "parse_item" method on each item-level directory. Take a item-level directory path, parse id, directory path, files by their file formats,         and add a dictionary to the "items" attribute. # NOTE: Collection IDs are parsed from item IDs This function parses the metadata export supplied by BHL to the vendor (either a CSV or xlsx file)         It stores the values of the DigFile Calc, CollectionTitle, ItemTitle, and ItemDate columns # momentarily set warnings to ignore to hide openpyxl's "UserWarning: Workbook contains no default style" message # File type: neither csv nor xlsx Add errors, organized by validation, to the BaroqueProject error attribute.         Each validation (e.g., structure) has its own list of errors.         Each error message is a dictionary with five key, value pairs:         - validation : the validation step where the error was found         - error_type : classification of the errror (either "requirement" or "warning")         - path : where the error occurs (e.g., "C:\\Users\\person\\Desktop\\2019103\\12345")         - id : what the error pertains to (e.g., "85429-SR-7")         - error : a message describing the error (e.g., "empty directory", "empty file") # NOTE: Needs to be "requirement" or "warning".