# NOTICE
# This software was produced for the U.S. Government under contract FA8702-21-C-0001,
# and is subject to the Rights in Data-General Clause 52.227-14, Alt. IV (DEC 2007)
# ©2021 The MITRE Corporation. All Rights Reserved.

'''
This module implements the case toolkit serialization methods
for serializing ontologies and json-ld data files.

Each serialized file is a binary file that contains three sections:
        The 8-byte identifier ("magic number"): "ontology" or "casedata"
        The python-pickled metadata dictionary
        The python-pickled Ontology or CaseData object

The python-pickled dictionary includes:
        source:      The path to the item that was serialized
        md5:         The md5 of the item that was serialized
        timestamp:   The creation date and time
        description: A user-supplied description

The reason for the metadata is because if there are multiple
serialized file for different versions of data, we can be sure
which one contains which data.

This file structure allows examination of the metadata without
having to read the entire object.
'''
import pickle
import pprint
import hashlib
import os

ONTOLOGY = 'ontology'
CASEDATA = 'casedata'

class DeserializeError(Exception):
    'Exception raised on deserializing a file not serialized by this toolkit'
    pass

def serialize(identifier, metadata, obj, output_filepath):
    '''
    Arguments:
        identifier       Eight-byte filetype (ontology or casedata)
        metadata         Dictionary of metadata (see the module doc block)
        obj              The __dict__ of the Ontology or CaseData object being serialized
        output_filepath  Full path to the output serialized file
    '''
    # Serialize and write
    with open(output_filepath, 'wb') as outfile:
        outfile.write(identifier.encode('ascii'))
        outfile.write(pickle.dumps(metadata))
        outfile.write(pickle.dumps(obj))


def deserialize(serialized_filepath):
    '''
    Arguments:
        serialized_filepath    Full path to input serialized file

    Return:
        3-tuple of (identifier, metadata, obj)

    Raise:
        DeserializeError if file was not serialized by this toolkit.
        other Exception on I/O errors, etc.
    '''
    with open(serialized_filepath, 'rb') as infile:
        identifier = _read_identifier(infile)
        metadata = pickle.load(infile)
        obj = pickle.load(infile)

    return identifier, metadata, obj


def get_metadata(serialized_filepath):
    '''
    Arguments:
        serialized_filepath    Full path to input serialized file

    Return:
        The metadata serialized into this file

    Raise:
        DeserializeError if file was not serialized by this toolkit.
        other Exception on I/O errors, etc.
    '''
    with open(serialized_filepath, 'rb') as infile:
        _identifier = _read_identifier(infile)
        metadata = pickle.load(infile)

    return metadata


def get_identifier(serialized_filepath):
    '''
    Arguments:
        serialized_filepath    Full path to serialized file

    Return:
        identifier    The 8-character file type (ontology or casedata)

    Raise:
        DeserializeError if file was not serialized by this toolkit.
        other Exception on I/O errors, etc.
    '''
    with open(serialized_filepath, 'rb') as infile:
        identifier = _read_identifier(infile)

    return identifier


def describe(serialized_filepath):
    '''
    Arguments:
        serialize_filepath   Full path to serialize file

    Return:
        A multi-line description of the file base on the metadata
    '''
    with open(serialized_filepath, 'rb') as infile:
        identifier = _read_identifier(infile)
        metadata = pickle.load(infile)

    lines = []
    lines.append('{} file'.format(identifier))
    lines.append('')
    comment = metadata.pop('comment', None)
    if comment:
        lines.append(comment)
        lines.append('')
    lines.append(pprint.pformat(metadata))
    return '\n'.join(lines)



def get_hash(filepath):
    '''
    Arguments:
       filepath   Full path to the jsonld data file

    Return:
        The hexlified md5 of the file
    '''
    if not os.path.isfile(filepath):
        raise Exception('filepath {} is not a file'.format(filepath))

    with open(filepath, 'rb') as infile:
        return hashlib.md5(infile.read()).hexdigest()


def get_hash_and_manifest(turtle_dirpath):
    '''
    Arguments:
        turtle_dirpath   Full path to the directory contining ontology turtle (.ttl) files

    Return: Tuple of two items
        The hexlified md5 of the directory
        The manifiest dictionary {filename:hexlified_md5}

    Note:
        The "md5 of the directory" is the md5 of the file that would result from
        concatenating all the .ttl files in the directory in alphabetical order.
    '''
    if not os.path.isdir(turtle_dirpath):
        raise Exception('turtle_dirpath {} is not a directory'.format(turtle_dirpath))

    dirhash = hashlib.md5()
    manifest = {}
    filenames = sorted(os.listdir(turtle_dirpath))

    for filename in filenames:
        if not filename.endswith('.ttl'):
            continue

        filepath = os.path.join(turtle_dirpath, filename)
        if not os.path.isfile(filepath):
            continue

        manifest[filename] = get_hash(filepath)
        with open(filepath, 'rb') as infile:
            dirhash.update(infile.read())

    if not manifest:
        raise Exception('turtle_dirpath {} does not contain any turtle (.ttl) files'.format(turtle_dirpath))

    return dirhash.hexdigest(), manifest


def _read_identifier(infile):
    '''
    Arguments:
        infile   File descriptor for serialized file, opened 'rb'

    Return:
        The eight-character identifier string

    Raise:
       DeserializeError  if file does not contain an expected 8-byte "magic" number
    '''
    # Read eight-byte "magic"
    try:
        identifier = infile.read(8).decode('ascii')
    except UnicodeDecodeError:
        raise DeserializeError('{} was not serialized by this toolkit'.format(infile.name))

    # Make sure "magic" is one of ours
    if identifier not in (ONTOLOGY, CASEDATA):
        raise DeserializeError('{} was not serialized by this toolkit'.format(infile.name))

    # Return "magic" string
    return identifier

# NOTICE
# This software was produced for the U.S. Government under contract FA8702-21-C-0001,
# and is subject to the Rights in Data-General Clause 52.227-14, Alt. IV (DEC 2007)
# ©2021 The MITRE Corporation. All Rights Reserved.

'''
This module implements the case toolkit serialization methods
for serializing ontologies and json-ld data files.

Each serialized file is a binary file that contains three sections:
        The 8-byte identifier ("magic number"): "ontology" or "casedata"
        The python-pickled metadata dictionary
        The python-pickled Ontology or CaseData object

The python-pickled dictionary includes:
        source:      The path to the item that was serialized
        md5:         The md5 of the item that was serialized
        timestamp:   The creation date and time
        description: A user-supplied description

The reason for the metadata is because if there are multiple
serialized file for different versions of data, we can be sure
which one contains which data.

This file structure allows examination of the metadata without
having to read the entire object.
'''
import pickle
import pprint
import hashlib
import os

ONTOLOGY = 'ontology'
CASEDATA = 'casedata'

class DeserializeError(Exception):
    'Exception raised on deserializing a file not serialized by this toolkit'
    pass

def serialize(identifier, metadata, obj, output_filepath):
    '''
    Arguments:
        identifier       Eight-byte filetype (ontology or casedata)
        metadata         Dictionary of metadata (see the module doc block)
        obj              The __dict__ of the Ontology or CaseData object being serialized
        output_filepath  Full path to the output serialized file
    '''
    # Serialize and write
    with open(output_filepath, 'wb') as outfile:
        outfile.write(identifier.encode('ascii'))
        outfile.write(pickle.dumps(metadata))
        outfile.write(pickle.dumps(obj))


def deserialize(serialized_filepath):
    '''
    Arguments:
        serialized_filepath    Full path to input serialized file

    Return:
        3-tuple of (identifier, metadata, obj)

    Raise:
        DeserializeError if file was not serialized by this toolkit.
        other Exception on I/O errors, etc.
    '''
    with open(serialized_filepath, 'rb') as infile:
        identifier = _read_identifier(infile)
        metadata = pickle.load(infile)
        obj = pickle.load(infile)

    return identifier, metadata, obj


def get_metadata(serialized_filepath):
    '''
    Arguments:
        serialized_filepath    Full path to input serialized file

    Return:
        The metadata serialized into this file

    Raise:
        DeserializeError if file was not serialized by this toolkit.
        other Exception on I/O errors, etc.
    '''
    with open(serialized_filepath, 'rb') as infile:
        _identifier = _read_identifier(infile)
        metadata = pickle.load(infile)

    return metadata


def get_identifier(serialized_filepath):
    '''
    Arguments:
        serialized_filepath    Full path to serialized file

    Return:
        identifier    The 8-character file type (ontology or casedata)

    Raise:
        DeserializeError if file was not serialized by this toolkit.
        other Exception on I/O errors, etc.
    '''
    with open(serialized_filepath, 'rb') as infile:
        identifier = _read_identifier(infile)

    return identifier


def describe(serialized_filepath):
    '''
    Arguments:
        serialize_filepath   Full path to serialize file

    Return:
        A multi-line description of the file base on the metadata
    '''
    with open(serialized_filepath, 'rb') as infile:
        identifier = _read_identifier(infile)
        metadata = pickle.load(infile)

    lines = []
    lines.append('{} file'.format(identifier))
    lines.append('')
    comment = metadata.pop('comment', None)
    if comment:
        lines.append(comment)
        lines.append('')
    lines.append(pprint.pformat(metadata))
    return '\n'.join(lines)



def get_hash(filepath):
    '''
    Arguments:
       filepath   Full path to the jsonld data file

    Return:
        The hexlified md5 of the file
    '''
    if not os.path.isfile(filepath):
        raise Exception('filepath {} is not a file'.format(filepath))

    with open(filepath, 'rb') as infile:
        return hashlib.md5(infile.read()).hexdigest()


def get_hash_and_manifest(turtle_dirpath):
    '''
    Arguments:
        turtle_dirpath   Full path to the directory contining ontology turtle (.ttl) files

    Return: Tuple of two items
        The hexlified md5 of the directory
        The manifiest dictionary {filename:hexlified_md5}

    Note:
        The "md5 of the directory" is the md5 of the file that would result from
        concatenating all the .ttl files in the directory in alphabetical order.
    '''
    if not os.path.isdir(turtle_dirpath):
        raise Exception('turtle_dirpath {} is not a directory'.format(turtle_dirpath))

    dirhash = hashlib.md5()
    manifest = {}
    filenames = sorted(os.listdir(turtle_dirpath))

    for filename in filenames:
        if not filename.endswith('.ttl'):
            continue

        filepath = os.path.join(turtle_dirpath, filename)
        if not os.path.isfile(filepath):
            continue

        manifest[filename] = get_hash(filepath)
        with open(filepath, 'rb') as infile:
            dirhash.update(infile.read())

    if not manifest:
        raise Exception('turtle_dirpath {} does not contain any turtle (.ttl) files'.format(turtle_dirpath))

    return dirhash.hexdigest(), manifest


def _read_identifier(infile):
    '''
    Arguments:
        infile   File descriptor for serialized file, opened 'rb'

    Return:
        The eight-character identifier string

    Raise:
       DeserializeError  if file does not contain an expected 8-byte "magic" number
    '''
    # Read eight-byte "magic"
    try:
        identifier = infile.read(8).decode('ascii')
    except UnicodeDecodeError:
        raise DeserializeError('{} was not serialized by this toolkit'.format(infile.name))

    # Make sure "magic" is one of ours
    if identifier not in (ONTOLOGY, CASEDATA):
        raise DeserializeError('{} was not serialized by this toolkit'.format(infile.name))

    # Return "magic" string
    return identifier

# NOTICE # This software was produced for the U.S. Government under contract FA8702-21-C-0001, # and is subject to the Rights in Data-General Clause 52.227-14, Alt. IV (DEC 2007) # ©2021 The MITRE Corporation. All Rights Reserved. This module implements the case toolkit serialization methods for serializing ontologies and json-ld data files.  Each serialized file is a binary file that contains three sections:         The 8-byte identifier ("magic number"): "ontology" or "casedata"         The python-pickled metadata dictionary         The python-pickled Ontology or CaseData object  The python-pickled dictionary includes:         source:      The path to the item that was serialized         md5:         The md5 of the item that was serialized         timestamp:   The creation date and time         description: A user-supplied description  The reason for the metadata is because if there are multiple serialized file for different versions of data, we can be sure which one contains which data.  This file structure allows examination of the metadata without having to read the entire object. Arguments:         identifier       Eight-byte filetype (ontology or casedata)         metadata         Dictionary of metadata (see the module doc block)         obj              The __dict__ of the Ontology or CaseData object being serialized         output_filepath  Full path to the output serialized file # Serialize and write Arguments:         serialized_filepath    Full path to input serialized file      Return:         3-tuple of (identifier, metadata, obj)      Raise:         DeserializeError if file was not serialized by this toolkit.         other Exception on I/O errors, etc. Arguments:         serialized_filepath    Full path to input serialized file      Return:         The metadata serialized into this file      Raise:         DeserializeError if file was not serialized by this toolkit.         other Exception on I/O errors, etc. Arguments:         serialized_filepath    Full path to serialized file      Return:         identifier    The 8-character file type (ontology or casedata)      Raise:         DeserializeError if file was not serialized by this toolkit.         other Exception on I/O errors, etc. Arguments:         serialize_filepath   Full path to serialize file      Return:         A multi-line description of the file base on the metadata Arguments:        filepath   Full path to the jsonld data file      Return:         The hexlified md5 of the file Arguments:         turtle_dirpath   Full path to the directory contining ontology turtle (.ttl) files      Return: Tuple of two items         The hexlified md5 of the directory         The manifiest dictionary {filename:hexlified_md5}      Note:         The "md5 of the directory" is the md5 of the file that would result from         concatenating all the .ttl files in the directory in alphabetical order. Arguments:         infile   File descriptor for serialized file, opened 'rb'      Return:         The eight-character identifier string      Raise:        DeserializeError  if file does not contain an expected 8-byte "magic" number # Read eight-byte "magic" # Make sure "magic" is one of ours # Return "magic" string